diff --git a/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql b/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql index 7911bc7581..2a10456ab0 100644 --- a/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql @@ -33,6 +33,7 @@ INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (145,'tendbcluster','dbconf','MySQL-5.6','my.cnf配置','MySQL-5.6','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'5.6_参数配置','2022-04-25 10:19:22','2023-12-22 04:16:53',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (144,'tendbcluster','dbconf','MySQL-5.7','my.cnf配置','MySQL-5.7','plat,app,module,cluster','cluster',1,1,0,NULL,5,365,0,'5.7配置','2022-04-25 10:19:22','2023-06-29 10:33:14',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (143,'tendbcluster','dbconf','MySQL-8.0','','MySQL-8.0','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'MySQL8.0配置','2022-06-02 17:27:34','2023-06-29 10:32:50',''); +INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (362,'tendbcluster','dbconf','RocksDB-5.7','','RocksDB-5.7','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'MySQL-RocksDB-5.7','2022-06-02 17:27:34','2024-12-05 16:26:29',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (147,'tendbcluster','dbconf','Spider-1','my.cnf配置','Spider 1.x','plat,app,module,cluster','cluster',1,1,0,NULL,5,365,0,'Spider 1.x 接入层','2022-04-25 10:19:22','2023-06-29 10:33:02',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (148,'tendbcluster','dbconf','Spider-3','my.cnf配置','Spider 3.x推荐版本','plat,app,module,cluster','cluster',1,1,0,NULL,5,365,0,'Spider 3.x 接入层','2022-04-25 10:19:22','2024-09-04 10:36:16',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (361,'tendbcluster','dbconf','Spider-3.5','my.cnf配置','Spider 3.5.x','plat,app,module,cluster','cluster',1,1,0,NULL,5,365,0,'Spider 3.x 接入层','2022-04-25 10:19:22','2023-06-29 10:33:05',''); @@ -40,6 +41,7 @@ INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (353,'tendbcluster','dbconf','Spider-3.7','my.cnf配置','Spider 3.7.x','plat,app,module,cluster','cluster',1,1,0,NULL,5,365,0,'Spider 3.x 接入层','2022-04-25 10:19:22','2023-06-29 10:33:05',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (355,'tendbcluster','dbconf','Spider-3.8','my.cnf配置','Spider 3.8.x','plat,app,module,cluster','cluster',1,1,0,NULL,5,365,0,'Spider 3.x 接入层','2022-04-25 10:19:22','2023-06-29 10:33:05',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (177,'tendbcluster','dbconf','Tdbctl','my.cnf配置','tdbctl中控配置','plat,app,module,cluster','cluster',1,1,0,NULL,5,365,0,'tdbctl中控配置','2022-04-25 10:19:22','2023-05-10 19:35:47',''); +INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (364,'tendbcluster','dbconf','TokuDB-5.6','','TokuDB-5.6','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'MySQL-TokuDB-5.6','2022-06-02 17:27:34','2024-12-05 16:26:29',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (270,'tendbcluster','dbconf','TXSQL-8.0','','TXSQL-8.0','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'MySQL8.0-txsql配置','2022-06-02 17:27:34','2023-11-14 15:09:13',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (192,'tendbcluster','deploy','deploy_info','部署配置',NULL,'plat,app,module,cluster','',0,1,0,NULL,5,365,0,NULL,'2023-03-09 17:40:06','2023-03-20 21:40:05',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (194,'tendbcluster','sys','sysfile','系统配置',NULL,'plat','',1,1,0,NULL,5,365,0,NULL,'2023-03-09 17:40:06','2023-03-20 21:40:05',''); @@ -108,7 +110,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16347,'tendbcluster','backup','dbbackup.ini','LogicalBackup.DefaultsFile','STRING','','','STRING',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 09:50:12',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16358,'tendbcluster','backup','dbbackup.ini','LogicalBackup.DisableCompress','STRING','false','false | true','BOOL',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-24 21:45:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23728,'tendbcluster','backup','dbbackup.ini','LogicalBackup.ExcludeDatabases','STRING','mysql,sys,test,information_schema,performance_schema,db_infobase','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:40:11',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16346,'tendbcluster','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','--skip-definer','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-25 09:59:19',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16346,'tendbcluster','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14164,'tendbcluster','backup','dbbackup.ini','LogicalBackup.FlushRetryCount','INT','3','','INT',1,0,0,0,0,NULL,'','',-1,NULL,'','2023-03-09 17:36:33','2023-03-22 12:23:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14168,'tendbcluster','backup','dbbackup.ini','LogicalBackup.Regex','STRING','{{.LogicalBackup.Regex}}','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-03-09 17:36:33','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25532,'tendbcluster','backup','dbbackup.ini','LogicalBackup.Tables','STRING','*','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:42:11',0); @@ -118,7 +120,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23726,'tendbcluster','backup','dbbackup.ini','LogicalBackupMysqldump.ExtraOpt','STRING','','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-11-11 10:22:39',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16343,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.DefaultsFile','STRING','{{.PhysicalBackup.DefaultsFile}}','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 10:24:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23705,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.DisableSlaveMultiThread','STRING','true','true | false','BOOL',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:35:52',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16354,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','--safe-slave-backup-timeout=60','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-24 21:46:16',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16354,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23716,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.MaxMyisamTables','INT','10','[0, 9999999]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:35:52',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16363,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.Threads','INT','2','[0, 8]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-10 15:21:23',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16348,'tendbcluster','backup','dbbackup.ini','PhysicalBackup.Throttle','INT','200','[0, 500]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-04-23 12:28:23',0); @@ -327,6 +329,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14361,'tendbcluster','dbconf','MySQL-5.6','mysqld.disconnect_on_expired_password','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'This variable controls how the server handles clients with expired passwords','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14362,'tendbcluster','dbconf','MySQL-5.6','mysqld.div_precision_increment','INT','4','[0,30]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This variable indicates the number of digits by which to increase the scale of the result of division operations performed with the operator','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14363,'tendbcluster','dbconf','MySQL-5.6','mysqld.end_markers_in_json','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Whether optimizer JSON output should add end markers. ','2023-03-09 17:36:33','2023-03-09 17:36:33',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25853,'tendbcluster','dbconf','MySQL-5.6','mysqld.enforce-gtid-consistency','STRING','OFF','OFF | ON','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Depending on the value of this variable, the server enforces GTID consistency by allowing execution of only statements that can be safely logged using a GTID','2023-03-09 17:57:45','2024-11-18 16:48:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14364,'tendbcluster','dbconf','MySQL-5.6','mysqld.eq_range_index_dive_limit','INT','200','[1,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The optimizer will use existing index statistics instead of doing index dives for equality ranges if the number of equality ranges for the index is larger than or equal to this number. If set to 0, index dives are always used.','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14365,'tendbcluster','dbconf','MySQL-5.6','mysqld.event_scheduler','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This variable indicates the status of the Event Scheduler','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14366,'tendbcluster','dbconf','MySQL-5.6','mysqld.expire_logs_days','INT','60','[0, 99]','RANGE',1,0,0,0,1,NULL,NULL,'',0,'',NULL,'2023-03-09 17:36:33','2023-04-26 20:27:21',0); @@ -336,6 +339,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14370,'tendbcluster','dbconf','MySQL-5.6','mysqld.ft_min_word_len','INT','4','[1,84]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'The minimum length of the word to be included in a FULLTEXT index. Note: FULLTEXT indexes must be rebuilt after changing this variable.','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14371,'tendbcluster','dbconf','MySQL-5.6','mysqld.ft_query_expansion_limit','INT','20','[0,1000]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'Number of best matches to use for query expansion','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14372,'tendbcluster','dbconf','MySQL-5.6','mysqld.group_concat_max_len','INT','1024','[4,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The maximum permitted result length in bytes for the GROUP_CONCAT() function','2023-03-09 17:36:33','2023-03-09 17:36:33',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25851,'tendbcluster','dbconf','MySQL-5.6','mysqld.gtid_mode','STRING','OFF','OFF | ON','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Controls whether GTID based logging is enabled and what type of transactions the logs can contain','2023-03-09 17:57:45','2024-11-18 16:48:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14373,'tendbcluster','dbconf','MySQL-5.6','mysqld.host_cache_size','INT','643','[0,65535]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The size of the internal host cache','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14375,'tendbcluster','dbconf','MySQL-5.6','mysqld.innodb_adaptive_flushing','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Specifies whether to dynamically adjust the rate of flushing dirty pages in the InnoDB buffer pool based on the workload.','2023-03-09 17:36:33','2023-03-09 17:36:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (14376,'tendbcluster','dbconf','MySQL-5.6','mysqld.innodb_adaptive_flushing_lwm','INT','10','[0,70]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Defines the low water mark representing percentage of redo log capacity at which adaptive flushing is enabled','2023-03-09 17:36:33','2023-03-09 17:36:33',0); @@ -2647,6 +2651,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15802,'tendbcluster','dbconf','Tdbctl','mysqldump.default-character-set','STRING','{{.Mysqld.CharacterSetServer}}','LATIN1| UTF8| GBK| UTF8MB4 |{{mysqld.character_set_server}}','ENUM',2,0,0,0,0,'{{mysqld.character_set_server}}',NULL,NULL,-1,NULL,NULL,'2023-03-09 17:57:45','2023-05-10 19:35:56',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25565,'tendbcluster','dbconf','TokuDB-5.6','mysqld.default_storage_engine','STRING','Tokudb','Tokudb','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2023-12-27 07:14:28',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25574,'tendbcluster','dbconf','TokuDB-5.6','mysqld.innodb_buffer_pool_size','INT','200M','[100m, 64000m]','BYTES',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-11-15 12:15:56',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25855,'tendbcluster','dbconf','TokuDB-5.6','mysqld.plugin-load','STRING','TokuDB=ha_tokudb.so;TokuDB_trx=ha_tokudb.so;TokuDB_locks=ha_tokudb.so;TokuDB_lock_waits=ha_tokudb.so;TokuDB_file_map=ha_tokudb.so;TokuDB_fractal_tree_info=ha_tokudb.so;TokuDB_fractal_tree_block_map=ha_tokudb.so','','STRING',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2023-12-27 07:14:28',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25566,'tendbcluster','dbconf','TokuDB-5.6','mysqld.tokudb_cache_size','STRING','16000M','[100m,12800m]','BYTES',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-11-15 14:37:17',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25567,'tendbcluster','dbconf','TokuDB-5.6','mysqld.tokudb_commit_sync','STRING','0','0 | 1','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-11-15 14:37:17',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25571,'tendbcluster','dbconf','TokuDB-5.6','mysqld.tokudb_data_dir','STRING','{{.Mysqld.Datadir}}/tokudb/data','','',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-11-15 14:39:00',0); diff --git a/dbm-services/common/db-config/assets/migrations/000020_tendbha_data.up.sql b/dbm-services/common/db-config/assets/migrations/000020_tendbha_data.up.sql index e538a16b0b..69befbab49 100644 --- a/dbm-services/common/db-config/assets/migrations/000020_tendbha_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000020_tendbha_data.up.sql @@ -33,6 +33,8 @@ INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (9,'tendbha','dbconf','MySQL-5.6','my.cnf配置','MySQL-5.6','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'5.6_参数配置','2022-04-25 10:19:22','2023-08-11 13:12:14',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (10,'tendbha','dbconf','MySQL-5.7','my.cnf配置','MySQL-5.7','plat,app,module,cluster','cluster',1,1,0,NULL,5,365,0,'5.7_参数配置','2022-04-25 10:19:22','2023-06-29 10:34:49',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (40,'tendbha','dbconf','MySQL-8.0','','MySQL-8.0','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'MySQL8.0配置','2022-06-02 17:27:34','2023-08-11 13:12:18',''); +INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (363,'tendbha','dbconf','RocksDB-5.7','','RocksDB-5.7','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'MySQL-RocksDB-5.7','2022-06-02 17:27:34','2024-12-05 16:26:29',''); +INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (365,'tendbha','dbconf','TokuDB-5.6','','TokuDB-5.6','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'MySQL-TokuDB-5.6','2022-06-02 17:27:34','2024-12-05 16:26:29',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (269,'tendbha','dbconf','TXSQL-8.0','','TXSQL-8.0','plat,app,module,cluster','cluster',1,1,0,'',0,0,0,'MySQL8.0-txsql配置','2022-06-02 17:27:34','2023-11-14 15:09:06',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (6,'tendbha','dbha','dbha','DBHA切换配置',NULL,'plat,app,city,module,cluster','',1,1,0,NULL,5,365,0,NULL,'2022-04-25 10:19:22','2023-03-20 21:40:05',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (4,'tendbha','deploy','deploy_info','部署配置',NULL,'plat,app,module,cluster','',0,1,0,NULL,5,365,0,NULL,'2022-04-25 10:19:22','2023-03-20 21:40:05',''); @@ -106,7 +108,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13010,'tendbha','backup','dbbackup.ini','LogicalBackup.DefaultsFile','STRING','','','STRING',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 09:50:12',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16311,'tendbha','backup','dbbackup.ini','LogicalBackup.DisableCompress','STRING','false','false | true','BOOL',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-24 21:45:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23729,'tendbha','backup','dbbackup.ini','LogicalBackup.ExcludeDatabases','STRING','mysql,sys,test,information_schema,performance_schema,db_infobase','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:40:11',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16312,'tendbha','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','--skip-definer','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-25 09:59:19',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16312,'tendbha','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13009,'tendbha','backup','dbbackup.ini','LogicalBackup.FlushRetryCount','INT','3','','INT',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-03-22 12:23:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13007,'tendbha','backup','dbbackup.ini','LogicalBackup.Regex','STRING','{{.LogicalBackup.Regex}}','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25533,'tendbha','backup','dbbackup.ini','LogicalBackup.Tables','STRING','*','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:42:11',0); @@ -116,7 +118,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23724,'tendbha','backup','dbbackup.ini','LogicalBackupMysqldump.ExtraOpt','STRING','','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-11-11 10:22:39',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16293,'tendbha','backup','dbbackup.ini','PhysicalBackup.DefaultsFile','STRING','{{.PhysicalBackup.DefaultsFile}}','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 10:24:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23704,'tendbha','backup','dbbackup.ini','PhysicalBackup.DisableSlaveMultiThread','STRING','true','true | false','BOOL',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:35:58',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16294,'tendbha','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','--safe-slave-backup-timeout=60','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-24 21:46:16',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16294,'tendbha','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23717,'tendbha','backup','dbbackup.ini','PhysicalBackup.MaxMyisamTables','INT','10','[0, 9999999]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-15 21:03:46',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16290,'tendbha','backup','dbbackup.ini','PhysicalBackup.Threads','INT','2','[0, 8]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-10 15:21:23',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16292,'tendbha','backup','dbbackup.ini','PhysicalBackup.Throttle','INT','200','[0, 500]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-04-23 12:28:18',0); @@ -325,6 +327,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5201,'tendbha','dbconf','MySQL-5.6','mysqld.disconnect_on_expired_password','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'This variable controls how the server handles clients with expired passwords','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5202,'tendbha','dbconf','MySQL-5.6','mysqld.div_precision_increment','INT','4','[0,30]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This variable indicates the number of digits by which to increase the scale of the result of division operations performed with the operator','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5203,'tendbha','dbconf','MySQL-5.6','mysqld.end_markers_in_json','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Whether optimizer JSON output should add end markers. ','2022-06-16 21:39:26','2022-10-20 12:26:09',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25854,'tendbha','dbconf','MySQL-5.6','mysqld.enforce-gtid-consistency','STRING','OFF','OFF | ON','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Depending on the value of this variable, the server enforces GTID consistency by allowing execution of only statements that can be safely logged using a GTID','2023-03-09 17:57:45','2024-11-18 16:48:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5204,'tendbha','dbconf','MySQL-5.6','mysqld.eq_range_index_dive_limit','INT','200','[1,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The optimizer will use existing index statistics instead of doing index dives for equality ranges if the number of equality ranges for the index is larger than or equal to this number. If set to 0, index dives are always used.','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5205,'tendbha','dbconf','MySQL-5.6','mysqld.event_scheduler','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This variable indicates the status of the Event Scheduler','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (85,'tendbha','dbconf','MySQL-5.6','mysqld.expire_logs_days','INT','60','[0, 99]','RANGE',1,0,0,0,1,NULL,NULL,'',0,'',NULL,'2022-04-25 10:00:47','2023-04-26 20:27:21',0); @@ -334,6 +337,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5209,'tendbha','dbconf','MySQL-5.6','mysqld.ft_min_word_len','INT','4','[1,84]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'The minimum length of the word to be included in a FULLTEXT index. Note: FULLTEXT indexes must be rebuilt after changing this variable.','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5210,'tendbha','dbconf','MySQL-5.6','mysqld.ft_query_expansion_limit','INT','20','[0,1000]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'Number of best matches to use for query expansion','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5211,'tendbha','dbconf','MySQL-5.6','mysqld.group_concat_max_len','INT','1024','[4,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The maximum permitted result length in bytes for the GROUP_CONCAT() function','2022-06-16 21:39:26','2022-10-20 12:26:09',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25852,'tendbha','dbconf','MySQL-5.6','mysqld.gtid_mode','STRING','OFF','OFF | ON','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Controls whether GTID based logging is enabled and what type of transactions the logs can contain','2023-03-09 17:57:45','2024-11-18 16:48:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5212,'tendbha','dbconf','MySQL-5.6','mysqld.host_cache_size','INT','643','[0,65535]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The size of the internal host cache','2022-06-16 21:39:26','2022-10-20 12:26:09',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (86,'tendbha','dbconf','MySQL-5.6','mysqld.init_connect','STRING','','','',2,0,0,0,0,NULL,NULL,NULL,-1,NULL,'','2022-04-25 10:00:47','2024-01-18 17:36:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (5213,'tendbha','dbconf','MySQL-5.6','mysqld.innodb_adaptive_flushing','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Specifies whether to dynamically adjust the rate of flushing dirty pages in the InnoDB buffer pool based on the workload.','2022-06-16 21:39:26','2022-10-20 12:26:09',0); @@ -1028,6 +1032,17 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25545,'tendbha','dbconf','RocksDB-5.7','mysqld.rocksdb_max_total_wal_size','INT','4G','[256m, 65536m]','BYTES',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-11-15 12:15:56',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25547,'tendbha','dbconf','RocksDB-5.7','mysqld.rocksdb_strict_collation_check','STRING','off','off | on','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2023-12-27 07:14:28',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25546,'tendbha','dbconf','RocksDB-5.7','mysqld.rocksdb_table_cache_numshardbits','INT','6','[1, 128]','RANGE',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2023-12-27 07:14:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25857,'tendbha','dbconf','TokuDB-5.6','mysqld.default_storage_engine','STRING','Tokudb','Tokudb','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25866,'tendbha','dbconf','TokuDB-5.6','mysqld.innodb_buffer_pool_size','INT','200M','[100m, 64000m]','BYTES',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25856,'tendbha','dbconf','TokuDB-5.6','mysqld.plugin-load','STRING','TokuDB=ha_tokudb.so;TokuDB_trx=ha_tokudb.so;TokuDB_locks=ha_tokudb.so;TokuDB_lock_waits=ha_tokudb.so;TokuDB_file_map=ha_tokudb.so;TokuDB_fractal_tree_info=ha_tokudb.so;TokuDB_fractal_tree_block_map=ha_tokudb.so','','STRING',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25858,'tendbha','dbconf','TokuDB-5.6','mysqld.tokudb_cache_size','STRING','16000M','[100m,12800m]','BYTES',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25859,'tendbha','dbconf','TokuDB-5.6','mysqld.tokudb_commit_sync','STRING','0','0 | 1','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25863,'tendbha','dbconf','TokuDB-5.6','mysqld.tokudb_data_dir','STRING','{{.Mysqld.Datadir}}/tokudb/data','','',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25861,'tendbha','dbconf','TokuDB-5.6','mysqld.tokudb_fsync_log_period','INT','1000','[0, 4294967295]','RANGE',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25860,'tendbha','dbconf','TokuDB-5.6','mysqld.tokudb_fs_reserve_percent','INT','0','[0, 100]','RANGE',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25862,'tendbha','dbconf','TokuDB-5.6','mysqld.tokudb_lock_timeout','INT','50000','[0, 100000000]','RANGE',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25864,'tendbha','dbconf','TokuDB-5.6','mysqld.tokudb_log_dir','STRING','{{.Mysqld.Datadir}}/tokudb/log','','',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25865,'tendbha','dbconf','TokuDB-5.6','mysqld.tokudb_tmp_dir','STRING','{{.Mysqld.Datadir}}/tmp','','',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2022-04-25 10:00:47','2024-12-05 16:33:38',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (18227,'tendbha','dbconf','TXSQL-8.0','mysql.default-character-set','STRING','{{.Mysqld.CharacterSetServer}}','LATIN1| UTF8| GBK| UTF8MB4 |{{mysqld.character_set_server}}','ENUM',2,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2023-11-14 15:18:34','2023-11-14 15:18:34',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (18228,'tendbha','dbconf','TXSQL-8.0','mysql.no-auto-rehash','STRING','true','true | false','ENUM',2,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2023-11-14 15:18:34','2023-11-14 15:18:34',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (18229,'tendbha','dbconf','TXSQL-8.0','mysqld.automatic_sp_privileges','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'whether automatically grants the EXECUTE and ALTER ROUTINE privileges to the creator of a stored routine','2023-11-14 15:18:34','2023-11-14 15:18:34',0); diff --git a/dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql b/dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql index b02cf4f656..beefdcb56f 100644 --- a/dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql @@ -99,7 +99,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16389,'tendbsingle','backup','dbbackup.ini','LogicalBackup.DefaultsFile','STRING','','','STRING',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 09:50:12',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16400,'tendbsingle','backup','dbbackup.ini','LogicalBackup.DisableCompress','STRING','false','false | true','BOOL',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-24 21:45:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23734,'tendbsingle','backup','dbbackup.ini','LogicalBackup.ExcludeDatabases','STRING','mysql,sys,test,information_schema,performance_schema,db_infobase','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:40:11',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16388,'tendbsingle','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','--skip-definer','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2023-05-25 09:59:19',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16388,'tendbsingle','backup','dbbackup.ini','LogicalBackup.ExtraOpt','STRING','','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13051,'tendbsingle','backup','dbbackup.ini','LogicalBackup.FlushRetryCount','INT','3','','INT',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-03-22 12:23:33',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13055,'tendbsingle','backup','dbbackup.ini','LogicalBackup.Regex','STRING','{{.LogicalBackup.Regex}}','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25534,'tendbsingle','backup','dbbackup.ini','LogicalBackup.Tables','STRING','*','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-06 12:42:11',0); @@ -109,7 +109,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23732,'tendbsingle','backup','dbbackup.ini','LogicalBackupMysqldump.ExtraOpt','STRING','','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2023-05-24 21:45:24','2024-11-11 10:22:39',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16385,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.DefaultsFile','STRING','{{.PhysicalBackup.DefaultsFile}}','','',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-25 10:24:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23702,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.DisableSlaveMultiThread','STRING','true','true | false','BOOL',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:36:04',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16396,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','--safe-slave-backup-timeout=60','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-24 21:46:16',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16396,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.ExtraOpt','STRING','','','',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-12-03 15:49:24',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23718,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.MaxMyisamTables','INT','10','[0, 9999999]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-07-04 14:35:52',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16405,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.Threads','INT','2','[0, 8]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-05-10 15:21:23',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16390,'tendbsingle','backup','dbbackup.ini','PhysicalBackup.Throttle','INT','200','[0, 500]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-04-23 12:28:34',0); diff --git a/dbm-services/common/db-config/assets/migrations/000041_doris_data.up.sql b/dbm-services/common/db-config/assets/migrations/000041_doris_data.up.sql index 0d0f28bf4a..fb8445e812 100644 --- a/dbm-services/common/db-config/assets/migrations/000041_doris_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000041_doris_data.up.sql @@ -1050,7 +1050,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24449,'doris','dbconf','2.1.5','fe.tablet_rebalancer_type','STRING','BeLoad','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','tablet_rebalancer_type','2023-01-13 11:26:00','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24450,'doris','dbconf','2.1.5','fe.tablet_repair_delay_factor_second','STRING','60','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','tablet_repair_delay_factor_second','2023-01-13 11:26:00','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24451,'doris','dbconf','2.1.5','fe.tablet_stat_update_interval_second','STRING','300','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','tablet_stat_update_interval_second','2023-01-13 11:26:00','2023-04-17 17:10:41',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24452,'doris','dbconf','2.1.5','fe.table_name_length_limit','STRING','64','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','table_name_length_limit','2023-01-13 11:26:00','2023-04-17 17:10:41',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24452,'doris','dbconf','2.1.5','fe.table_name_length_limit','STRING','128','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','table_name_length_limit','2023-01-13 11:26:00','2024-12-02 15:12:20',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24453,'doris','dbconf','2.1.5','fe.thrift_backlog_num','STRING','1024','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','thrift_backlog_num','2023-01-13 11:26:00','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24454,'doris','dbconf','2.1.5','fe.thrift_client_timeout_ms','STRING','0','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','thrift_client_timeout_ms','2023-01-13 11:26:00','2023-04-17 17:10:41',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24455,'doris','dbconf','2.1.5','fe.thrift_server_max_worker_threads','STRING','4096','NULL','',2,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','thrift_server_max_worker_threads','2023-01-13 11:26:00','2023-04-17 17:10:41',0); diff --git a/dbm-services/common/db-resource/internal/controller/controller.go b/dbm-services/common/db-resource/internal/controller/controller.go index 578bd0539e..954dc697fe 100644 --- a/dbm-services/common/db-resource/internal/controller/controller.go +++ b/dbm-services/common/db-resource/internal/controller/controller.go @@ -76,6 +76,7 @@ func (c *BackStageHandler) RegisterRouter(engine *gin.Engine) { { r.POST("/cc/module/check", c.RunModuleCheck) r.POST("/cc/async", c.RunAsyncCmdb) + r.POST("/cc/sync/os/info", c.SyncOsInfo) } } @@ -96,3 +97,12 @@ func (c BackStageHandler) RunAsyncCmdb(r *gin.Context) { } c.SendResponse(r, nil, "async success") } + +// SyncOsInfo sync os info +func (c BackStageHandler) SyncOsInfo(r *gin.Context) { + err := task.SyncOsNameInfo() + if err != nil { + logger.Error("SyncOsNameInfo failed %v", err) + } + c.SendResponse(r, nil, "async success") +} diff --git a/dbm-services/common/db-resource/internal/controller/manage/rs_import.go b/dbm-services/common/db-resource/internal/controller/manage/rs_import.go index a2d023954d..b094af4df0 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/rs_import.go +++ b/dbm-services/common/db-resource/internal/controller/manage/rs_import.go @@ -23,6 +23,7 @@ import ( "dbm-services/common/db-resource/internal/svr/bk" "dbm-services/common/db-resource/internal/svr/task" "dbm-services/common/db-resource/internal/svr/yunti" + "dbm-services/common/db-resource/internal/util" "dbm-services/common/go-pubpkg/cc.v3" "dbm-services/common/go-pubpkg/errno" "dbm-services/common/go-pubpkg/logger" @@ -377,7 +378,7 @@ func (p ImportMachParam) transHostInfoToDbModule(h *cc.Host, bkCloudId int, labe OsType: model.ConvertOsTypeToHuman(osType), OsBit: h.BkOsBit, OsVerion: h.BkOsVersion, - OsName: strings.TrimSpace(strings.ToLower(strings.ReplaceAll(h.OSName, " ", ""))), + OsName: util.CleanOsName(h.OSName), UpdateTime: time.Now(), CreateTime: time.Now(), } diff --git a/dbm-services/common/db-resource/internal/model/TbRpDailySnapShot.go b/dbm-services/common/db-resource/internal/model/TbRpDailySnapShot.go index af6b8f845d..878a2a8507 100644 --- a/dbm-services/common/db-resource/internal/model/TbRpDailySnapShot.go +++ b/dbm-services/common/db-resource/internal/model/TbRpDailySnapShot.go @@ -83,7 +83,7 @@ func SyncDbRpDailySnapShot() (err error) { city, sub_zone, sub_zone_id, - label, + labels, status, update_time, create_time diff --git a/dbm-services/common/db-resource/internal/svr/apply/apply.go b/dbm-services/common/db-resource/internal/svr/apply/apply.go index 8ecb02a5d2..0be73ead9d 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/apply.go +++ b/dbm-services/common/db-resource/internal/svr/apply/apply.go @@ -27,7 +27,6 @@ import ( "github.com/samber/lo" "gorm.io/gorm" - "gorm.io/gorm/clause" ) // SearchContext TODO @@ -298,23 +297,29 @@ func (o *SearchContext) MatchOsType(db *gorm.DB) { db.Where("os_type = ? ", osType) } -// MatchOsName TODO +// MatchOsName match os name os_name = "tlinux-1.2" func (o *SearchContext) MatchOsName(db *gorm.DB) { // match os name like Windows Server 2012 - if len(o.ObjectDetail.OsNames) > 0 { - conditions := []clause.Expression{} - for _, osname := range o.ObjectDetail.OsNames { - conditions = append(conditions, clause.Like{ - Column: "os_name", - Value: "%" + strings.TrimSpace(strings.ToLower(osname)) + "%", - }) - } - if len(conditions) == 1 { - db.Clauses(clause.AndConditions{Exprs: conditions}) - } else { - // 有多个条件，使用or，才会被用（）包括起来所有的or条件 - db.Clauses(clause.OrConditions{Exprs: conditions}) - } + // conditions := []clause.Expression{} + // for _, osname := range o.ObjectDetail.OsNames { + // conditions = append(conditions, clause.Like{ + // Column: "os_name", + // Value: "%" + strings.TrimSpace(strings.ToLower(osname)) + "%", + // }) + // } + // if len(conditions) == 1 { + // db.Clauses(clause.AndConditions{Exprs: conditions}) + // } else { + // // 有多个条件，使用or，才会被用（）包括起来所有的or条件 + // db.Clauses(clause.OrConditions{Exprs: conditions}) + // } + if len(o.ObjectDetail.OsNames) == 0 { + return + } + if o.ObjectDetail.ExcludeOsName { + db.Where("os_name not in (?)", o.ObjectDetail.OsNames) + } else { + db.Where("os_name in (?)", o.ObjectDetail.OsNames) } } diff --git a/dbm-services/common/db-resource/internal/svr/apply/apply_base.go b/dbm-services/common/db-resource/internal/svr/apply/apply_base.go index 60fe75402c..c540091366 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/apply_base.go +++ b/dbm-services/common/db-resource/internal/svr/apply/apply_base.go @@ -223,9 +223,10 @@ type ObjectDetail struct { Affinity string `json:"affinity"` // Windows,Linux - OsType string `json:"os_type"` - OsNames []string `json:"os_names"` - Count int `json:"count" binding:"required,min=1"` // 申请数量 + OsType string `json:"os_type"` + OsNames []string `json:"os_names"` + ExcludeOsName bool `json:"exclude_os_name"` + Count int `json:"count" binding:"required,min=1"` // 申请数量 } // Hosts bk hosts diff --git a/dbm-services/common/db-resource/internal/svr/task/task.go b/dbm-services/common/db-resource/internal/svr/task/task.go index 69792e51cd..525fb47ead 100644 --- a/dbm-services/common/db-resource/internal/svr/task/task.go +++ b/dbm-services/common/db-resource/internal/svr/task/task.go @@ -19,6 +19,7 @@ import ( "dbm-services/common/db-resource/internal/model" "dbm-services/common/db-resource/internal/svr/bk" + "dbm-services/common/db-resource/internal/util" "dbm-services/common/go-pubpkg/cc.v3" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/logger" @@ -215,3 +216,39 @@ func AsyncResourceHardInfo() (err error) { } return nil } + +// SyncOsNameInfo sync os name info +func SyncOsNameInfo() (err error) { + logger.Info("start async from cmdb ...") + var rsList []model.TbRpDetail + err = model.DB.Self.Table(model.TbRpDetailName()).Find(&rsList).Error + if err != nil { + if err == sql.ErrNoRows { + return nil + } + logger.Error("query total_storage_cap less than 0,err %w ", err) + return err + } + bizHostMap := make(map[int][]string) + for _, rs := range rsList { + bizHostMap[rs.BkBizId] = append(bizHostMap[rs.BkBizId], rs.IP) + } + for bizId, hosts := range bizHostMap { + ccInfos, _, err := bk.BatchQueryHostsInfo(bizId, hosts) + if err != nil { + logger.Warn("query machine hardinfo from cmdb failed %s", err.Error()) + continue + } + for _, ccInfo := range ccInfos { + err = model.DB.Self.Table(model.TbRpDetailName()).Where("ip = ? and bk_biz_id = ? ", ccInfo.InnerIP, bizId). + Updates(map[string]interface{}{ + "os_name": util.CleanOsName(ccInfo.OSName), + "os_version": ccInfo.BkOsVersion, + }).Error + if err != nil { + logger.Warn("request cmdb api failed %s", err.Error()) + } + } + } + return nil +} diff --git a/dbm-services/common/db-resource/internal/util/util.go b/dbm-services/common/db-resource/internal/util/util.go new file mode 100644 index 0000000000..dba5b035fa --- /dev/null +++ b/dbm-services/common/db-resource/internal/util/util.go @@ -0,0 +1,31 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +// Package util TODO +package util + +import ( + "regexp" + "strings" +) + +// CleanOsName clean os name +func CleanOsName(osName string) string { + tr := regexp.MustCompile(`(?i)^tencent`) + if tr.MatchString(strings.TrimSpace(osName)) { + r := regexp.MustCompile(`\d+(.\d)+`) + return "tliunx-" + r.FindString(osName) + } + wr := regexp.MustCompile(`(?i)Windows\s*Server\s*\d\d\d\d`) + if wr.MatchString(osName) { + return strings.TrimSpace(strings.ReplaceAll(wr.FindString(osName), " ", "")) + } + return osName +} diff --git a/dbm-services/common/go-pubpkg/cmutil/osinfo.go b/dbm-services/common/go-pubpkg/cmutil/osinfo.go index 6469381685..56d497cca7 100644 --- a/dbm-services/common/go-pubpkg/cmutil/osinfo.go +++ b/dbm-services/common/go-pubpkg/cmutil/osinfo.go @@ -2,6 +2,7 @@ package cmutil import ( "path/filepath" + "regexp" "strings" "github.com/pkg/errors" @@ -155,3 +156,18 @@ func IsSameTopLevelDir(dir1, dir2 string) (bool, error) { } return topDir1 == topDir2, nil } + +// GetGlibcVersion get os glibc version using ExeCommand("ldd --version |grep libc") +func GetGlibcVersion() (string, error) { + outStr, errStr, err := ExecCommand(false, "", + "/usr/bin/ldd", "--version", "|", "grep", "libc") + if err != nil { + return "", errors.WithMessage(err, errStr) + } + verMatch := regexp.MustCompile(`ldd \(.*\) (\d+\.\d+)`) + ms := verMatch.FindStringSubmatch(outStr) + if len(ms) == 2 { + return ms[1], nil + } + return "", errors.New("ldd --version | grep glibc fail to get glibc version") +} diff --git a/dbm-services/mongodb/db-tools/mongo-toolkit-go/Makefile b/dbm-services/mongodb/db-tools/mongo-toolkit-go/Makefile index 6c4c43557f..82d7b23d91 100644 --- a/dbm-services/mongodb/db-tools/mongo-toolkit-go/Makefile +++ b/dbm-services/mongodb/db-tools/mongo-toolkit-go/Makefile @@ -41,7 +41,7 @@ check: build: @echo " > Building Linux binary..." # go build -v $(LDFLAGS) -o target/$(toolkit)_$(OS) cmd/mongo-toolkit-go/main.go - GOOS=linux GOARCH=amd64 go build $(TRIMPATH) -v $(BUILD_FLAG) -o ./build/$(SRV_NAME)_Linux cmd/$(SRV_NAME)/main.go + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(TRIMPATH) -v $(BUILD_FLAG) -o ./build/$(SRV_NAME)_Linux cmd/$(SRV_NAME)/main.go local: @echo " > Building $(OS) binary..." diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/cutover/base.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/cutover/base.go index eb71953ccc..51e0d61720 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/cutover/base.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/cutover/base.go @@ -43,7 +43,6 @@ const ( // Ins tance TODO type Ins struct { native.Instance - dbConn *native.DbWorker `json:"-"` } // Proxies TODO @@ -202,7 +201,7 @@ func (m *MySQLClusterDetail) CheckAltSlaveMasterAddr() (err error) { m.MasterIns.Addr(), ) logger.Error(msg) - return fmt.Errorf(msg) + return errors.New(msg) } return err } @@ -257,7 +256,7 @@ func (c *MasterInfo) LockTablesPreCheck(backupUser string) (err error) { // FlushTablesWithReadLock 执行flush table with read lock func (c *MasterInfo) FlushTablesWithReadLock() (err error) { - if _, err := c.lockConn.ExecContext(context.Background(), "set lock_wait_timeout = 10;"); err != nil { + if _, err = c.lockConn.ExecContext(context.Background(), "set lock_wait_timeout = 10;"); err != nil { return err } err = util.Retry( @@ -302,7 +301,7 @@ func (c *MasterInfo) KillBackupUserProcesslist(backupUser string) (err error) { if err != nil { return err } - if len(processLists) <= 0 { + if len(processLists) == 0 { logger.Info("没有发现关于备份用户[%s]相关的processlist~", backupUser) return nil } @@ -321,11 +320,11 @@ func (c *MasterInfo) FindLongQuery() (err error) { if err != nil { return err } - if len(activeProcessLists) <= 0 { + if len(activeProcessLists) == 0 { return nil } var errs []error - errs = []error{errors.New("active processlist exist:\n")} + errs = []error{errors.New("active processlist exist")} for _, p := range activeProcessLists { errs = append( errs, fmt.Errorf( @@ -364,7 +363,7 @@ func realVal(v sql.NullString) string { // return // } -// RecordBinPos 记录切换时候的bin postion +// RecordBinPos 记录切换时候的bin position func (s *AltSlaveInfo) RecordBinPos() (binPosJsonStr string, err error) { pos, _ := s.dbConn.ShowMasterStatus() logger.Info("show master status on %s,detail: File:%s,Pos:%d", s.Addr(), pos.File, pos.Position) diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/cutover/cutover.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/cutover/cutover.go index 820213d662..4363265f72 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/cutover/cutover.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/cutover/cutover.go @@ -23,7 +23,7 @@ import ( "dbm-services/mysql/db-tools/dbactuator/pkg/util" ) -// CutOverParam TODO +// CutOverParam cutover 参数 type CutOverParam struct { Host string `json:"host" validate:"required,ip"` Cluster *MySQLClusterDetail `json:"cluster"` @@ -182,11 +182,12 @@ func (m *CutOverToSlaveComp) Example() interface{} { func (m *CutOverToSlaveComp) PreCheck() (err error) { // 以下是强制检查的内容 // 检查下proxy backend 是不是 源Master - if err := m.cluster.CheckBackends(m.cluster.MasterIns.Host, m.cluster.MasterIns.Port); err != nil { + if err = m.cluster.CheckBackends(m.cluster.MasterIns.Host, m.cluster.MasterIns.Port); err != nil { + logger.Error("proxy backend is not %s:%d", m.cluster.MasterIns.Host, m.cluster.MasterIns.Port) return err } // 检查alt Slave repl 的地址不是 cluster.MasterIns - if err := m.cluster.CheckAltSlaveMasterAddr(); err != nil { + if err = m.cluster.CheckAltSlaveMasterAddr(); err != nil { return err } @@ -203,10 +204,10 @@ func (m *CutOverToSlaveComp) PreCheck() (err error) { } // 客户端连接检查 if m.Params.ClientConnCheck { - prcsls, err := m.cluster.AltSlaveIns.dbConn.ShowApplicationProcesslist(m.sysUsers) - if err != nil { - logger.Error("show processlist failed %s", err.Error()) - return err + prcsls, errx := m.cluster.AltSlaveIns.dbConn.ShowApplicationProcesslist(m.sysUsers) + if errx != nil { + logger.Error("show processlist failed %s", errx.Error()) + return errx } if len(prcsls) > 0 { return fmt.Errorf("there is a connection for non system users %v", prcsls) @@ -224,7 +225,7 @@ func (m *CutOverToSlaveComp) PreCheck() (err error) { if err = m.cluster.AltSlaveIns.dbConn.CheckSlaveReplStatus(func() (resp native.ShowSlaveStatusResp, err error) { return m.cluster.AltSlaveIns.dbConn.ShowSlaveStatus() }); err != nil { - logger.Error("检查主从同步状态出错: %s", err.Error()) + logger.Error("检查从实例[%s:%d]同步状态出错: %s", m.cluster.AltSlaveIns.Host, m.cluster.AltSlaveIns.Port, err.Error()) return err } @@ -232,6 +233,8 @@ func (m *CutOverToSlaveComp) PreCheck() (err error) { if err = m.cluster.AltSlaveIns.Slave.dbConn.CheckSlaveReplStatus(func() (resp native.ShowSlaveStatusResp, err error) { return m.cluster.AltSlaveIns.Slave.dbConn.ShowSlaveStatus() }); err != nil { + logger.Error("检查从实例[%s:%d]同步状态出错: %s", m.cluster.AltSlaveIns.Slave.Host, m.cluster.AltSlaveIns.Slave.Port, + err.Error()) return err } } @@ -259,7 +262,10 @@ func (m *CutOverToSlaveComp) PreCheck() (err error) { func (m *CutOverToSlaveComp) CutOver() (binPos string, err error) { defer func() { if m.Params.LockedSwitch { - m.cluster.MasterIns.UnlockTables() + errx := m.cluster.MasterIns.UnlockTables() + if errx != nil { + logger.Error("unlock tables failed %s", errx.Error()) + } } if err != nil { e := m.cluster.UpdateProxiesBackend(m.cluster.MasterIns.Host, m.cluster.MasterIns.Port) diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbconsole_dump.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbconsole_dump.go index 5b4605c066..7fc5ecadf8 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbconsole_dump.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbconsole_dump.go @@ -245,19 +245,40 @@ func (c *DbConsoleDumpComp) Run() (err error) { dumpOption.GtidPurgedOff = true } - dumper = mysqlutil.MySQLDumper{ - DumpDir: c.backupDir, - Ip: c.Params.Host, - Port: c.Params.Port, - DbBackupUser: c.GeneralParam.RuntimeAccountParam.AdminUser, - DbBackupPwd: c.GeneralParam.RuntimeAccountParam.AdminPwd, - DbNames: c.dbs, - Tables: c.realTables, - IgnoreTables: c.realIgnoreTables, - Where: c.Params.DumpDetail.Where, - DumpCmdFile: c.dumpCmd, - Charset: c.charset, - MySQLDumpOption: dumpOption, + if len(c.realTables) > 0 && len(c.dbs) > 1 { + err := fmt.Errorf("mysqldump only support one database if --tables not empty") + logger.Error(err.Error()) + return err + } + + if len(c.realTables) > 0 { + dumper = mysqlutil.MySQLDumper{ + DumpDir: c.backupDir, + Ip: c.Params.Host, + Port: c.Params.Port, + DbBackupUser: c.GeneralParam.RuntimeAccountParam.AdminUser, + DbBackupPwd: c.GeneralParam.RuntimeAccountParam.AdminPwd, + Tables: append(c.dbs, c.realTables...), + IgnoreTables: c.realIgnoreTables, + Where: c.Params.DumpDetail.Where, + DumpCmdFile: c.dumpCmd, + Charset: c.charset, + MySQLDumpOption: dumpOption, + } + } else { + dumper = mysqlutil.MySQLDumper{ + DumpDir: c.backupDir, + Ip: c.Params.Host, + Port: c.Params.Port, + DbBackupUser: c.GeneralParam.RuntimeAccountParam.AdminUser, + DbBackupPwd: c.GeneralParam.RuntimeAccountParam.AdminPwd, + DbNames: c.dbs, + IgnoreTables: c.realIgnoreTables, + Where: c.Params.DumpDetail.Where, + DumpCmdFile: c.dumpCmd, + Charset: c.charset, + MySQLDumpOption: dumpOption, + } } for _, db := range c.dbs { diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go index 67ed737ae7..d5eb93d64e 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/install_mysql.go @@ -264,6 +264,7 @@ func (i *InstallMySQLComp) InitDefaultParam() (err error) { i.Checkfunc = append(i.Checkfunc, i.precheckMysqlDir) i.Checkfunc = append(i.Checkfunc, i.precheckMysqlProcess) i.Checkfunc = append(i.Checkfunc, i.precheckMysqlPackageBitOS) + i.Checkfunc = append(i.Checkfunc, i.precheckGlibcVersion) i.Checkfunc = append(i.Checkfunc, i.Params.Medium.Check) i.Params.PartitionYWAccount.AccessHosts = []string{ i.Params.Host, @@ -361,6 +362,19 @@ func (i *InstallMySQLComp) precheckMysqlPackageBitOS() error { return nil } +func (i *InstallMySQLComp) precheckGlibcVersion() error { + glibcVer, err := cmutil.GetGlibcVersion() + if err != nil { + logger.Error("failed to glibc version, err:%s", err.Error()) + return err + } + // mysql 8.0 不安装在 tlinux1.2上 (mysqld能运行，但周边 xtrabackup 8.0 依赖 glibc>=2.14) + if cmutil.MySQLVersionParse(i.Params.MysqlVersion) >= cmutil.MySQLVersionParse("8.0.0") && glibcVer < "2.14" { + return errors.Errorf("glibc version %s it not allowed to install %s", glibcVer, i.Params.MysqlVersion) + } + return nil +} + // initMySQLInstanceMem TODO // GetInstMemByIP 返回的内存单位是 MB func (i *InstallMySQLComp) initMySQLInstanceMem() (err error) { diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go index 15c36571ee..fbd27fc14c 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go @@ -133,6 +133,7 @@ func (m *DBLoader) chooseDBBackupLoader() error { QpressTool: m.Tools.MustGet(tools.ToolQPress), LoaderDir: m.targetDir, StorageType: strings.ToLower(m.indexObj.StorageEngine), + MySQLVersion: m.BackupInfo.indexObj.MysqlVersion, }, } } else { diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go index a3a5703c7a..92a7f2a037 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover.go @@ -30,7 +30,8 @@ type Xtrabackup struct { // 在 PreRun 时初始化，本地实例的配置文件 myCnf *util.CnfFile - StorageType string `json:"-"` + StorageType string `json:"-"` + MySQLVersion string `json:"-"` } // PreRun 以下所有步骤必须可重试 @@ -53,7 +54,7 @@ func (x *Xtrabackup) PreRun() error { return err } - logger.Info("clean local mysqld data dirs") + logger.Info("start to clean local mysqld data dirs") // 清理本地目录 if err := x.cleanXtraEnv(); err != nil { return err @@ -79,7 +80,7 @@ func (x *Xtrabackup) PostRun() (err error) { return err } - logger.Info("start local mysqld with skip-grant-tables") + logger.Info("start local mysqld with --skip-grant-tables --skip-slave-start") // 启动mysql-修复权限 startParam := computil.StartMySQLParam{ MediaDir: cst.MysqldInstallPath, @@ -88,6 +89,7 @@ func (x *Xtrabackup) PostRun() (err error) { MySQLPwd: x.TgtInstance.Pwd, Socket: x.TgtInstance.Socket, SkipGrantTables: true, // 以 skip-grant-tables 启动来修复 ADMIN + SkipSlaveFlag: true, } if _, err = startParam.StartMysqlInstance(); err != nil { return errors.WithMessage(err, "start mysqld after xtrabackup") @@ -106,6 +108,11 @@ func (x *Xtrabackup) PostRun() (err error) { serverVersion = "5.7.20" // fake } } + if _, err = x.dbWorker.Exec("reset slave all"); err != nil { + logger.Warn("failed to reset slave all, err", err.Error()) + } + + //return nil logger.Info("repair user 'ADMIN' host and password") // 物理备份，ADMIN密码与 backup instance(cluster?) 相同，修复成 // 修复ADMIN用户，而不是 x.TgtInstance.User，主要是修复 host，密码修复成临时用户 DBM_JOB_xxx 的密码 @@ -130,10 +137,12 @@ func (x *Xtrabackup) PostRun() (err error) { logger.Info("restart local mysqld %d", x.TgtInstance.Port) // 重启mysql（去掉 skip-grant-tables） startParam.SkipGrantTables = false + startParam.SkipSlaveFlag = false startParam.MySQLUser = native.DBUserAdmin if _, err := startParam.RestartMysqlInstance(); err != nil { return errors.WithMessage(err, "RestartMysqlInstance") } + logger.Info("reconnect use ADMIN and temp_job_user pwd(already repaired) %d", x.TgtInstance.Port) tmpAdminPassInst := deepcopy.Copy(x.TgtInstance).(native.InsObject) tmpAdminPassInst.User = native.DBUserAdmin @@ -165,14 +174,13 @@ func (x *Xtrabackup) cleanXtraEnv() error { "datadir", "innodb_log_group_home_dir", "innodb_data_home_dir", - "tokudb_log_dir", - "tokudb_data_dir", "relay-log", "log_bin", "tmpdir", } if x.StorageType == "tokudb" { - dirs = []string{"tokudb_log_dir", "tokudb_data_dir", "tmpdir"} + dirs = []string{"tokudb_log_dir", "tokudb_data_dir", "tmpdir", "relay-log", + "innodb_log_group_home_dir", "innodb_data_home_dir"} // replace ibdata1 } return x.CleanEnv(dirs) } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go index 33e4fd013b..6f06bd17b0 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader/xtrabackup_recover_repaire.go @@ -13,6 +13,7 @@ package dbbackup_loader import ( "database/sql" "fmt" + "os" "path" "path/filepath" "regexp" @@ -185,8 +186,8 @@ func (x *Xtrabackup) RepairPrivileges() error { var batchSQLs []string // delete src host's ADMIN/sync user - sql1 := fmt.Sprintf("DELETE FROM mysql.user WHERE `user` IN (%s) AND `host` = %s;", myUsersUnsafe, srcHostUnsafe) - batchSQLs = append(batchSQLs, sql1) + batchSQLs = append(batchSQLs, + fmt.Sprintf("DELETE FROM mysql.user WHERE `user` IN (%s) AND `host` = %s;", myUsersUnsafe, srcHostUnsafe)) // update src host to new, but not ADMIN/sync/repl sql2s := []string{ @@ -203,16 +204,15 @@ func (x *Xtrabackup) RepairPrivileges() error { tgtHostUnsafe, srcHostUnsafe, myUsersUnsafe, ), } - batchSQLs = append(batchSQLs, sql2s...) - - /* - // delete src host users, but not localhost - sql3 := fmt.Sprintf( - "DELETE FROM mysql.user WHERE `host` IN(%s);", srcHostUnsafe, - ) - batchSQLs = append(batchSQLs, sql3) - */ + if cmutil.MySQLVersionParse(x.MySQLVersion) >= cmutil.MySQLVersionParse("8.0.0") { + // mysql 8.0 的 BACKUP_ADMIN 权限记录在 mysql.global_grants 中 + sql2s = append(sql2s, fmt.Sprintf( + "UPDATE mysql.global_grants SET `HOST`=%s WHERE `HOST`=%s AND User NOT IN (%s);", + tgtHostUnsafe, srcHostUnsafe, myUsersUnsafe, + )) + } + batchSQLs = append(batchSQLs, sql2s...) // flush sql4 := fmt.Sprintf("flush privileges;") batchSQLs = append(batchSQLs, sql4) @@ -230,7 +230,7 @@ func (x *Xtrabackup) CleanEnv(dirs []string) error { return fmt.Errorf("port %d is still opened", x.TgtInstance.Port) } - var dirArray []string + var pathsToReset []string for _, v := range dirs { if strings.TrimSpace(x.myCnf.GetMyCnfByKeyWithDefault(util.MysqldSec, v, "")) == "" { logger.Warn(fmt.Sprintf("my.cnf %s is Emtpty!!", v)) @@ -245,7 +245,7 @@ func (x *Xtrabackup) CleanEnv(dirs []string) error { reg := regexp.MustCompile(cst.RelayLogFileMatch) if result := reg.FindStringSubmatch(val); len(result) == 2 { relayLogDir := result[1] - dirArray = append(dirArray, "rm -rf "+relayLogDir+"/*") + pathsToReset = append(pathsToReset, relayLogDir) } case "log_bin", "log-bin": val, err := x.myCnf.GetMySQLLogDir() @@ -256,26 +256,20 @@ func (x *Xtrabackup) CleanEnv(dirs []string) error { if result := reg.FindStringSubmatch(val); len(result) == 2 { binlogDir := result[1] // TODO 所有 rm -rf 的地方都应该要检查是否可能 rm -rf / binlog.xxx 这种误删可能 - dirArray = append(dirArray, "rm -rf "+binlogDir+"/*") + pathsToReset = append(pathsToReset, binlogDir) } case "slow_query_log_file", "slow-query-log-file": if val := x.myCnf.GetMyCnfByKeyWithDefault(util.MysqldSec, "slow_query_log_file", ""); val != "" { - dirArray = append(dirArray, "rm -f "+val) + pathsToReset = append(pathsToReset, val) } default: val := x.myCnf.GetMyCnfByKeyWithDefault(util.MysqldSec, v, "") if strings.TrimSpace(val) != "" && strings.TrimSpace(val) != "/" { - dirArray = append(dirArray, "rm -rf "+val+"/*") + pathsToReset = append(pathsToReset, val) } } } - scripts := strings.Join(dirArray, "\n") - logger.Info("CleanEnv: %s", scripts) - // run with mysql os user - if _, err := osutil.ExecShellCommand(false, scripts); err != nil { - return err - } - return nil + return ResetPath(pathsToReset) } // ReplaceMycnf godoc @@ -339,3 +333,31 @@ func (x *Xtrabackup) getSocketName() string { sock := x.myCnf.GetMyCnfByKeyWithDefault(util.MysqldSec, "socket", "/tmp/mysql.sock") return sock } + +// ResetPath clean files +// if filepath is dir, clean all files in it (file permission and owner is NOT preserved) +// if filepath is file, remove it +// this function is used to avoid "/bin/rm: Argument list too long" when using rm -rf /xxx/path/* +func ResetPath(paths []string) error { + for _, pa := range paths { + if strings.TrimSpace(pa) == "/" { + return errors.Errorf("path %s is not allowed to clean", pa) + } + if cmutil.IsDirectory(pa) { + logger.Info("Clean Dir: %s", pa) + if err := os.RemoveAll(pa); err != nil { + return errors.WithMessage(err, "clean dir") + } else { // recreate dir + if err = os.MkdirAll(pa, 0755); err != nil { + return errors.WithMessage(err, "recreate dir") + } + } + } else { + logger.Info("Remove File: %s", pa) + if err := os.RemoveAll(pa); err != nil { + return errors.WithMessage(err, "remove file") + } + } + } + return nil +} diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/xload_repaire_util.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/xload_repaire_util.go index 7c80c6be0c..26b4965ce1 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/xload_repaire_util.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/xload_repaire_util.go @@ -12,6 +12,7 @@ import ( "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/logger" "dbm-services/common/go-pubpkg/mysqlcomm" + "dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_loader" "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" "dbm-services/mysql/db-tools/dbactuator/pkg/native" "dbm-services/mysql/db-tools/dbactuator/pkg/util" @@ -167,7 +168,7 @@ func (x *XLoad) CleanEnv(dirs []string) error { return fmt.Errorf("port %d is still opened", x.TgtInstance.Port) } - var dirArray []string + var pathsToReset []string for _, v := range dirs { if strings.TrimSpace(x.myCnf.GetMyCnfByKeyWithDefault(util.MysqldSec, v, "")) == "" { logger.Warn(fmt.Sprintf("my.cnf %s is Emtpty!!", v)) @@ -182,7 +183,7 @@ func (x *XLoad) CleanEnv(dirs []string) error { reg := regexp.MustCompile(cst.RelayLogFileMatch) if result := reg.FindStringSubmatch(val); len(result) == 2 { relaylogdir := result[1] - dirArray = append(dirArray, "rm -rf "+relaylogdir+"/*") + pathsToReset = append(pathsToReset, relaylogdir) } case "log_bin", "log-bin": val, err := x.myCnf.GetMySQLLogDir() @@ -193,26 +194,21 @@ func (x *XLoad) CleanEnv(dirs []string) error { if result := reg.FindStringSubmatch(val); len(result) == 2 { binlogdir := result[1] // TODO 所有 rm -rf 的地方都应该要检查是否可能 rm -rf / binlog.xxx 这种误删可能 - dirArray = append(dirArray, "rm -rf "+binlogdir+"/*") + pathsToReset = append(pathsToReset, binlogdir) } case "slow_query_log_file", "slow-query-log-file": if val := x.myCnf.GetMyCnfByKeyWithDefault(util.MysqldSec, "slow_query_log_file", ""); val != "" { - dirArray = append(dirArray, "rm -f "+val) + pathsToReset = append(pathsToReset, val) } default: val := x.myCnf.GetMyCnfByKeyWithDefault(util.MysqldSec, v, "") if strings.TrimSpace(val) != "" && strings.TrimSpace(val) != "/" { - dirArray = append(dirArray, "rm -rf "+val+"/*") + pathsToReset = append(pathsToReset, val) } } } - scripts := strings.Join(dirArray, "\n") - logger.Info("CleanEnv: %s", scripts) - // run with mysql os user - if _, err := osutil.ExecShellCommand(false, scripts); err != nil { - return err - } - return nil + + return dbbackup_loader.ResetPath(pathsToReset) } // ReplaceMycnf godoc diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go index 300deebad7..ec4ee037c8 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/semantic_dump_schema.go @@ -142,17 +142,12 @@ func (c *SemanticDumpSchemaComp) Init() (err error) { logger.Error("获取version failed %s", err.Error()) return err } + c.dumpCmd = path.Join(cst.MysqldInstallPath, "bin", "mysqldump") + c.isSpider = strings.Contains(version, "tdbctl") + if cmutil.MySQLVersionParse(version) > cmutil.MySQLVersionParse("5.6.9") { c.gtidPurgedOff = true } - c.isSpider = strings.Contains(version, "tdbctl") - // 临时方案 - if cmutil.FileExists("/home/mysql/dbbackup/mysqldump_x86_64") { - c.dumpCmd = "/home/mysql/dbbackup/mysqldump_x86_64" - c.useTmysqldump = true - } else { - c.dumpCmd = path.Join(cst.MysqldInstallPath, "bin", "mysqldump") - } // to export the table structure from the central control // you need to use the mysqldump that comes with the central control if c.isSpider { diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/native/dbworker.go b/dbm-services/mysql/db-tools/dbactuator/pkg/native/dbworker.go index afb9c4af96..9da7ce9838 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/native/dbworker.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/native/dbworker.go @@ -351,6 +351,15 @@ func (h *DbWorker) SelectVersion() (version string, err error) { return } +// IsEnableGtid just enable gtid +func (h *DbWorker) IsEnableGtid() (enables bool, err error) { + val, err := h.GetSingleGlobalVar("gtid_mode") + if err != nil { + return false, err + } + return strings.ToUpper(strings.TrimSpace(val)) == "ON", err +} + // HasTokudb 判断是否安装tokudb引擎 func (h *DbWorker) HasTokudb() (has bool, err error) { var engine string @@ -598,6 +607,7 @@ type TableColumnDef struct { ColType string } +// TableColumnInfo TODO type TableColumnInfo map[string]TableColumnDef // TableSchema TODO diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go index 20de195143..77a289581d 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go @@ -91,6 +91,7 @@ var dumpCmd = &cobra.Command{ }{} body.Name = "dbbackup-by-host" body.Content = fmt.Sprintf("run dbbackup failed %s", err.Error()) + body.Dimension = map[string]interface{}{} if sendErr := manager.SendEvent(body.Name, body.Content, body.Dimension); sendErr != nil { logger.Log.Error("SendEvent failed", sendErr.Error()) } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_spider.go b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_spider.go index 625c893cdd..8fd0e6f87e 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_spider.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_spider.go @@ -93,9 +93,9 @@ func findSpiderBackupConfigFile(cnfFiles []string) (string, error) { var spiderScheduleCmd = &cobra.Command{ Use: "schedule", - SilenceUsage: true, Short: "spiderbackup schedule", Long: `Start spider global backup. Will initialize backup tasks using one backup-id on spider master`, + SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { if err := logger.InitLog("dbbackup_spider.log"); err != nil { return err @@ -125,9 +125,10 @@ var spiderScheduleCmd = &cobra.Command{ } var spiderCheckCmd = &cobra.Command{ - Use: "check", - Short: "spiderbackup check", - Long: `Check or run backup todo tasks`, + Use: "check", + Short: "spiderbackup check", + Long: `Check or run backup todo tasks`, + SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { if err := logger.InitLog("dbbackup_spider.log"); err != nil { return err @@ -136,7 +137,11 @@ var spiderCheckCmd = &cobra.Command{ if err != nil { return err } - publicConfigs, err := batchParseCnfFiles(cnfFiles) + configFile, err := findSpiderBackupConfigFile(cnfFiles) + if err != nil { + return err + } + publicConfigs, err := batchParseCnfFiles([]string{configFile}) if err != nil { return err } @@ -150,9 +155,10 @@ var spiderCheckCmd = &cobra.Command{ } var spiderQueryCmd = &cobra.Command{ - Use: "query", - Short: "spiderbackup query", - Long: `Query spider backup task status, only run on spider master`, + Use: "query", + Short: "spiderbackup query", + Long: `Query spider backup task status, only run on spider master`, + SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { if err := logger.InitLog("dbbackup_spider.log"); err != nil { return err diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup-go-deps-txsql/bin/tokudb_back.pl b/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup-go-deps-txsql/bin/tokudb_back.pl new file mode 100644 index 0000000000..a890fd6d36 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup-go-deps-txsql/bin/tokudb_back.pl @@ -0,0 +1,553 @@ +#!/usr/bin/perl + +use FindBin qw($Bin); + +use warnings; +use strict; +use Getopt::Long qw(:config posix_default); +use POSIX qw(strftime); +use File::Copy qw(copy); +use Data::Dumper; +use DBI; +use File::Find; +no warnings 'File::Find'; + +my $user=''; +my $password=''; +my $sock=''; +my $host='localhost'; +my $port=3306; +my $increment=0; +my $lock_wait_timeout = undef; +my $dump_slave = undef; +my $global_time_offset=0; +my @tokudb_backup_warnings; +sub tmysql_version_parser { + my $mysql_version = shift; + if($mysql_version =~ /tmysql/){ + return sprintf( '%03d%03d%03d', $mysql_version =~ m/tmysql-([012345678])\.(\d+)\.*(\d*)/g ); + }else{ + return '000000000'; + } +} + +Getopt::Long::Configure ("bundling_override"); +GetOptions( + 'host|h=s' => \$host, + 'password|p=s' => \$password, + 'port|P=s' => \$port, + 'user|u=s' => \$user, + 'socket|S=s' => \$sock, + 'increment|i!' => \$increment, + 'flush-wait-timeout=s' => \$lock_wait_timeout, + 'dump-slave!' => \$dump_slave +)or die "usage: xxx -u user -p password -h host -P port [-i|increment] backdir/target_name(when enable increment,there shoud be previous file in backdir) global_time_offset"; + + +my $backdir= $ARGV[0]; + +if( -d $backdir){ +}else{ + mkdir("$backdir") or die "mkdir $backdir fail"; +} +if(defined $ARGV[1] and $ARGV[1]=~ /\s*(\d+)\s*/){ + $global_time_offset=$1; +} +mkdir("$backdir/mysql_data"); +mkdir("$backdir/tokudb_data"); +mkdir("$backdir/tokudb_log"); + +if ($sock =~ /.+\/(\d+)\// ) { + $port = $1; +} + +my $data_dir = undef; +my $tokudb_data_dir = undef; +my $tokudb_log_dir = undef; +my $myconf = ($port == 3306)? "/etc/my.cnf" : "/etc/my.cnf.$port"; + +open( my $my_cnf_fp, "< $myconf" ) or die "$myconf open error" ; +while ( my $line = <$my_cnf_fp> ) { + chomp $line; + if ( not defined $data_dir ) { + if ( $line =~ /\s*datadir\s*=([\S]+)/ ) { + $data_dir = $1; + next; + } + } + if ( not defined $tokudb_data_dir ) { + if ( $line =~ /\s*tokudb_data_dir\s*=([\S]+)/ ) { + $tokudb_data_dir = $1; + next; + } + } + if ( not defined $tokudb_log_dir ) { + if ( $line =~ /\s*tokudb_log_dir\s*=([\S]+)/ ) { + $tokudb_log_dir = $1; + next; + } + } +} +close($my_cnf_fp); + +unless (defined $data_dir && + defined $tokudb_data_dir && + defined $tokudb_log_dir ) +{ + die "some key in $myconf lost"; +} + +my $fully_stamp=(time)+$global_time_offset; +#system("date '+%s' > $backdir/TOKUDB.BEGIN"); +my $date=strftime("%Y%m%d_%H%M%S", localtime($fully_stamp)); + +my $backed_debug_log_file="$backdir/debug_log.txt"; +my $debug_log_file="$Bin/debug_log_tokudb.txt"; +open my $backed_debug_log,">$backed_debug_log_file" or die "failed to open $backed_debug_log_file for:$!\n"; +open my $debug_log,">>$debug_log_file" or die "failed to open $debug_log_file for:$!\n"; +print $debug_log "\nport:$port time:$date\n"; +print $backed_debug_log "\nport:$port time:$date\n"; + +system("echo $date >> $backdir/TOKUDB.BEGIN"); +my $tname=( split(/\//,$ARGV[0]) )[-1]; +my $dir; +if($ARGV[0] =~ /(.+)\/${tname}$/){ + $dir=$1; +}else{ + die "failed while parsing backdir"; +} +my $fully_name=$tname; +my $dbh = DBI->connect ("DBI:mysql:mysql:host=$host:port=$port:mysql_socket=$sock", $user, $password); + +my $sql = qq{ select version() }; +my $row_ref = $dbh->selectrow_arrayref($sql); +my $tmysql_ver = tmysql_version_parser($row_ref->[0]); + +if($tmysql_ver lt tmysql_version_parser("tmysql-2.1.3")){ + die "tokudb physical backup only avaliable above tmysql-2.1.3"; +} + +if(defined $lock_wait_timeout and $lock_wait_timeout ne '0'){ + $dbh->do("SET LOCK_WAIT_TIMEOUT=$lock_wait_timeout;") or die "Set lock_wait_timeout failed!"; +} + +##### STEP 1. set tokudb_commit_sync=1:close redo log buffer +#my $ret=$dbh->do("start transaction"); +#die "unable to start transaction for close redo log buffer" if not defined $ret or $ret<0; +#$sql = qq{ select \@\@tokudb_commit_sync }; +#$row_ref = $dbh->selectrow_arrayref($sql); +#my $old_global_tokudb_commit_sync=$row_ref->[0]; +#$ret=$dbh->do("set global tokudb_commit_sync=1"); +#die "unable to set global tokudb_commit_sync=1" if not defined $ret or $ret<0; +#$ret=$dbh->do("commit"); +#die "unable commit set global tokudb_commit_sync=1" if not defined $ret or $ret<0; +#print qx(date); +#print "unable redo log buffer ok\n\n"; + + +##### STEP 2. get METADATA lock for each table +if($tmysql_ver ge tmysql_version_parser("tmysql-2.1.3")){ + $dbh->do("lock tables for backup;") or die "Get metadata lock failed"; + print $debug_log qx(date); + print $debug_log "get metadata lock ok\n\n"; +} + +my @white_databases=qw(information_schema db_infobase mysql performance_schema test sys); +my @backup_databases; +my @other_engines; +eval{ + local $SIG{__DIE__} = ""; + $sql="show databases"; + my $select=$dbh->prepare($sql); + $select->execute(); + while(my $result=($select->fetchrow_array)[0]){ + if(not grep(/$result/,@white_databases)){ + push(@backup_databases,$result); + } + } + if(@backup_databases){ + my $database_string=join("','",@backup_databases); + $sql="select distinct engine from information_schema.tables where table_schema in('$database_string')"; + $select=$dbh->prepare($sql); + $select->execute(); + while(my @row_array=$select->fetchrow_array){ + my ($engine)=@row_array; + if(not $engine eq 'TokuDB'){ + push(@other_engines,$engine); + } + } + } +}; +if($@){ + die "check engine for all backup table failed:$@\n"; +} + + +my $metadata_before_backup=qx(ls -l $data_dir $tokudb_log_dir/*); +##### STEP 3. get checkpoint lock +#set tokudb_checkpoint_lock=on:let dml only change redo log +$row_ref=$dbh->selectrow_arrayref("select * from information_schema.global_variables where variable_name='tokudb_checkpoint_lock'"); +print $debug_log "global:$row_ref->[0]:$row_ref->[1]\n"; +$row_ref=$dbh->selectrow_arrayref("select * from information_schema.session_variables where variable_name='tokudb_checkpoint_lock'"); +print $debug_log "session:$row_ref->[0]:$row_ref->[1]\n"; +$dbh->do("SET TOKUDB_CHECKPOINT_LOCK=ON;") or die "Get tokudb_checkpoint lock failed"; +print $debug_log qx(date); +$row_ref=$dbh->selectrow_arrayref("select * from information_schema.global_variables where variable_name='tokudb_checkpoint_lock'"); +print $debug_log "global:$row_ref->[0]:$row_ref->[1]\n"; +$row_ref=$dbh->selectrow_arrayref("select * from information_schema.session_variables where variable_name='tokudb_checkpoint_lock'"); +print $debug_log "session:$row_ref->[0]:$row_ref->[1]\n"; +print $debug_log "set tokudb_checkpoint_lock=on ok\n\n"; + + +##### STEP 4. Copy tokudb.* and redo log, and get binlog position in a blocking-binlog or stopping-slave; +if(defined $dump_slave){ + $dbh->do("set \@old_rpl_stop_slave_timeout=\@\@rpl_stop_slave_timeout;") or die("failed to get the value of rpl_stop_slave_timeout"); + print($debug_log "rpl_stop_slave_timeout:".(($dbh->selectrow_arrayref("select \@\@rpl_stop_slave_timeout"))->[0])."\n"); + $dbh->do("set global rpl_stop_slave_timeout=10;") or die("failed to set the value of rpl_stop_slave_timeout"); + print($debug_log qx(date)); + my $ret=$dbh->do("stop slave;"); + print($debug_log qx(date)."\n"); + $dbh->do("set global rpl_stop_slave_timeout=\@old_rpl_stop_slave_timeout;") or die("failed to recover the value of rpl_stop_slave_timeout"); + if(not defined $ret){#Assuming that there are dead locks,and stop slave time out + my $dbh = DBI->connect ("DBI:mysql:mysql:host=$host:port=$port:mysql_socket=$sock", $user, $password); + $ret=qx(mysql -h $host -P $port -u$user -p$password -e "show full processlist") or die "failed to execute 'show full processlist':$!"; + print($debug_log $ret); + $dbh->do("SET TOKUDB_CHECKPOINT_LOCK=OFF;") or die "release tokudb_checkpoint lock failed"; + system("rm -rf $backdir/mysql_data")==0 or die "failed:$!"; + close $debug_log; + close $backed_debug_log; + exit(223); + } + print $debug_log qx(date); + print $debug_log "stop slave ok\n\n"; +}else{ + $dbh->do("lock binlog for backup;") or die "lock binlog for backup failed"; + print $debug_log qx(date); + print $debug_log "lock binlog for backup ok\n\n"; +} + +print $backed_debug_log "Binlog time: " . qx(date)."\n"; +print $debug_log "Binlog time: " . qx(date)."\n"; +my $binlog_stamp=time; + +print $debug_log qx(date); +print $debug_log "copy tokudb redolog in: $tokudb_log_dir ..."; +system("ls $tokudb_log_dir| xargs -I '{}' cp -r $tokudb_log_dir/{} $backdir/tokudb_log")==0 or die "failed:$!\n"; +print $debug_log "\tdone.\n"; +print $debug_log qx(date)."\n"; + +print $debug_log qx(date); +print $debug_log "copy tokudb rollback log ..."; +system("ls $data_dir/tokudb.*| xargs -I '{}' cp -r {} $backdir/")==0 or die "failed:$!"; +print $debug_log "\tdone.\n"; +print $debug_log qx(date)."\n"; + +#get tokudb_data list +my @tokudb_data_files; +open FILELIST,">$backdir/tokudb_data/filelist.txt" or die "can't open $backdir/tokudb_data/filelist.txt:$!\n"; +my $select_file=$dbh->prepare("select distinct internal_file_name from information_schema.TokuDB_file_map"); +$select_file->execute(); +while(my @tokudb_files=$select_file->fetchrow_array){ + push(@tokudb_data_files,$tokudb_files[0]); + printf(FILELIST "$tokudb_files[0]\n"); +} +close FILELIST; + +$sql = qq{show master status}; +$row_ref = $dbh->selectrow_hashref($sql); +if(defined $row_ref and defined $row_ref->{'File'} and defined $row_ref->{'Position'} ){ + open(my $master_info, ">", "$backdir/xtrabackup_binlog_info") or die "could not open binlog_info file\n"; + print $master_info "$row_ref->{'File'} $row_ref->{'Position'}\n"; + close $master_info; +}else{ + die "Get master info failed\n"; +} + +if(defined $dump_slave){ + $sql = qq{show slave status}; + $row_ref = $dbh->selectrow_hashref($sql); + if (defined $row_ref and defined $row_ref->{'Relay_Master_Log_File'} and defined $row_ref->{'Exec_Master_Log_Pos'}) + { + open(my $slave_info, ">", "$backdir/xtrabackup_slave_info") or die "could not open slave_info file"; + print $slave_info "CHANGE MASTER TO MASTER_LOG_FILE='$row_ref->{Relay_Master_Log_File}', MASTER_LOG_POS=$row_ref->{'Exec_Master_Log_Pos'} \n"; + close $slave_info; + }else{ + die "Get salve info failed\n"; + } +} + +print $debug_log qx(date); +print $debug_log "copy mysql configure file: $myconf ..."; +system("cp $myconf $backdir/backup-my.cnf") ==0 or die "failed:$!\n"; +print $debug_log "\tdone.\n"; +print $debug_log qx(date)."\n"; + +if(defined $dump_slave){ + $dbh->do("start slave") or die "start slave faild\n"; + print $debug_log qx(date); + print $debug_log "start slave ok,time:".(qx(date))."\n"; +}else{ + $dbh->do("unlock binlog") or die "start slave faild\n"; + print $debug_log qx(date); + print $debug_log "unlock binlog ok,time:".(qx(date))."\n"; +} + + +##### STEP 5. Copy frm +print $debug_log qx(date); +print $debug_log "copy mysql data dir: $data_dir/ ..."; +system("ls $data_dir| xargs -I '{}' cp -r $data_dir/{} $backdir/mysql_data")==0 or die "failed:$!"; +print $debug_log "\tdone.\n"; +print $debug_log qx(date)."\n"; + +##### STEP 6. recovery tokudb_commit_sync; +#$ret=$dbh->do("set global tokudb_commit_sync=$old_global_tokudb_commit_sync"); +#die "unable to restore tokudb_commit_sync" if not defined $ret or $ret<0; +##sleep(100); +#print qx(date); +#print "enable tokudb redo log buffer ok\n\n"; +# +#goto SKIP; +##### STEP 7. copy data or increment data +if(is_low_space("$Bin/history_backup_size","$Bin/last_backup_size",$backdir,0.96,$debug_log)){ + push(@tokudb_backup_warnings,"SMS#low space,previous file for port:${port} will be deleted,and a fully backup will be done today"); + #delete old file + print $debug_log "\tlow space,delete old file and then fully backup"; + qx(ls $backdir|grep _${port}_|grep _${host}_|xargs rm -rf); + #fully backup + foreach my $file(@tokudb_data_files){ + system("cp -r $tokudb_data_dir/$file $backdir/tokudb_data") ==0 or die "copy $tokudb_data_dir/$file to $backdir/tokudb_data failed:$!"; + } +}elsif(not $increment){ + print $debug_log qx(date); + print $debug_log "fully copy tokudb data dir: $tokudb_data_dir/ ..."; + foreach my $file(@tokudb_data_files){ + system("cp -r $tokudb_data_dir/$file $backdir/tokudb_data") ==0 or die "copy $tokudb_data_dir/$file to $backdir/tokudb_data failed:$!"; + } + #system("ls $tokudb_data_dir|xargs -I '{}' cp -r $tokudb_data_dir/{} $backdir/tokudb_data") ==0 or die "failed"; +}else{ + print $debug_log qx(date); + print $debug_log "incrementally backup($tokudb_data_dir/)"; + my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time()+$global_time_offset); + if(($port%7)==$wday){ #default fully backup day,20000%7=1 + print $debug_log "\tfully day"; + foreach my $file(@tokudb_data_files){ + system("cp -r $tokudb_data_dir/$file $backdir/tokudb_data") ==0 or die "copy $tokudb_data_dir/$file to $backdir/tokudb_data failed:$!"; + } + #system("ls $tokudb_data_dir|xargs -I '{}' cp -r $tokudb_data_dir/{} $backdir/tokudb_data") ==0 or die "failed"; + }else{ +#1 get the fully backup up info file + print $debug_log "\tincremental day"; + opendir (DIR,$dir) or die "can't open the directory $dir"; + my @dirs=readdir DIR; + close DIR; + + #my $fully_backup_port=$wday; + #while($fully_backup_port<$port){$fully_backup_port+=7;} + #my $days_since_fully=$fully_backup_port-$port; + +# time = strftime("%Y%m%d_%H%M%S", localtime(time)); +# target_name = "${app_name}_${hostname}_${host_address}_${port}_${time}"; +# fully day info file name should be: \S+_${host}_${port}_${fullyday}_\d{6}\.info + #print "\tport:$port fully_backup_port:$fully_backup_port days_since_fully:$days_since_fully"; + my $tmpstamp; + my $tmpname; + my $i; + my $fully_stamp_and_name={}; + for($i=0;$i<7;$i++){ + my $fullyday=strftime("%Y%m%d", localtime($fully_stamp-$i*24*3600)); + #print "\$port:$port \$fully_backup_port:$fully_backup_port \$days_since_fully:$days_since_fully \$fullyday:$fullyday\n"; + #print "now begin to exam ".(scalar @dirs)." file\n"; + foreach my $file(@dirs){ + chomp($file); + if($file =~ /(\S+_${port}_${fullyday}_\d{6}_TOKUDB_INCREASE)\.info$/){ + my $current_name=$1; + open INFO,"<$dir/$file" or print $debug_log "unable to open $dir/$file",last; + #print "\nexaming file: $file\n"; + while(){ + chomp; + my $tmpstr=$_; + if($tmpstr =~ /\s*FULLY_STAMP\s*=\s*(\d+)/){ + $tmpstamp=$1; + }elsif($tmpstr=~/\s*FULLY_NAME\s*=\s*(\S+)/){ + $tmpname=$1; + } + } + close INFO; + if(defined $tmpstamp and defined $tmpname and $tmpname eq $current_name){#find fully backup info file + $fully_stamp_and_name->{$tmpstamp}=$tmpname; + } + $tmpstamp=undef; + $tmpname=undef;#for multi fully backup one day + } + } + if(scalar keys %$fully_stamp_and_name){ + last; + } + } + my @fully_stamps=sort keys %$fully_stamp_and_name; + if(@fully_stamps){ #previous backup is ok + print $debug_log "\tincremental backup"; + $fully_stamp=$fully_stamps[-1]; + $fully_name=$fully_stamp_and_name->{$fully_stamp}; + print $debug_log "\nfully_stamp:$fully_stamp\nfully_name:$fully_name\n"; + foreach my $file(@tokudb_data_files){ + my($device, $inode, $mode, $nlink, $uid, $gid, $rdev, $size,$atime, $mtime, $ctime, $blksize, $blocks) = stat("$tokudb_data_dir/$file"); + #printf("$file\tmtime:$mtime($fully_stamp)\tctime:$ctime($fully_stamp)\n"); + if($mtime>$fully_stamp ||($ctime>$fully_stamp)){ + system("cp $tokudb_data_dir/$file $backdir/tokudb_data") ==0 or die "failed:$!"; + #printf("copied $file\n"); + } + } + }else{ #previous backup failed + #print "\tfully backup(i:$i fully_stamp:$fully_stamp)"; + print $backed_debug_log "\ttmpstamp:$tmpstamp" if defined $tmpstamp; + print $debug_log "\ttmpstamp:$tmpstamp" if defined $tmpstamp; + foreach my $file(@tokudb_data_files){ + system("cp -r $tokudb_data_dir/$file $backdir/tokudb_data") ==0 or die "copy $tokudb_data_dir/$file to $backdir/tokudb_data failed:$!"; + } + #system("ls $tokudb_data_dir|xargs -I '{}' cp -r $tokudb_data_dir/{} $backdir/tokudb_data") ==0 or die "failed"; + } + } +} +SKIP: +print $debug_log "\tdone.\n"; +print $debug_log qx(date)."\n"; + +##### STEP 8. check file change time +my $metadata_after_backup=qx(ls -l $data_dir $tokudb_log_dir/*); +print $backed_debug_log "############################################# metadata before backup ######################################################\n"; +print $backed_debug_log $metadata_before_backup; +print $backed_debug_log "############################################# metadata after backup ######################################################\n"; +print $backed_debug_log $metadata_after_backup; +print $debug_log "############################################# metadata before backup ######################################################\n"; +print $debug_log $metadata_before_backup; +print $debug_log "############################################# metadata after backup ######################################################\n"; +print $debug_log $metadata_after_backup; +#begin check that all file's ctime should not be new than $binlog_stamp +my @changed_file; +opendir (DIR,$tokudb_data_dir) or die "can't open the directory $dir"; +my @dirs=readdir DIR; +close DIR; +print $backed_debug_log "#################################################### changed file ###########################################################\n"; +print $debug_log "#################################################### changed file ###########################################################\n"; +foreach my $file(@dirs){ + if($file eq "\." or $file eq "\.\."){next;} + my($device, $inode, $mode, $nlink, $uid, $gid, $rdev, $size,$atime, $mtime, $ctime, $blksize, $blocks) = stat("$tokudb_data_dir/$file"); + if($mtime>$binlog_stamp){ + my $full_file_path="$tokudb_data_dir/$file"; + my $ls_l_of_file=qx(ls -l --time-style='+%Y-%m-%d %H:%M:%S' $full_file_path); + my $adate=strftime("%Y-%m-%d %H:%M:%S", localtime($atime)); + my $mdate=strftime("%Y-%m-%d %H:%M:%S", localtime($mtime)); + my $cdate=strftime("%Y-%m-%d %H:%M:%S", localtime($ctime)); + my $bdate=strftime("%Y-%m-%d %H:%M:%S", localtime($binlog_stamp)); + chomp($ls_l_of_file); + print $backed_debug_log "$ls_l_of_file\tchanged:a($adate) m($mdate) c($cdate) b($bdate)\n"; + print $debug_log "$ls_l_of_file\tchanged:a($adate) m($mdate) c($cdate) b($bdate)\n"; + #print "$tokudb_data_dir/$file changed\n"; + push(@changed_file,"$tokudb_data_dir/$file"); + } +} + +##### STEP 9. release metadata lock +if($tmysql_ver ge tmysql_version_parser("tmysql-2.1.3")){ + $dbh->do("unlock tables") or die "failed to release metadata lock"; + print $debug_log qx(date); + print $debug_log "release all metadata lock ok\n\n"; +} + + +##### STEP 10. release tokudb_checkpoint_lock +$dbh->do("SET TOKUDB_CHECKPOINT_LOCK=OFF") or die "release tokudb checkpoint lock failed"; +$row_ref=$dbh->selectrow_arrayref("select * from information_schema.global_variables where variable_name='tokudb_checkpoint_lock'"); +print $debug_log "global:$row_ref->[0]:$row_ref->[1]\n"; +$row_ref=$dbh->selectrow_arrayref("select * from information_schema.session_variables where variable_name='tokudb_checkpoint_lock'"); +print $debug_log "session:$row_ref->[0]:$row_ref->[1]\n"; +print $debug_log qx(date); +print $debug_log "set tokudb_checkpoint_lock=off ok\n\n"; +$dbh->disconnect; + + +##### STEP 11. fully backup info to file +#save fully backup info to file +open my $latest_fully_backup_info,">$Bin/latest_fully_backup_info.$port" +or die("ERROR: Can not open latest fully backup info file $Bin/latest_fully_backup_info.$port for $@"); +printf $latest_fully_backup_info "FULLY_STAMP=$fully_stamp\n"; +printf $latest_fully_backup_info "FULLY_NAME=$fully_name\n"; +close($latest_fully_backup_info); +if(scalar @changed_file){ + #print "changed file:".(join(',',@changed_file))."\n"; + my $file_string=join(",",@changed_file); + print $backed_debug_log "changed file:$file_string\n"; + print $debug_log "changed file:$file_string\n"; + push(@tokudb_backup_warnings,"INFO#warning:some file changed during backup,see more information in $debug_log_file"); +} +if(scalar @other_engines){ + my $other_engine_string=join(",",@other_engines); + my $backup_databases_string=join(",",@backup_databases); + push(@tokudb_backup_warnings,"SMS#engine other than tokudb exists, other engine:$other_engine_string database:$backup_databases_string"); + print "warning:engine other than tokudb($other_engine_string) exists in $backup_databases_string\n"; +} +print_log("$Bin/tokudb_backup.log.$port",@tokudb_backup_warnings); +close $backed_debug_log; +close $debug_log; +#system("date '+%s'> $backdir/TOKUDB.END"); +$date=strftime("%Y%m%d_%H%M%S", localtime(time()+$global_time_offset)); +system("echo $date >> $backdir/TOKUDB.END"); + +sub print_log{ +#logarray entry format:level#message + my($logfile,@logarray)=@_; + if(scalar @logarray){ + open LOG,">$logfile" or die "unable to open $logfile"; + foreach my $log(@logarray){ + print LOG $log; + } + close LOG; + }else{ + if(-e $logfile){ + system("rm $logfile")==0 or print "WARNING:failed to rm previous logfile:$logfile:$!\n"; + } + } +} +sub get_history_max_backup_size{ + my($history_file)=@_; + if(-e $history_file and -r $history_file){ + my $max=qx(cat $history_file|grep _${port}_|tail -7|awk '{print \$2}'|sort -n|tail -1); + chomp($max) if defined $max; + if(defined $max and $max=~/\d+/){ + return $max; + }else{ + return 0; + } + }else{ + return 0; + } +} +sub is_low_space{ + my($history_file,$last_file,$backup_dir,$max_percent,$debug_log)=@_; + my $max_size=get_history_max_backup_size($history_file); + my $last_size=get_history_max_backup_size($last_file); + my $max=($max_size>$last_size)?$max_size:$last_size; + if($max==0){ + return 0; + }else{ + my $backup_disk = (split(/\//, $backup_dir))[1]; + my $disk_total = qx#df -P /$backup_disk | sed '1d' | awk '{print \$2}'#; + my $disk_used = qx#df -P /$backup_disk | sed '1d' | awk '{print \$3}'#; + chomp $disk_total; #in kB + chomp $disk_used; #in kB + $disk_total *= 1024; #to Byte + $disk_used *= 1024; #to Byte + my $space_may_used=$disk_used+$max; + my $warn_level=$disk_total*$max_percent; + print $debug_log "total-space:${disk_total}B used-space:${disk_used}B backup-may-use:${max}B warn-level:${warn_level}B \n"; + if($space_may_used>$warn_level){ + print $debug_log "use-space + space-backup-may-use=${space_may_used}B > warn_level=${warn_level}B\n"; + return 1; + }else{ + print $debug_log "use-space + space-backup-may-use=${space_may_used}B <= warn_level=${warn_level}B\n"; + return 0; + } + } +} \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup-go-deps-txsql/bin/tokudb_recovery.pl b/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup-go-deps-txsql/bin/tokudb_recovery.pl new file mode 100644 index 0000000000..71af7443ba --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup-go-deps-txsql/bin/tokudb_recovery.pl @@ -0,0 +1,100 @@ +#!/usr/bin/perl + +use FindBin qw($Bin); + +use warnings; +use strict; +use Getopt::Long qw(:config posix_default); +use POSIX qw(strftime); +use File::Basename; +use Data::Dumper; +use DBI; + + +my $backdir= undef; +my $myconf = undef; +my $increment=0; + +my @normal_dbs = ("mysql", "test", "performance_schema", "db_infobase", "sys"); + +Getopt::Long::Configure ("bundling_override"); +GetOptions( + 'backup-path=s' => \$backdir, + 'defaults-file=s' => \$myconf, + 'increment|i!' => \$increment +)or die "usage: xxx --backup-path={path} --defaults=file={path} [<-i|increment> ]"; + + +unless (defined $backdir && defined $myconf){ + die "back-path or defaults-file undefined"; +} + +die "$backdir is not a dir" unless( -d $backdir); +my $fully_dir=undef; +if($increment){ + die "when enable increment, fully-backup-file-dir is needed!" unless(@ARGV==1); + die "$ARGV[0] is not a dir" unless(-d $ARGV[0]); + $fully_dir=$ARGV[0]; +} +my $data_dir = undef; +my $tokudb_data_dir = undef; +my $tokudb_log_dir = undef; + +open( my $my_cnf_fp, "< $myconf" ) or die "$myconf open error" ; +while ( my $line = <$my_cnf_fp> ) { + chomp $line; + if ( not defined $data_dir ) { + if ( $line =~ /\s*datadir\s*=\s*([\S]+)/ ) { + $data_dir = $1; + next; + } + } + if ( not defined $tokudb_data_dir ) { + if ( $line =~ /\s*tokudb_data_dir\s*=\s*([\S]+)/ ) { + $tokudb_data_dir = $1; + next; + } + } + if ( not defined $tokudb_log_dir ) { + if ( $line =~ /\s*tokudb_log_dir\s*=\s*([\S]+)/ ) { + $tokudb_log_dir = $1; + next; + } + } +} +close($my_cnf_fp); + +unless (defined $data_dir && + defined $tokudb_data_dir && + defined $tokudb_log_dir ) +{ + die "some key in $myconf lost"; +} + +system("ls $backdir/tokudb_log|xargs -I '{}' cp -r $backdir/tokudb_log/{} $tokudb_log_dir") == 0 or die "failed"; +if($increment){ + if(!open FILELIST,"<$backdir/tokudb_data/filelist.txt"){ + die "open filelist.txt failed"; + } + while(){ + chomp; + my $file=$_; + if(-e "$backdir/tokudb_data/$file"){ + $file="$backdir/tokudb_data/$file"; + }else{ + die "can't find $file in both $backdir/tokudb_data and $fully_dir/tokudb_data" unless(-e ($file="$fully_dir/tokudb_data/$file")); + } + system("cp $file $tokudb_data_dir") == 0 or die "failed"; + } +}else{ + eval{system("ls $backdir/tokudb_data |xargs -I '{}' cp -r $backdir/tokudb_data/{} $tokudb_data_dir")}; die $@ if $@; +} +system("ls $backdir/tokudb.*|xargs -I '{}' cp {} $data_dir") == 0 or die "failed"; + +foreach my $file (glob("$backdir/mysql_data/*/")){ + my $tmp_file=basename($file); + if(not grep(/$tmp_file/,@normal_dbs) ){ + print "copy database: $tmp_file\n"; + system("cp -r $file $data_dir") == 0 or die "failed"; + } +} \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup-go-deps-txsql/lib/.gitkeep b/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup-go-deps-txsql/lib/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup_main.sh b/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup_main.sh index dbfe710d54..237dad7b0b 100755 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup_main.sh +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/dbbackup_main.sh @@ -92,8 +92,8 @@ errPorts="" okPorts="" for conf_file in $configFiles do - #port=`echo $conf_file |awk -F. '{print $(NF-1)}'` - port=`grep MysqlPort $conf_file |grep -Ev "#|MysqlPort = 0" |head -1 | cut -d= -f2` + port=`grep -Ei 'MysqlPort|\[Public\]' $conf_file |grep -v '^#' | grep -A1 'Public' |grep 'MysqlPort'` + port=$(echo `echo $port |head -1 | cut -d= -f2`) echo "now doing dbbackup for config file=$conf_file port=$port" echo "${scriptDir}/dbbackup dumpbackup --config=$conf_file $dbbackupOpt 2>&1 >> $logfile" ${scriptDir}/dbbackup dumpbackup --config=$conf_file $dbbackupOpt 2>&1 >> $logfile @@ -104,7 +104,7 @@ do fi done # 输出可用于判断哪些 ports 成功，哪些失败 -echo "okPorts:$okPorts,errPorts:$errPorts" +echo "okPorts:$okPorts, errPorts:$errPorts" if [ -n "$errPorts" ];then echo "ports backup failed: $errPorts" >&2 exit 1 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md index 7a52e750a1..197c01ed08 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md @@ -74,7 +74,40 @@ IOLimitMBPerSec = 300 ### 6. 关于 tendbcluster 集群备份，请参考 [spider](spiderbackup.md) -### 常见备份失败处理 +### 7. 关于逻辑备份字符集说明 -1. log copying being too slow -it looks like InnoDB log has wrapped around before xtrabackup could process all records due to either log copying being too slow, or log files being too small. \ No newline at end of file + +### 8. 常见备份失败处理 + +#### 1. log copying being too slow +> it looks like InnoDB log has wrapped around before xtrabackup could process all records due to either log copying being too slow, or log files being too small. + +实例写入速度比备份速度快，可能是刚刚备份时间段有大批量 DML 操作，修改备份开始时间，或者加快备份速度: + +调大 PhysicalBackup.Throttle, 调大 PhysicalBackup.Threads + +#### 2. mydumper 不支持 centos 6.x +> mydumper: error while loading shared libraries: libpcre.so +> +> /lib64/libc.so.6: version `GLIBC_2.14' not found (required by xxx) + +mydumper / myloader 依赖 glibc>=2.14, centos 6.x(or tlinux 1.2) 是 glibc 2.12，可能会报如上错误。查看 glibc 版本`ldd --version |grep libc`。 + +如果必须使用逻辑备份，可以设置 +``` +[LogicalBackup] +UseMysqldump = auto +Databases = * +Tables = * + +[Public] +BackupType = logical +``` +使用 mysqldump 备份 slave 数据，会短暂停止同步 sql thread来获取一致性 binlog 位点，可能会触发告警。备份结束(成功/失败)会自动恢复 sql thread（非 kill 掉的情况） + +#### 3. xtrabackup 8.0 不支持 centos 6.x +> xtrabackup: error while loading shared libraries: libsystemd.so.0: cannot open shared object file: No such file or directory +> +> /usr/lib64/libstdc++.so.6: version `GLIBCXX_3.4.15' not found (required by xxx) + +mysql 8.0 的物理备份工具 xtrabackup 也依赖 glibc>=2.14 版本，可能会看到如上报错。 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md index e4f78593bd..e56ede465f 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md @@ -319,4 +319,36 @@ Threads = 4 CopyBack = false IndexFilePath = xx DefaultsFile = /etc/my.cnf.3306 -``` \ No newline at end of file +``` + +## 参数解释 +### Public +- Public.KillLongQueryTime + 这个参数对逻辑备份和物理备份作用不同 + - 逻辑备份 mydumper + 相当于`--kill-long-queries --long-query-guard xx`: 发出 FTWRL 之前如果发现有超过这个时间的长 sql，则 kill 掉 + - 物理备份 xtrabackup + 相当于`--kill-long-queries-timeout=xx`: 发出 FTWRL 之后如果被阻塞，则等待多久之后把引起阻塞的长 sql kill 掉 + +需要备份账号有 super 权限。默认为 0 则不 kill。 + +- Public.FtwrlWaitTimeout + 发起备份前检查长 sql，(如果不自动 kill/ kill失败) 则等待长 sql 多久后，放弃 ftwrl，放弃备份。 + 此时还未发起 `FLUSH TABLE WITH READ LOCK` 命令 + 默认 120s，对 mydumper / xtrabackup 有效 + +- Public.AcquireLockWaitTimeout + 备份加锁超时，比如 `LOCK TABLES FOR BACKUP` / `FLUSH TABLE WITH READ LOCK`，相当于 `set session lock-wait-timeout=xxx` + 默认 10s 超时，对 mydumper / xtrabackup 有效 + +- Public.BackupTimeOut + 备份超时结束时间，只对 master 有效，用于保护 master 不被备份影响。 + 默认`09:00:00`，即备份执行到这个时间点还未结束，则退出 + +- Public.OldFileLeftDay + 备份文件本地最大保留时间天数，每次备份之前会先清理旧的备份。如果备份空间不足，可能会继续清理备份文件，优先保证备份成功 + +- Public.IOLimitMasterFactor + master机器专用限速因子，因为备份速度可能有多个选项来控制 + - master 文件io打包限速: `Public.IOLimitMBPerSec * IOLimitMasterFactor` + - 物理备份限速: `PhysicalBackup.Throttle * IOLimitMasterFactor` \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/defaults.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/defaults.go index 5c1f265e88..73ea9ac8f6 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/defaults.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/defaults.go @@ -25,6 +25,9 @@ func SetDefaults() { viper.SetDefault("Public.BackupDir", "/data/dbbak") //viper.SetDefault("Public.MysqlCharset", "binary") viper.SetDefault("Public.BackupTimeOut", "09:00:01") + viper.SetDefault("Public.KillLongQueryTime", 0) + viper.SetDefault("Public.FtwrlWaitTimeout", 120) + viper.SetDefault("Public.AcquireLockWaitTimeout", 10) viper.SetDefault("PhysicalBackup.MaxMyisamTables", 10) viper.SetDefault("PhysicalBackup.DisableSlaveMultiThread", false) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go index cdf209a8f0..93a26e8ed0 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go @@ -75,6 +75,20 @@ type Public struct { // EncryptOpt backup files encrypt options EncryptOpt *cmutil.EncryptOpt `ini:"EncryptOpt"` + // KillLongQueryTime . if 0 not kill. Default 0 + // mydumper --kill-long-queries --long-query-guard xx: 发出 FTWRL 之前如果发现有超过这个时间的长 sql，则 kill 掉 + // xtrabackup --kill-long-queries-timeout: 发出 FTWRL 之后如果被阻塞，则等待多久之后把引起阻塞的长 sql kill 掉 + KillLongQueryTime int `ini:"KillLongQueryTime"` + // FtwrlWaitTimeout 在发出 FTWRL 之前，如果有长 sql，则等待长 sql 多久后，放弃 ftwrl，放弃备份. Default 120s + // 长 sql 的判断标准是 running time > --ftwrl-wait-threshold ( --lock-wait-threshold for 5.6) + // xtrabackup --ftwrl-wait-timeout (--lock-wait-timeout for 5.6) + // mydumper --long-query-guard + FtwrlWaitTimeout int `ini:"FtwrlWaitTimeout"` + // AcquireLockWaitTimeout If LOCK TABLES FOR BACKUP does not return within given timeout, abort the backup. + // issue lock to mysqld: default 10s + // lock-ddl-timeout(xtrabackup57), backup-lock-timeout(xtrabackup80) --lock-wait-timeout(mydumper) + AcquireLockWaitTimeout int `ini:"AcquireLockWaitTimeout"` + cnfFilename string targetName string // isFullBackup 1: true, -1: false, 0: unknown diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go index afe95d0fe4..1980fdd218 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/validate" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" @@ -15,7 +16,6 @@ import ( "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck" - "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util" ) // Dumper TODO @@ -45,7 +45,7 @@ func BuildDumper(cnf *config.BackupConfig, db *sql.DB) (dumper Dumper, err error if strings.ToLower(cnf.Public.BackupType) == cst.BackupLogical { if cnf.LogicalBackup.UseMysqldump == cst.LogicalMysqldumpAuto || cnf.LogicalBackup.UseMysqldump == "" { - if glibcVer, err := util.GetGlibcVersion(); err != nil { + if glibcVer, err := cmutil.GetGlibcVersion(); err != nil { logger.Log.Warn("failed to glibc version, err:", err) } else if glibcVer < "2.14" { // mydumper need glibc version >= 2.14 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go index 4ceebf8e8a..ea23d54bab 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go @@ -65,15 +65,29 @@ func (l *LogicalDumper) Execute(enableTimeOut bool) error { "-u", l.cnf.Public.MysqlUser, "-p", l.cnf.Public.MysqlPasswd, "-o", filepath.Join(l.cnf.Public.BackupDir, l.cnf.Public.TargetName()), + "--trx-consistency-only", + "--long-query-retry-interval=10", fmt.Sprintf("--long-query-retries=%d", l.cnf.LogicalBackup.FlushRetryCount), fmt.Sprintf("--set-names=%s", l.cnf.Public.MysqlCharset), fmt.Sprintf("--chunk-filesize=%d", l.cnf.LogicalBackup.ChunkFilesize), fmt.Sprintf("--threads=%d", l.cnf.LogicalBackup.Threads), - "--trx-consistency-only", - "--long-query-retry-interval=10", // "--disk-limits=1GB:5GB", } - + if l.cnf.Public.KillLongQueryTime > 0 { + args = append(args, "--kill-long-queries", + fmt.Sprintf("--long-query-guard=%d", l.cnf.Public.KillLongQueryTime)) + } else { + if l.cnf.Public.FtwrlWaitTimeout > 0 { + args = append(args, fmt.Sprintf("--long-query-guard=%d", l.cnf.Public.FtwrlWaitTimeout)) + } else { + args = append(args, "--long-query-guard=999999") // 不退出 + } + } + /* + if l.cnf.Public.AcquireLockWaitTimeout > 0 { + args = append(args, fmt.Sprintf("--lock-wait-timeout=%d", l.cnf.Public.AcquireLockWaitTimeout)) + } + */ if !l.cnf.LogicalBackup.DisableCompress { args = append(args, "--compress") } @@ -124,7 +138,11 @@ func (l *LogicalDumper) Execute(enableTimeOut bool) error { }...) } } - // ToDo extropt + /* + if l.cnf.LogicalBackup.ExtraOpt != "" { + args = append(args, l.cnf.LogicalBackup.ExtraOpt) + } + */ var cmd *exec.Cmd if enableTimeOut { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go index d64dbd96ec..67ec171b60 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go @@ -189,7 +189,7 @@ func (l *LogicalDumperMysqldump) Execute(enableTimeOut bool) (err error) { "--single-transaction", "--master-data=2", "--max-allowed-packet=1G", "--no-autocommit", "--hex-blob", - // "--set-gtid-purged=off", // 5.7 需要 + // "--set-gtid-purged=off", // 5.7 需要 如果支持 } args = append(args, "--default-character-set", l.cnf.Public.MysqlCharset) if l.cnf.Public.MysqlRole == cst.RoleSlave { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go index b8fc460ff9..d8a5a1dd09 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go @@ -1,7 +1,6 @@ package backupexe import ( - "bytes" "context" "fmt" "os" @@ -90,35 +89,52 @@ func (p *PhysicalDumper) buildArgs() []string { fmt.Sprintf("--password=%s", p.cnf.Public.MysqlPasswd), "--compress", } + if p.cnf.PhysicalBackup.Threads > 0 { + args = append(args, []string{ + fmt.Sprintf("--compress-threads=%d", p.cnf.PhysicalBackup.Threads), + fmt.Sprintf("--parallel=%d", p.cnf.PhysicalBackup.Threads), + }...) + } targetPath := filepath.Join(p.cnf.Public.BackupDir, p.cnf.Public.TargetName()) if strings.Compare(p.mysqlVersion, "005007000") < 0 { args = append(args, targetPath) } else { - args = append(args, - fmt.Sprintf("--target-dir=%s", targetPath), - "--backup", "--lock-ddl", - ) + args = append(args, fmt.Sprintf("--target-dir=%s", targetPath), "--backup") + } + if strings.Compare(p.mysqlVersion, "005007000") > 0 { + args = append(args, "--lock-ddl") + if strings.Compare(p.mysqlVersion, "008000000") < 0 { // ver >=5.7 and ver < 8.0 + args = append(args, "--binlog-info=ON") + } } - if p.cnf.PhysicalBackup.Threads > 0 { - args = append(args, []string{ - fmt.Sprintf("--compress-threads=%d", p.cnf.PhysicalBackup.Threads), - fmt.Sprintf("--parallel=%d", p.cnf.PhysicalBackup.Threads), - }...) + if p.cnf.Public.KillLongQueryTime > 0 { // all version support + args = append(args, fmt.Sprintf("--kill-long-queries-timeout=%d", p.cnf.Public.KillLongQueryTime)) + } + if p.cnf.Public.FtwrlWaitTimeout > 0 { + if strings.Compare(p.mysqlVersion, "005007000") >= 0 { + args = append(args, fmt.Sprintf("--ftwrl-wait-timeout=%d", p.cnf.Public.FtwrlWaitTimeout)) + } else { // 5.5, 5.6 + args = append(args, fmt.Sprintf("--lock-wait-timeout=%d", p.cnf.Public.FtwrlWaitTimeout)) + } + } + if p.cnf.Public.AcquireLockWaitTimeout > 0 { + if strings.Compare(p.mysqlVersion, "005007000") > 0 { + args = append(args, fmt.Sprintf("--lock-ddl-timeout=%d", p.cnf.Public.AcquireLockWaitTimeout)) + } + if strings.Compare(p.mysqlVersion, "008000000") >= 0 { + args = append(args, fmt.Sprintf("--backup-lock-timeout=%d", p.cnf.Public.AcquireLockWaitTimeout)) + } } if p.cnf.PhysicalBackup.Throttle > 0 { - args = append(args, []string{ - fmt.Sprintf("--throttle=%d", p.cnf.PhysicalBackup.Throttle), - }...) + args = append(args, fmt.Sprintf("--throttle=%d", p.cnf.PhysicalBackup.Throttle)) } if strings.ToLower(p.cnf.Public.MysqlRole) == cst.RoleSlave { - args = append(args, []string{ - "--slave-info", - //"--safe-slave-backup", - }...) + // --safe-slave-backup + args = append(args, "--slave-info") } if strings.Compare(p.mysqlVersion, "008000000") >= 0 { @@ -128,12 +144,10 @@ func (p *PhysicalDumper) buildArgs() []string { } else { // xtrabackup_80 has no this args, and will report errors args = append(args, "--no-timestamp", "--lazy-backup-non-innodb", "--wait-last-flush=2") args = append(args, fmt.Sprintf("--ibbackup=%s", filepath.Join(p.dbbackupHome, p.innodbCmd.xtrabackupBin))) - if strings.Compare(p.mysqlVersion, "005007000") > 0 { - args = append(args, "--binlog-info=ON") - } } - - // ToDo extropt + if p.cnf.PhysicalBackup.ExtraOpt != "" { + args = append(args, p.cnf.PhysicalBackup.ExtraOpt) + } return args } @@ -191,8 +205,9 @@ func (p *PhysicalDumper) Execute(enableTimeOut bool) error { fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " "))) } - outFile, err := os.Create(filepath.Join(logger.GetLogDir(), - fmt.Sprintf("xtrabackup_%d_%d.log", p.cnf.Public.MysqlPort, int(time.Now().Weekday())))) + xtrabackupLogFile := filepath.Join(logger.GetLogDir(), + fmt.Sprintf("xtrabackup_%d_%d.log", p.cnf.Public.MysqlPort, int(time.Now().Weekday()))) + outFile, err := os.OpenFile(xtrabackupLogFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) if err != nil { logger.Log.Error("create log file failed: ", err) return err @@ -202,15 +217,22 @@ func (p *PhysicalDumper) Execute(enableTimeOut bool) error { }() cmd.Stdout = outFile - //cmd.Stderr = outFile - var stderr bytes.Buffer - cmd.Stderr = &stderr + cmd.Stderr = outFile // xtrabackup 的运行日志都是打印在 stderr ... unbelievable logger.Log.Info("xtrabackup command: ", cmd.String()) err = cmd.Run() if err != nil { - logger.Log.Error("run physical backup failed: ", err, stderr.String()) - return errors.WithMessage(err, stderr.String()) + errStrPrefix := fmt.Sprintf("tail 5 error from %s", xtrabackupLogFile) + errStrDetail, _ := util.GrepLinesFromFile(xtrabackupLogFile, []string{"ERROR", "fatal", "unknown"}, + 5, false, true) + if len(errStrDetail) > 0 { + logger.Log.Info(errStrPrefix) + logger.Log.Error(errStrDetail) + } else { + logger.Log.Warn("tail can not find more detail error message from ", xtrabackupLogFile) + } + logger.Log.Error("run physical backup failed", err, p.cnf.Public.MysqlPort) + return errors.WithMessagef(err, fmt.Sprintf("%s\n%s", errStrPrefix, errStrDetail)) } return nil } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go index de4a6f6ace..e1f91ad01a 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/mysqlcomm" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" @@ -74,8 +75,9 @@ func BackupGrant(cfg *config.Public) error { defer func() { _ = db.Close() }() - - rows, err := db.Query("select user, host from mysql.user where user not in ('ADMIN','yw','dba_bak_all_sel')") + usersExclude := []string{"ADMIN", "yw", "dba_bak_all_sel", "mysql.infoschema", "mysql.session", "mysql.sys"} + rows, err := db.Query(fmt.Sprintf("select `user`, `host` from `mysql`.`user` where `user` not in (%s)", + mysqlcomm.UnsafeIn(usersExclude, "'"))) if err != nil { logger.Log.Errorf("can't send query to Mysql server %v\n", err) return err diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go index dcea221446..f1f892468e 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go @@ -152,14 +152,13 @@ func (p *PhysicalLoader) apply() error { if err != nil { logger.Log.Error("physical apply failed: ", err, errStr) // 尝试读取 xtrabackup.log 里 ERROR 关键字 - grepError := []string{"grep", "-Ei", "ERROR|fatal", logfile, "|", "tail", "-5"} errStrPrefix := fmt.Sprintf("tail 5 error from %s", logfile) - errStrDetail, _, _ := cmutil.ExecCommand(true, "", grepError[0], grepError[1:]...) - if len(strings.TrimSpace(errStrDetail)) > 0 { + errStrDetail, _ := util.GrepLinesFromFile(logfile, []string{"ERROR", "fatal"}, 5, false, true) + if len(errStrDetail) > 0 { logger.Log.Info(errStrPrefix) logger.Log.Error(errStrDetail) } else { - logger.Log.Warn("can not find more detail error message from ", logfile) + logger.Log.Warn("tail can not find more detail error message from ", logfile) } return errors.WithMessagef(err, fmt.Sprintf("%s: %s\n%s", errStr, errStrPrefix, errStrDetail)) } @@ -217,16 +216,15 @@ func (p *PhysicalLoader) load() error { if err != nil { logger.Log.Error("xtrabackup copy data failed: ", err, errStr) // 尝试读取 xtrabackup.log 里 ERROR 关键字 - grepError := []string{"grep", "-Ei", "ERROR|fatal", logfile, "|", "tail", "-5"} errStrPrefix := fmt.Sprintf("tail 5 error from %s", logfile) - errStrDetail, _, _ := cmutil.ExecCommand(true, "", grepError[0], grepError[1:]...) - if len(strings.TrimSpace(errStrDetail)) > 0 { + errStrDetail, _ := util.GrepLinesFromFile(logfile, []string{"ERROR", "fatal"}, 5, false, true) + if len(errStrDetail) > 0 { logger.Log.Info(errStrPrefix) logger.Log.Error(errStrDetail) } else { - logger.Log.Warn("can not find more detail error message from ", logfile) + logger.Log.Warn("tail can not find more detail error message from ", logfile) } - return errors.WithMessagef(err, fmt.Sprintf("%s: %s\n%s", errStr, errStrPrefix, errStrDetail)) + return errors.WithMessagef(err, fmt.Sprintf("%s\n%s", errStrPrefix, errStrDetail)) } logger.Log.Info("xtrabackup recover success: ", outStr) return nil diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go index 9a4c05b019..b82735b693 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go @@ -43,25 +43,17 @@ func parseMysqldumpMetadata(sqlFilePath string) (*mydumperMetadata, error) { var bufScanner *bufio.Scanner if strings.HasSuffix(sqlFilePath, cst.ZstdSuffix) { - /* - var compressedBuf = make([]byte, 2048) - if _, err := sqlFile.Read(compressedBuf); err != nil && err != io.EOF { - return nil, errors.WithMessagef(err, "read first 10240 bytes from %s", sqlFilePath) - } - var textBuf = make([]byte, 2048) - dec, _ := zstd.NewReader(bytes.NewBuffer(compressedBuf), zstd.WithDecoderConcurrency(1)) - defer dec.Close() - if _, err := dec.Read(textBuf); err != nil { - logger.Log.Warnf("zstd decode first 10240 bytes failed from %s, err:%s", sqlFilePath, err.Error()) - } - bufScanner = bufio.NewScanner(bytes.NewBuffer(textBuf)) - */ - cmds := []string{"cut", "-b", "-2048", sqlFilePath, "|", CmdZstd, "-d", "-c"} + cmds := []string{"cut", "-b", "-4096", sqlFilePath, "|", CmdZstd, "-d", "-c"} outBuf, _, err := cmutil.ExecCommandReturnBytes(true, "", cmds[0], cmds[1:]...) + + if len(outBuf) < 100 { // 返回小于这个长度，肯定非法了，重试一遍 + cmds = []string{"cut", "-b", "-10240", sqlFilePath, "|", CmdZstd, "-d", "-c"} + outBuf, _, err = cmutil.ExecCommandReturnBytes(true, "", cmds[0], cmds[1:]...) + } if err != nil { logger.Log.Warnf("zstd decode first 2048 bytes failed from %s, err:%s", sqlFilePath, err.Error()) } - if len(outBuf) < 100 { // 返回小于这个长度，肯定非法了 + if len(outBuf) < 100 { // 返回小于这个长度，非法报错 return nil, errors.Errorf("failed to get binlog position from zst file %s", sqlFilePath) } bufScanner = bufio.NewScanner(bytes.NewBuffer(outBuf)) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go index 263f2f03e3..5c78783db1 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go @@ -11,6 +11,7 @@ package precheck import ( + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger" @@ -37,7 +38,7 @@ func CheckBackupType(cnf *config.BackupConfig, storageEngine string) error { } else { cnf.Public.BackupType = cst.BackupLogical } - if glibcVer, err := util.GetGlibcVersion(); err != nil { + if glibcVer, err := cmutil.GetGlibcVersion(); err != nil { logger.Log.Warn("failed to glibc version, err:", err) } else if glibcVer < "2.14" { // mydumper need glibc version >= 2.14 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_disk_space.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_disk_space.go index a945b2c2a0..57a42cf469 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_disk_space.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_disk_space.go @@ -53,9 +53,9 @@ func DeleteOldBackup(cnf *config.Public, expireDays int) error { for _, fi := range dir { fileMatchOld := fmt.Sprintf("%s_%s", hostName, cnf.MysqlHost) - filePrefix := fmt.Sprintf("%d_%d_%s", cnf.BkBizId, cnf.ClusterId, cnf.MysqlHost) + fileMatch := fmt.Sprintf("_%d_%s", cnf.ClusterId, cnf.MysqlHost) // cnf.BkBizId 转业务了历史的也要删除 if fi.ModTime().Compare(expireTime) <= 0 { - if strings.HasPrefix(fi.Name(), filePrefix) || strings.Contains(fi.Name(), fileMatchOld) { + if strings.Contains(fi.Name(), fileMatch) || strings.Contains(fi.Name(), fileMatchOld) { fileName := filepath.Join(cnf.BackupDir, fi.Name()) if fi.Size() > 4*1024*1024*1024 { logger.Log.Infof("remove old backup file %s limit %dMB/s ", fileName, cnf.IOLimitMBPerSec) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/spider/spider_service.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/spider/spider_service.go index 4b0c6c6e16..ab3b5bd027 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/spider/spider_service.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/spider/spider_service.go @@ -156,7 +156,7 @@ func (b GlobalBackupModel) queryBackupTasks(retries int, db *sqlx.DB) (backupTas if err != nil { return nil, err } - logger.Log.Infof("queryBackupTasks port=%d, sqlStr:%s, sqlArgs:%v", b.Port, sqlStr, sqlArgs) + logger.Log.Infof("queryBackupTasks for port=%d, sqlStr:%s, sqlArgs:%v", b.Port, sqlStr, sqlArgs) if err = db.Select(&backupTasks, sqlStr, sqlArgs...); err != nil { logger.Log.Warnf("fail to queryBackupTasks: %s, retries %d", err.Error(), retries) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go index 75e8808260..31b5d1ea6e 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go @@ -14,6 +14,7 @@ import ( "syscall" "github.com/pkg/errors" + "github.com/spf13/cast" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" @@ -184,21 +185,6 @@ func CheckDiskSpace(backupDir string, mysqlPort int, backupSize uint64) (sizeLef return sizeLeft, nil } -func GetGlibcVersion() (string, error) { - //ExeCommand("ldd --version |grep libc") - outStr, errStr, err := cmutil.ExecCommand(false, "", - "/usr/bin/ldd", "--version", "|", "grep", "libc") - if err != nil { - return "", errors.WithMessage(err, errStr) - } - verMatch := regexp.MustCompile(`ldd \(.*\) (\d+\.\d+)`) - ms := verMatch.FindStringSubmatch(outStr) - if len(ms) == 2 { - return ms[1], nil - } - return "", errors.New("ldd --version | grep libc fail to get glibc version") -} - // ExeCommand execute shell command func ExeCommand(cmdStr string) error { res, exeErr := exec.Command("/bin/bash", "-c", cmdStr).CombinedOutput() @@ -286,3 +272,36 @@ func FindBackupConfigFiles(dir string) ([]string, error) { } return cnfFilesNew, nil } + +// GrepLinesFromFile 从文件里面 grep 错误关键字 +// 如果不指定 keywords，则直接 tail / head 文件行 +func GrepLinesFromFile(logFilePath string, keywords []string, linesRet int, sensitive bool, tail bool) (string, error) { + var grepCommand []string + lineNum := "-" + cast.ToString(linesRet) + if len(keywords) > 0 { + grepExpr := "'" + strings.Join(keywords, "|") + "'" + if sensitive { + grepCommand = append(grepCommand, "grep", "-E") + } else { + grepCommand = append(grepCommand, "grep", "-Ei") + } + grepCommand = append(grepCommand, grepExpr, logFilePath) + if tail { + grepCommand = append(grepCommand, "|", "tail", lineNum) + } else { + grepCommand = append(grepCommand, "|", "head", lineNum) + } + } else { + if tail { + grepCommand = append(grepCommand, "tail", lineNum, logFilePath) + } else { + grepCommand = append(grepCommand, "head", lineNum, logFilePath) + } + } + errStrDetail, cmdStdErr, err := cmutil.ExecCommand(true, "", grepCommand[0], grepCommand[1:]...) + if strings.TrimSpace(errStrDetail) != "" { + return errStrDetail, nil + } else { + return "", errors.WithMessage(err, cmdStdErr) + } +} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/go.mod b/dbm-services/mysql/db-tools/mysql-monitor/go.mod index 70edb8ab12..7479b73d27 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/go.mod +++ b/dbm-services/mysql/db-tools/mysql-monitor/go.mod @@ -7,6 +7,7 @@ require ( github.com/go-playground/validator/v10 v10.15.4 github.com/go-sql-driver/mysql v1.6.0 github.com/jmoiron/sqlx v1.3.5 + github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b github.com/juju/ratelimit v1.0.2 github.com/olekukonko/tablewriter v0.0.5 github.com/pingcap/errors v0.11.4 diff --git a/dbm-services/mysql/db-tools/mysql-monitor/go.sum b/dbm-services/mysql/db-tools/mysql-monitor/go.sum index 4fc3e5d5c4..eebfdc097c 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/go.sum +++ b/dbm-services/mysql/db-tools/mysql-monitor/go.sum @@ -145,6 +145,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b h1:FQ7+9fxhyp82ks9vAuyPzG0/vVbWwMwLJ+P6yJI5FN8= +github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b/go.mod h1:HMcgvsgd0Fjj4XXDkbjdmlbI505rUPBs6WBMYg2pXks= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go index 580cf09081..75423dce7d 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go @@ -1,34 +1,38 @@ package ibdstatistic import ( - "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" - "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" - "encoding/json" - "fmt" "log/slog" "os" "path/filepath" "slices" "time" + "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/reportlog" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" + "github.com/pkg/errors" ) type tableSizeStruct struct { - BkCloudId int `json:"bk_cloud_id"` - BkBizId int `json:"bk_biz_id"` - ImmuteDomain string `json:"cluster_domain"` - DBModule int `json:"db_module"` - MachineType string `json:"machine_type"` - Ip string `json:"instance_host"` - Port int `json:"instance_port"` - Role string `json:"instance_role"` - ServiceInstanceId int64 `json:"bk_target_service_instance_id"` - OriginalDBName string `json:"original_database_name"` - DBName string `json:"database_name"` - DBSize int64 `json:"database_size"` - TableName string `json:"table_name"` - TableSize int64 `json:"table_size"` + // ReportTime time.RFC3339, format like 024-11-29T11:19:02+08:00 + ReportTime time.Time `json:"report_time"` + BkCloudId int `json:"bk_cloud_id"` + BkBizId int `json:"bk_biz_id"` + ImmuteDomain string `json:"cluster_domain"` + DBModule int `json:"db_module"` + MachineType string `json:"machine_type"` + Ip string `json:"instance_host"` + Port int `json:"instance_port"` + Role string `json:"instance_role"` + ServiceInstanceId int64 `json:"bk_target_service_instance_id"` + // OriginalDBName original DBName for spider remote(with shard_id suffix) + OriginalDBName string `json:"original_database_name"` + DBName string `json:"database_name"` + DBSize int64 `json:"database_size"` + TableName string `json:"table_name"` + TableSize int64 `json:"table_size"` } func reportLog(result map[string]map[string]int64) error { @@ -38,26 +42,11 @@ func reportLog(result map[string]map[string]int64) error { slog.Error("failed to create database size reports directory", slog.String("error", err.Error())) return errors.Wrap(err, "failed to create database size reports directory") } - - filePath := filepath.Join( - dbsizeReportBaseDir, - fmt.Sprintf(`report.log.%d`, time.Now().Weekday()), - ) - err = os.RemoveAll(filePath) - if err != nil { - slog.Error("failed to remove database size reports directory", slog.String("error", err.Error())) - return errors.Wrap(err, "failed to remove database size reports directory") - } - - f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + resultReport, err := reportlog.NewReporter(dbsizeReportBaseDir, "report.log", nil) if err != nil { - slog.Error("failed to open log file", "file", filePath) - return errors.Wrap(err, "failed to open file") + return err } - defer func() { - _ = f.Close() - }() - + reportTs := cmutil.TimeToSecondPrecision(time.Now()) for originalDBName, dbInfo := range result { // 根据 dbm 枚举约定, remote 是 tendbcluster 的存储机器类型 dbName := originalDBName @@ -98,18 +87,11 @@ func reportLog(result map[string]map[string]int64) error { for _, row := range tablesInfo { row.DBSize = dbSize - b, err := json.Marshal(row) - if err != nil { - slog.Error("ibd-statistic report", slog.String("error", err.Error())) - return errors.Wrap(err, "failed to marshal row") - } - - b = append(b, '\n') - _, err = f.Write(b) - if err != nil { - slog.Error("ibd-statistic report", slog.String("error", err.Error())) - return errors.Wrap(err, "failed to write row") + row.ReportTime = reportTs + if slices.Index(systemDBs, row.DBName) >= 0 { + continue } + resultReport.Println(row) } } return nil diff --git a/dbm-services/redis/db-tools/dbmon/pkg/redisbinlogbackup/job.go b/dbm-services/redis/db-tools/dbmon/pkg/redisbinlogbackup/job.go index fa40d3c2bd..c7de568a4e 100644 --- a/dbm-services/redis/db-tools/dbmon/pkg/redisbinlogbackup/job.go +++ b/dbm-services/redis/db-tools/dbmon/pkg/redisbinlogbackup/job.go @@ -274,18 +274,8 @@ func (job *Job) DeleteTooOldBinlogbackup(port int) { Days15Ago := time.Now().Local().AddDate(0, 0, -15) mylog.Logger.Debug(fmt.Sprintf("port:%d start DeleteTooOldBinlogbackup", port)) - // 15 天以前的,本地文件已删除的,记录直接删除 - job.Err = job.sqdb.Where("start_time<=? and local_file_removed=?", Days15Ago, 1). - Delete(&RedisBinlogHistorySchema{}).Error - if job.Err != nil { - job.Err = fmt.Errorf( - "DeleteTooOldBinlogbackup gorm delete fail,err:%v,start_time:(%s) local_file_removed:%d", - job.Err, Days15Ago, 1) - mylog.Logger.Error(job.Err.Error()) - return - } // OldFileLeftDay天以以前的,本地文件未删除的,remove本地文件,并记录下该行为 - job.Err = job.sqdb.Where("start_time<=? and local_file_removed=?", NDaysAgo, 0).Find(&toDoRows).Error + job.Err = job.sqdb.Where("end_time<=? and local_file_removed=?", NDaysAgo, 0).Find(&toDoRows).Error if job.Err != nil && job.Err != gorm.ErrRecordNotFound { job.Err = fmt.Errorf("DeleteTooOldBinlogbackup gorm find fail,err:%v", job.Err) mylog.Logger.Error(job.Err.Error()) @@ -293,6 +283,7 @@ func (job *Job) DeleteTooOldBinlogbackup(port int) { } job.Err = nil for _, row := range toDoRows { + mylog.Logger.Info(fmt.Sprintf("begin delete old file:%s,4 ModfyTime:%s", row.BackupFile, row.EndTime.GoString())) removeOK = true if util.FileExists(row.BackupFile) { err = os.Remove(row.BackupFile) @@ -312,4 +303,15 @@ func (job *Job) DeleteTooOldBinlogbackup(port int) { mylog.Logger.Error(job.Err.Error()) } } + + // 15 天以前的,本地文件已删除的,记录直接删除 + job.Err = job.sqdb.Where("end_time<=? and local_file_removed=?", Days15Ago, 1). + Delete(&RedisBinlogHistorySchema{}).Error + if job.Err != nil { + job.Err = fmt.Errorf( + "DeleteTooOldBinlogbackup gorm delete fail,err:%v,start_time:(%s) local_file_removed:%d", + job.Err, Days15Ago, 1) + mylog.Logger.Error(job.Err.Error()) + return + } } diff --git a/dbm-ui/backend/bk_dataview/dashboards/json/es.json b/dbm-ui/backend/bk_dataview/dashboards/json/es.json index 63c838d1c1..954f8e240c 100644 --- a/dbm-ui/backend/bk_dataview/dashboards/json/es.json +++ b/dbm-ui/backend/bk_dataview/dashboards/json/es.json @@ -1,41 +1,4 @@ { - "__inputs": [ - { - "name": "DS_蓝鲸监控_- 指标数据", - "label": "蓝鲸监控 - 指标数据", - "description": "", - "type": "datasource", - "pluginId": "bkmonitor-timeseries-datasource", - "pluginName": "BlueKing Monitor TimeSeries" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "datasource", - "id": "bkmonitor-timeseries-datasource", - "name": "BlueKing Monitor TimeSeries", - "version": "3.6.0" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "9.1.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], "annotations": { "list": [ { @@ -58,10 +21,10 @@ } ] }, - "editable": false, + "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": null, + "id": 23668, "links": [], "liveNow": false, "panels": [ @@ -73,15 +36,15 @@ "x": 0, "y": 0 }, - "id": null, + "id": 2, "panels": [], - "title": "集群信息", + "title": "Cluster Info", "type": "row" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -95,10 +58,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -111,7 +70,7 @@ "x": 0, "y": 1 }, - "id": null, + "id": 4, "options": { "colorMode": "value", "graphMode": "area", @@ -124,15 +83,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -190,13 +151,13 @@ "type": "range" } ], - "title": "master数量", + "title": "Master Nodes", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -222,7 +183,7 @@ "x": 3, "y": 1 }, - "id": null, + "id": 18, "options": { "colorMode": "value", "graphMode": "area", @@ -235,15 +196,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -300,13 +263,13 @@ "type": "range" } ], - "title": "热节点数量", + "title": "Hot Data Nodes", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -333,7 +296,7 @@ "x": 6, "y": 1 }, - "id": null, + "id": 9, "options": { "colorMode": "value", "graphMode": "area", @@ -346,15 +309,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -369,13 +334,13 @@ "type": "range" } ], - "title": "冷节点数量", + "title": "Cold Data Nodes", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -402,7 +367,7 @@ "x": 9, "y": 1 }, - "id": null, + "id": 10, "options": { "colorMode": "value", "graphMode": "area", @@ -415,15 +380,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -438,13 +405,13 @@ "type": "range" } ], - "title": "client数量", + "title": "Client Nodes", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -470,7 +437,7 @@ "x": 12, "y": 1 }, - "id": null, + "id": 11, "options": { "colorMode": "value", "graphMode": "area", @@ -483,15 +450,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -542,13 +511,13 @@ "type": "range" } ], - "title": "indices数量", + "title": "Number of Indices", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -582,7 +551,7 @@ "x": 15, "y": 1 }, - "id": null, + "id": 12, "options": { "colorMode": "value", "graphMode": "area", @@ -595,15 +564,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -653,13 +624,13 @@ "type": "range" } ], - "title": "shard数量", + "title": "Number of Shards", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -686,7 +657,7 @@ "x": 18, "y": 1 }, - "id": null, + "id": 13, "options": { "colorMode": "value", "graphMode": "area", @@ -699,15 +670,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -757,13 +730,13 @@ "type": "range" } ], - "title": "document数量", + "title": "Number of Documents", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -790,7 +763,7 @@ "x": 21, "y": 1 }, - "id": null, + "id": 16, "options": { "colorMode": "value", "graphMode": "area", @@ -803,15 +776,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -861,13 +836,13 @@ "type": "range" } ], - "title": "总数据量", + "title": "Data Stoarage Size", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -893,7 +868,7 @@ "x": 0, "y": 4 }, - "id": null, + "id": 8, "options": { "colorMode": "value", "graphMode": "area", @@ -906,15 +881,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -972,13 +949,13 @@ "type": "range" } ], - "title": "上报master数量", + "title": "Alive Master Nodes", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1004,7 +981,7 @@ "x": 3, "y": 4 }, - "id": null, + "id": 17, "options": { "colorMode": "value", "graphMode": "area", @@ -1017,15 +994,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -1083,13 +1062,13 @@ "type": "range" } ], - "title": "上报热节点数量", + "title": "Alive Hot Data Nodes", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1116,7 +1095,7 @@ "x": 6, "y": 4 }, - "id": null, + "id": 19, "options": { "colorMode": "value", "graphMode": "area", @@ -1129,15 +1108,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -1152,13 +1133,13 @@ "type": "range" } ], - "title": "上报冷节点数量", + "title": "Alive Cold Data Nodes", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1185,7 +1166,7 @@ "x": 9, "y": 4 }, - "id": null, + "id": 20, "options": { "colorMode": "value", "graphMode": "area", @@ -1198,15 +1179,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -1221,13 +1204,13 @@ "type": "range" } ], - "title": "上报client数量", + "title": "Alive Client Nodes", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1257,7 +1240,7 @@ "x": 12, "y": 4 }, - "id": null, + "id": 24, "options": { "colorMode": "value", "graphMode": "area", @@ -1270,15 +1253,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -1328,13 +1313,13 @@ "type": "range" } ], - "title": "unassigned shards", + "title": "Unassigned Shards", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1364,7 +1349,7 @@ "x": 15, "y": 4 }, - "id": null, + "id": 29, "options": { "colorMode": "value", "graphMode": "area", @@ -1377,15 +1362,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -1435,13 +1422,13 @@ "type": "range" } ], - "title": "pending tasks", + "title": "Pending Tasks", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1468,7 +1455,7 @@ "x": 18, "y": 4 }, - "id": null, + "id": 26, "options": { "colorMode": "value", "graphMode": "area", @@ -1481,15 +1468,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -1540,13 +1529,13 @@ "type": "range" } ], - "title": "active shard percent", + "title": "Active Shards percentage", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1605,7 +1594,7 @@ "x": 21, "y": 4 }, - "id": null, + "id": 25, "options": { "colorMode": "background", "graphMode": "none", @@ -1618,15 +1607,17 @@ "fields": "", "values": false }, - "textMode": "value" + "showPercentChange": false, + "textMode": "value", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -1778,13 +1769,13 @@ "type": "range" } ], - "title": "集群状态", + "title": "Cluster Status", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1811,7 +1802,7 @@ "x": 3, "y": 7 }, - "id": null, + "id": 30, "options": { "colorMode": "value", "graphMode": "area", @@ -1824,15 +1815,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -1890,13 +1883,13 @@ "type": "range" } ], - "title": "热节点容量", + "title": "Hot Nodes Capacity", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -1924,7 +1917,7 @@ "x": 6, "y": 7 }, - "id": null, + "id": 21, "options": { "colorMode": "value", "graphMode": "area", @@ -1937,15 +1930,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -2018,13 +2013,13 @@ "type": "range" } ], - "title": "冷节点容量", + "title": "Cold Nodes Capacity", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -2054,7 +2049,7 @@ "x": 9, "y": 7 }, - "id": null, + "id": 22, "options": { "colorMode": "value", "graphMode": "area", @@ -2067,15 +2062,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -2125,13 +2122,13 @@ "type": "range" } ], - "title": "relocating shards", + "title": "Relocating Shards", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -2161,7 +2158,7 @@ "x": 12, "y": 7 }, - "id": null, + "id": 14, "options": { "colorMode": "value", "graphMode": "area", @@ -2174,15 +2171,17 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.0", + "pluginVersion": "10.4.10", "targets": [ { "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -2232,13 +2231,13 @@ "type": "range" } ], - "title": "initializing shards", + "title": "Initializing Shards", "type": "stat" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "description": "0: red\n1: green\n2: yellow", "fieldConfig": { @@ -2247,6 +2246,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2260,6 +2260,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2316,7 +2317,7 @@ "x": 0, "y": 10 }, - "id": null, + "id": 28, "options": { "legend": { "calcs": [ @@ -2336,12 +2337,13 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, + "enableDownSampling": true, "expressionList": [ { "active": true, - "alias": "状态", + "alias": "status", "expression": "a*1+b*2+c*3", "functions": [] } @@ -2353,7 +2355,7 @@ "promqlAlias": "状态", "query_configs": [ { - "alias": "状态", + "alias": "status", "data_label": "exporter_dbm_elasticsearch_exporter", "data_source_label": "bk_monitor", "data_type_label": "time_series", @@ -2368,7 +2370,7 @@ "refId": "a", "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", "result_table_label": "component", - "time_field": "", + "time_field": "time", "where": [ { "key": "app", @@ -2396,7 +2398,7 @@ ] }, { - "alias": "", + "alias": "status", "data_label": "exporter_dbm_elasticsearch_exporter", "data_source_label": "bk_monitor", "data_type_label": "time_series", @@ -2439,7 +2441,7 @@ ] }, { - "alias": "", + "alias": "status", "data_label": "exporter_dbm_elasticsearch_exporter", "data_source_label": "bk_monitor", "data_type_label": "time_series", @@ -2488,13 +2490,13 @@ "type": "range" } ], - "title": "集群状态", + "title": "Cluster Status", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -2502,6 +2504,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2515,6 +2518,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2546,32 +2550,7 @@ ] } }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "relocating" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -2579,7 +2558,7 @@ "x": 12, "y": 10 }, - "id": null, + "id": 32, "options": { "legend": { "calcs": [ @@ -2600,7 +2579,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -2654,7 +2633,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -2708,7 +2687,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -2759,13 +2738,13 @@ "type": "range" } ], - "title": "initializing/relocating/unassigned shards", + "title": "Number of Initializing/Relocating/Unassigned Shards", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -2773,6 +2752,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2786,6 +2766,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2825,7 +2806,7 @@ "x": 0, "y": 18 }, - "id": null, + "id": 31, "options": { "legend": { "calcs": [ @@ -2846,7 +2827,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -2896,13 +2877,13 @@ "type": "range" } ], - "title": "shards数量", + "title": "Number of Shards", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -2910,6 +2891,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2923,6 +2905,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2962,7 +2945,7 @@ "x": 12, "y": 18 }, - "id": null, + "id": 33, "options": { "legend": { "calcs": [ @@ -2983,7 +2966,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -3033,13 +3016,13 @@ "type": "range" } ], - "title": "indices数量", + "title": "Number of Indices", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -3047,6 +3030,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3060,6 +3044,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3099,7 +3084,7 @@ "x": 0, "y": 26 }, - "id": null, + "id": 35, "options": { "legend": { "calcs": [ @@ -3120,7 +3105,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -3227,7 +3212,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -3288,13 +3273,13 @@ "type": "range" } ], - "title": "集群每秒请求（doc数，含副本）", + "title": "Request Rating（include replicas）", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "description": "", "fieldConfig": { @@ -3303,6 +3288,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3316,6 +3302,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3356,7 +3343,7 @@ "x": 12, "y": 26 }, - "id": null, + "id": 36, "options": { "legend": { "calcs": [ @@ -3377,7 +3364,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -3507,7 +3494,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -3640,7 +3627,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -3676,13 +3663,13 @@ "type": "range" } ], - "title": "集群请求延迟", + "title": "Request Delay", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -3690,6 +3677,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3703,6 +3691,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3742,7 +3731,7 @@ "x": 0, "y": 34 }, - "id": null, + "id": 34, "options": { "legend": { "calcs": [ @@ -3763,7 +3752,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -3813,13 +3802,13 @@ "type": "range" } ], - "title": "documents数量", + "title": "Number of Documents", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -3827,6 +3816,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3840,6 +3830,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3878,7 +3869,7 @@ "x": 12, "y": 34 }, - "id": null, + "id": 38, "options": { "legend": { "calcs": [ @@ -3899,7 +3890,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -3960,7 +3951,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -4022,7 +4013,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -4084,7 +4075,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -4143,13 +4134,13 @@ "type": "range" } ], - "title": "上报节点数量", + "title": "Number of Alive Nodes", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -4157,6 +4148,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4170,6 +4162,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4208,7 +4201,7 @@ "x": 0, "y": 42 }, - "id": null, + "id": 39, "options": { "legend": { "calcs": [ @@ -4229,7 +4222,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -4279,13 +4272,13 @@ "type": "range" } ], - "title": "pending tasks", + "title": "Number of Pending Tasks", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -4293,6 +4286,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4306,6 +4300,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -4345,7 +4340,7 @@ "x": 12, "y": 42 }, - "id": null, + "id": 37, "options": { "legend": { "calcs": [ @@ -4366,14 +4361,15 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, + "enableDownSampling": false, "expressionList": [], "format": "time_series", "host": [], "mode": "code", "module": [], - "promqlAlias": "数据量", + "promqlAlias": "Data Storage Size", "query_configs": [], "refId": "A", "source": "sum(sum_over_time(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_indices_store_size_bytes{app=\"$app\",cluster_domain=\"$cluster_domain\"}[1m]))", @@ -4381,3082 +4377,3038 @@ "type": "range" } ], - "title": "集群总数据量（含冷热、副本）", + "title": "Data Stoarge Size", "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, - "id": null, - "panels": [ - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "id": 41, + "panels": [], + "title": "Indices Info", + "type": "row" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [ - { - "active": true, - "alias": "", - "expression": "", - "functions": [] - } - ], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_index", - "query_configs": [ - { - "alias": "$tag_index", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [ - { - "id": "rate", - "params": [ - { - "id": "window", - "value": "2m" - } - ] - }, - { - "id": "topk", - "params": [ - { - "id": "k", - "value": "20" - } - ] - } - ], - "group_by": [ - "index" - ], - "interval": 60, - "interval_unit": "s", - "method": "sum_without_time", - "metric_field": "elasticsearch_index_stats_indexing_index_total", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - } - ] - } - ], - "refId": "A", - "source": "topk(20, sum by(index) (rate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_index_stats_indexing_index_total{app=\"$app\",cluster_domain=\"$cluster_domain\"}[2m])))", - "step": "", - "type": "range" - } - ], - "title": "每秒indexing请求（top20）", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 51 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_index", - "query_configs": [ - { - "alias": "$tag_index", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "filter_dict": {}, - "functions": [ - { - "id": "topk", - "params": [ - { - "id": "k", - "value": "20" - } - ] - }, - { - "id": "rate", - "params": [ - { - "id": "window", - "value": "2m" - } - ] - } - ], - "group_by": [ - "index" - ], - "interval": 60, - "interval_unit": "s", - "method": "SUM", - "metric_field": "elasticsearch_index_stats_search_query_total", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - } - ] - } - ], - "refId": "A", - "source": "topk(20, sum by(index) (rate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_index_stats_search_query_total{app=\"$app\",cluster_domain=\"$cluster_domain\"}[2m])))", - "step": "", - "type": "range" - } + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "max", + "last" ], - "title": "每秒search请求（top20）", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { + "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 59 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "expressionList": [ + { + "active": true, + "alias": "", + "expression": "", + "functions": [] } - }, - "targets": [ + ], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_index", + "query_configs": [ { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "ui", - "module": [], - "promqlAlias": "", - "query_configs": [ - { - "alias": "$tag_index", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [ + "alias": "$tag_index", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [ + { + "id": "rate", + "params": [ { - "id": "topk", - "params": [ - { - "id": "k", - "value": "20" - } - ] + "id": "window", + "value": "2m" } - ], - "group_by": [ - "index" - ], - "interval": "auto", - "interval_unit": "s", - "method": "max_without_time", - "metric_field": "elasticsearch_indices_store_size_bytes_total", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, + ] + }, + { + "id": "topk", + "params": [ { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] + "id": "k", + "value": "20" } ] } ], - "refId": "A", - "source": "", - "step": "", - "type": "range" - } - ], - "title": "indices数据量（top20）", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 59 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "last" + "group_by": [ + "index" ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_index-$tag_shard", - "query_configs": [ - { - "alias": "$tag_index-$tag__shard", - "data_label": "exporter_dbm_elasticsearch_exporter", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [ - { - "id": "topk", - "params": [ - { - "id": "k", - "value": "20" - } - ] - } - ], - "group_by": [ - "index", - "_shard" - ], - "interval": 60, - "interval_unit": "s", - "method": "avg_without_time", - "metric_field": "elasticsearch_indices_shards_docs", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - } + "interval": 60, + "interval_unit": "s", + "method": "sum_without_time", + "metric_field": "elasticsearch_index_stats_indexing_index_total", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" ] } - ], - "refId": "A", - "source": "topk(20, avg by (index, shard) (bkmonitor:exporter_dbm_elasticsearch_exporter:elasticsearch_indices_shards_docs{app=\"$app\",cluster_domain=\"$cluster_domain\"}))", - "step": "", - "type": "range" + ] } ], - "title": "shard docs（top20）", - "type": "timeseries" + "refId": "A", + "source": "topk(20, sum by(index) (rate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_index_stats_indexing_index_total{app=\"$app\",cluster_domain=\"$cluster_domain\"}[2m])))", + "step": "", + "type": "range" } ], - "title": "indices信息", - "type": "row" + "title": "Indexing Rating (top 20)", + "type": "timeseries" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 51 + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" }, - "id": null, - "panels": [ - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 52 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "expressionList": [ - { - "active": true, - "alias": "", - "expression": "a / b", - "functions": [] - } - ], - "format": "time_series", - "hide": false, - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_name", - "query_configs": [ - { - "alias": "$tag_name", - "data_label": "exporter_dbm_elasticsearch_exporter", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": false, - "filter_dict": {}, - "functions": [], - "group_by": [ - "name" - ], - "interval": 60, - "interval_unit": "s", - "method": "max_without_time", - "metric_field": "elasticsearch_jvm_memory_used_bytes", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "es_master_node", - "method": "eq", - "value": [ - "true" - ] - } - ] - }, - { - "alias": "$tag_name", - "data_label": "exporter_dbm_elasticsearch_exporter", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "name" - ], - "interval": 60, - "interval_unit": "s", - "method": "max_without_time", - "metric_field": "elasticsearch_jvm_memory_max_bytes", - "refId": "b", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "es_master_node", - "method": "eq", - "value": [ - "true" - ] - } - ] - } - ], - "refId": "A", - "source": "max by (name) (bkmonitor:exporter_dbm_elasticsearch_exporter:elasticsearch_jvm_memory_used_bytes{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\"}) / max by (name) (bkmonitor:exporter_dbm_elasticsearch_exporter:elasticsearch_jvm_memory_max_bytes{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\"})", - "step": "", - "type": "range" - } + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "max", + "last" ], - "title": "jvm heap使用率", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { + "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 52 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "uid": "P3XSxkgSz" }, - "targets": [ + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_index", + "query_configs": [ { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [ + "alias": "$tag_index", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "filter_dict": {}, + "functions": [ { - "active": true, - "alias": "", - "expression": "a*60", - "functions": [] - } - ], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "young-$tag_name", - "query_configs": [ - { - "alias": "young-$tag___name", - "data_label": "exporter_dbm_elasticsearch_exporter", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [ - { - "id": "irate", - "params": [ - { - "id": "window", - "value": "2m" - } - ] - } - ], - "group_by": [ - "__name" - ], - "interval": 60, - "interval_unit": "s", - "method": "AVG", - "metric_field": "elasticsearch_jvm_gc_collection_seconds_count", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "es_master_node", - "method": "eq", - "value": [ - "true" - ] - }, + "id": "topk", + "params": [ { - "condition": "and", - "key": "gc", - "method": "eq", - "value": [ - "young" - ] + "id": "k", + "value": "20" } ] - } - ], - "refId": "A", - "source": "avg by(name) (irate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_gc_collection_seconds_count{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\",gc=\"young\"}[2m])) * 60", - "step": "", - "type": "range" - }, - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [ + }, { - "active": true, - "alias": "old-$tag___name", - "expression": "a * 60", - "functions": [] - } - ], - "format": "time_series", - "hide": false, - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "old-$tag_name", - "query_configs": [ - { - "alias": "old-$tag___name", - "data_label": "exporter_dbm_elasticsearch_exporter", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "filter_dict": {}, - "functions": [ - { - "id": "irate", - "params": [ - { - "id": "window", - "value": "2m" - } - ] - } - ], - "group_by": [ - "__name" - ], - "interval": 60, - "interval_unit": "s", - "method": "AVG", - "metric_field": "elasticsearch_jvm_gc_collection_seconds_count", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "es_master_node", - "method": "eq", - "value": [ - "true" - ] - }, + "id": "rate", + "params": [ { - "condition": "and", - "key": "gc", - "method": "eq", - "value": [ - "old" - ] + "id": "window", + "value": "2m" } ] } ], - "refId": "B", - "source": "avg by(name) (irate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_gc_collection_seconds_count{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\",gc=\"old\"}[2m])) * 60", - "step": "", - "type": "range" - } - ], - "title": "jvm gc次数（1m）", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 60 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" + "group_by": [ + "index" ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [ + "interval": 60, + "interval_unit": "s", + "method": "SUM", + "metric_field": "elasticsearch_index_stats_search_query_total", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "", + "where": [ { - "active": true, - "alias": "", - "expression": "a + b", - "functions": [] - } - ], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_name", - "query_configs": [ - { - "alias": "$tag___name", - "data_label": "exporter_dbm_elasticsearch_exporter", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [ - { - "id": "irate", - "params": [ - { - "id": "window", - "value": "3m" - } - ] - } - ], - "group_by": [ - "__name" - ], - "interval": 90, - "interval_unit": "s", - "method": "AVG", - "metric_field": "elasticsearch_jvm_gc_collection_seconds_sum", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "time", - "where": [ - { - "key": "__name", - "method": "reg", - "value": [ - "master-.*|Master-.*" - ] - }, - { - "condition": "and", - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "gc", - "method": "eq", - "value": [ - "old" - ] - } + "key": "app", + "method": "eq", + "value": [ + "$app" ] }, { - "alias": "$tag___name", - "data_label": "exporter_dbm_elasticsearch_exporter", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "filter_dict": {}, - "functions": [ - { - "id": "irate", - "params": [ - { - "id": "window", - "value": "3m" - } - ] - } - ], - "group_by": [ - "__name" - ], - "interval": 90, - "interval_unit": "s", - "method": "AVG", - "metric_field": "elasticsearch_jvm_gc_collection_seconds_sum", - "refId": "b", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "time", - "where": [ - { - "key": "__name", - "method": "reg", - "value": [ - "master-.*|Master-.*" - ] - }, - { - "condition": "and", - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "gc", - "method": "eq", - "value": [ - "young" - ] - } + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" ] } - ], - "refId": "A", - "source": "avg by (name) (irate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_gc_collection_seconds_sum{name=~\"master-.*|Master-.*\",app=\"$app\",cluster_domain=\"$cluster_domain\",gc=\"old\"}[3m])) + avg by (name) (irate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_gc_collection_seconds_sum{name=~\"master-.*|Master-.*\",app=\"$app\",cluster_domain=\"$cluster_domain\",gc=\"young\"}[3m]))", - "step": "", - "type": "range" + ] } ], - "title": "jvm gc时间占比", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "refId": "A", + "source": "topk(20, sum by(index) (rate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_index_stats_search_query_total{app=\"$app\",cluster_domain=\"$cluster_domain\"}[2m])))", + "step": "", + "type": "range" + } + ], + "title": "Searching Rating（top 20）", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "dtdurations" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 60 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_name", - "query_configs": [ - { - "alias": "$tag___name", - "data_label": "exporter_dbm_elasticsearch_exporter", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "__name" - ], - "interval": "auto", - "interval_unit": "s", - "method": "max_without_time", - "metric_field": "elasticsearch_jvm_uptime_seconds", - "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "es_master_node", - "method": "eq", - "value": [ - "true" - ] - } - ] - } - ], - "refId": "A", - "source": "max by(name) (bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_uptime_seconds{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\"})", - "step": "", - "type": "range" - } - ], - "title": "节点uptime", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 68 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + { + "color": "red", + "value": 80 + } + ] }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip", - "query_configs": [ - { - "alias": "", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "bk_target_ip" - ], - "interval": 60, - "interval_unit": "s", - "method": "avg_without_time", - "metric_field": "usage", - "refId": "a", - "result_table_id": "dbm_system.cpu_summary", - "result_table_label": "os", - "time_field": "", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] - } - ] - } - ], - "refId": "A", - "source": "avg by(bk_target_ip) (bkmonitor:dbm_system:cpu_summary:usage{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})", - "step": "", - "type": "range" - } + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "max", + "last" ], - "title": "CPU使用率", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { + "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 68 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "uid": "P3XSxkgSz" }, - "targets": [ + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "ui", + "module": [], + "promqlAlias": "", + "query_configs": [ { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip", - "query_configs": [ - { - "alias": "", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "bk_target_ip" - ], - "interval": "auto", - "interval_unit": "s", - "method": "avg_without_time", - "metric_field": "load1", - "refId": "a", - "result_table_id": "dbm_system.load", - "result_table_label": "os", - "time_field": "", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, + "alias": "$tag_index", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [ + { + "id": "topk", + "params": [ { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] + "id": "k", + "value": "20" } ] } ], - "refId": "A", - "source": "avg by(bk_target_ip) (bkmonitor:dbm_system:load:load1{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})", - "step": "", - "type": "range" - } - ], - "title": "系统负载(1m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 76 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" + "group_by": [ + "index" ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip", - "query_configs": [ - { - "alias": "", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "bk_target_ip" - ], - "interval": "auto", - "interval_unit": "s", - "method": "avg_without_time", - "metric_field": "speed_recv", - "refId": "a", - "result_table_id": "dbm_system.net", - "result_table_label": "os", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluser_domain" - ] - }, - { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] - } + "interval": "auto", + "interval_unit": "s", + "method": "max_without_time", + "metric_field": "elasticsearch_indices_store_size_bytes_total", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" ] } - ], - "refId": "A", - "source": "avg by(bk_target_ip) (bkmonitor:dbm_system:net:speed_recv{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})*8", - "step": "", - "type": "range" + ] } ], - "title": "入流量(单位：bits/s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "refId": "A", + "source": "", + "step": "", + "type": "range" + } + ], + "title": "Stoarage of Indices（top20）", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 76 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip", - "query_configs": [ - { - "alias": "", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "bk_target_ip" - ], - "interval": "auto", - "interval_unit": "s", - "method": "avg_without_time", - "metric_field": "speed_sent", - "refId": "a", - "result_table_id": "dbm_system.net", - "result_table_label": "os", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] - } - ] - } - ], - "refId": "A", - "source": "avg by(bk_target_ip) (bkmonitor:dbm_system:net:speed_sent{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})*8", - "step": "", - "type": "range" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 45, + "options": { + "legend": { + "calcs": [ + "max", + "last" ], - "title": "出流量(单位：bits/s)", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last", + "sortDesc": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { + "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 84 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "uid": "P3XSxkgSz" }, - "targets": [ + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_index-$tag_shard", + "query_configs": [ { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip", - "query_configs": [ - { - "alias": "$tag_bk_target_ip", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "bk_target_ip" - ], - "interval": "auto", - "interval_unit": "s", - "method": "max_without_time", - "metric_field": "cur_tcp_estab", - "refId": "a", - "result_table_id": "dbm_system.netstat", - "result_table_label": "os", - "time_field": "", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, + "alias": "$tag_index-$tag__shard", + "data_label": "exporter_dbm_elasticsearch_exporter", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [ + { + "id": "topk", + "params": [ { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] + "id": "k", + "value": "20" } ] } ], - "refId": "A", - "source": "max by(bk_target_ip) (bkmonitor:dbm_system:netstat:cur_tcp_estab{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})", - "step": "", - "type": "range" + "group_by": [ + "index", + "_shard" + ], + "interval": 60, + "interval_unit": "s", + "method": "avg_without_time", + "metric_field": "elasticsearch_indices_shards_docs", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + } + ] + } + ], + "refId": "A", + "source": "topk(20, avg by (index, shard) (bkmonitor:exporter_dbm_elasticsearch_exporter:elasticsearch_indices_shards_docs{app=\"$app\",cluster_domain=\"$cluster_domain\"}))", + "step": "", + "type": "range" + } + ], + "title": "Number of Docs in Shard（top20）", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 48, + "panels": [], + "title": "Master Nodes Info", + "type": "row" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 50, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [ + { + "active": true, + "alias": "", + "expression": "a / b", + "functions": [] + } + ], + "format": "time_series", + "hide": false, + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_name", + "query_configs": [ + { + "alias": "$tag_name", + "data_label": "exporter_dbm_elasticsearch_exporter", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": false, + "filter_dict": {}, + "functions": [], + "group_by": [ + "name" + ], + "interval": 60, + "interval_unit": "s", + "method": "max_without_time", + "metric_field": "elasticsearch_jvm_memory_used_bytes", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "es_master_node", + "method": "eq", + "value": [ + "true" + ] + } + ] + }, + { + "alias": "$tag_name", + "data_label": "exporter_dbm_elasticsearch_exporter", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "name" + ], + "interval": 60, + "interval_unit": "s", + "method": "max_without_time", + "metric_field": "elasticsearch_jvm_memory_max_bytes", + "refId": "b", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "es_master_node", + "method": "eq", + "value": [ + "true" + ] + } + ] + } + ], + "refId": "A", + "source": "max by (name) (bkmonitor:exporter_dbm_elasticsearch_exporter:elasticsearch_jvm_memory_used_bytes{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\"}) / max by (name) (bkmonitor:exporter_dbm_elasticsearch_exporter:elasticsearch_jvm_memory_max_bytes{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\"})", + "step": "", + "type": "range" + } + ], + "title": "JVM Heap Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 54, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [ + { + "active": true, + "alias": "", + "expression": "a*60", + "functions": [] + } + ], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "young-$tag_name", + "query_configs": [ + { + "alias": "young-$tag___name", + "data_label": "exporter_dbm_elasticsearch_exporter", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [ + { + "id": "irate", + "params": [ + { + "id": "window", + "value": "2m" + } + ] + } + ], + "group_by": [ + "__name" + ], + "interval": 60, + "interval_unit": "s", + "method": "AVG", + "metric_field": "elasticsearch_jvm_gc_collection_seconds_count", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "es_master_node", + "method": "eq", + "value": [ + "true" + ] + }, + { + "condition": "and", + "key": "gc", + "method": "eq", + "value": [ + "young" + ] + } + ] + } + ], + "refId": "A", + "source": "avg by(name) (irate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_gc_collection_seconds_count{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\",gc=\"young\"}[2m])) * 60", + "step": "", + "type": "range" + }, + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [ + { + "active": true, + "alias": "old-$tag___name", + "expression": "a * 60", + "functions": [] + } + ], + "format": "time_series", + "hide": false, + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "old-$tag_name", + "query_configs": [ + { + "alias": "old-$tag___name", + "data_label": "exporter_dbm_elasticsearch_exporter", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "filter_dict": {}, + "functions": [ + { + "id": "irate", + "params": [ + { + "id": "window", + "value": "2m" + } + ] + } + ], + "group_by": [ + "__name" + ], + "interval": 60, + "interval_unit": "s", + "method": "AVG", + "metric_field": "elasticsearch_jvm_gc_collection_seconds_count", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "es_master_node", + "method": "eq", + "value": [ + "true" + ] + }, + { + "condition": "and", + "key": "gc", + "method": "eq", + "value": [ + "old" + ] + } + ] + } + ], + "refId": "B", + "source": "avg by(name) (irate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_gc_collection_seconds_count{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\",gc=\"old\"}[2m])) * 60", + "step": "", + "type": "range" + } + ], + "title": "JVM GC Times", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 53, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [ + { + "active": true, + "alias": "", + "expression": "a + b", + "functions": [] + } + ], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_name", + "query_configs": [ + { + "alias": "$tag___name", + "data_label": "exporter_dbm_elasticsearch_exporter", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [ + { + "id": "irate", + "params": [ + { + "id": "window", + "value": "3m" + } + ] + } + ], + "group_by": [ + "__name" + ], + "interval": 90, + "interval_unit": "s", + "method": "AVG", + "metric_field": "elasticsearch_jvm_gc_collection_seconds_sum", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", + "where": [ + { + "key": "__name", + "method": "reg", + "value": [ + "master-.*|Master-.*" + ] + }, + { + "condition": "and", + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "gc", + "method": "eq", + "value": [ + "old" + ] + } + ] + }, + { + "alias": "$tag___name", + "data_label": "exporter_dbm_elasticsearch_exporter", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "filter_dict": {}, + "functions": [ + { + "id": "irate", + "params": [ + { + "id": "window", + "value": "3m" + } + ] + } + ], + "group_by": [ + "__name" + ], + "interval": 90, + "interval_unit": "s", + "method": "AVG", + "metric_field": "elasticsearch_jvm_gc_collection_seconds_sum", + "refId": "b", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", + "where": [ + { + "key": "__name", + "method": "reg", + "value": [ + "master-.*|Master-.*" + ] + }, + { + "condition": "and", + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "gc", + "method": "eq", + "value": [ + "young" + ] + } + ] + } + ], + "refId": "A", + "source": "avg by (name) (irate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_gc_collection_seconds_sum{name=~\"master-.*|Master-.*\",app=\"$app\",cluster_domain=\"$cluster_domain\",gc=\"old\"}[3m])) + avg by (name) (irate(bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_gc_collection_seconds_sum{name=~\"master-.*|Master-.*\",app=\"$app\",cluster_domain=\"$cluster_domain\",gc=\"young\"}[3m]))", + "step": "", + "type": "range" + } + ], + "title": "JVM GC Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 51, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_name", + "query_configs": [ + { + "alias": "$tag___name", + "data_label": "exporter_dbm_elasticsearch_exporter", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "__name" + ], + "interval": "auto", + "interval_unit": "s", + "method": "max_without_time", + "metric_field": "elasticsearch_jvm_uptime_seconds", + "refId": "a", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "es_master_node", + "method": "eq", + "value": [ + "true" + ] + } + ] + } + ], + "refId": "A", + "source": "max by(name) (bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_uptime_seconds{app=\"$app\",cluster_domain=\"$cluster_domain\",es_master_node=\"true\"})", + "step": "", + "type": "range" + } + ], + "title": "Node Uptime", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip", + "query_configs": [ + { + "alias": "", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "bk_target_ip" + ], + "interval": 60, + "interval_unit": "s", + "method": "avg_without_time", + "metric_field": "usage", + "refId": "a", + "result_table_id": "dbm_system.cpu_summary", + "result_table_label": "os", + "time_field": "", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" + ] + } + ] + } + ], + "refId": "A", + "source": "avg by(bk_target_ip) (bkmonitor:dbm_system:cpu_summary:usage{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})", + "step": "", + "type": "range" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 84 + }, + "id": 55, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip", + "query_configs": [ + { + "alias": "", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "bk_target_ip" + ], + "interval": "auto", + "interval_unit": "s", + "method": "avg_without_time", + "metric_field": "load1", + "refId": "a", + "result_table_id": "dbm_system.load", + "result_table_label": "os", + "time_field": "", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" + ] + } + ] + } + ], + "refId": "A", + "source": "avg by(bk_target_ip) (bkmonitor:dbm_system:load:load1{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})", + "step": "", + "type": "range" + } + ], + "title": "Load(1m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 92 + }, + "id": 57, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip", + "query_configs": [ + { + "alias": "", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "bk_target_ip" + ], + "interval": "auto", + "interval_unit": "s", + "method": "avg_without_time", + "metric_field": "speed_recv", + "refId": "a", + "result_table_id": "dbm_system.net", + "result_table_label": "os", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluser_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" + ] + } + ] + } + ], + "refId": "A", + "source": "avg by(bk_target_ip) (bkmonitor:dbm_system:net:speed_recv{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})*8", + "step": "", + "type": "range" + } + ], + "title": "Received Bits in Second (Unit：bits/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 92 + }, + "id": 59, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip", + "query_configs": [ + { + "alias": "", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "bk_target_ip" + ], + "interval": "auto", + "interval_unit": "s", + "method": "avg_without_time", + "metric_field": "speed_sent", + "refId": "a", + "result_table_id": "dbm_system.net", + "result_table_label": "os", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" + ] + } + ] + } + ], + "refId": "A", + "source": "avg by(bk_target_ip) (bkmonitor:dbm_system:net:speed_sent{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})*8", + "step": "", + "type": "range" + } + ], + "title": "Sent Bits in Second ( Unit: bits/s )", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 100 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip", + "query_configs": [ + { + "alias": "$tag_bk_target_ip", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "bk_target_ip" + ], + "interval": "auto", + "interval_unit": "s", + "method": "max_without_time", + "metric_field": "cur_tcp_estab", + "refId": "a", + "result_table_id": "dbm_system.netstat", + "result_table_label": "os", + "time_field": "", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" + ] + } + ] + } + ], + "refId": "A", + "source": "max by(bk_target_ip) (bkmonitor:dbm_system:netstat:cur_tcp_estab{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"})", + "step": "", + "type": "range" + } + ], + "title": "Established TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 100 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip", + "query_configs": [ + { + "alias": "$tag_bk_target_ip", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [ + { + "id": "irate", + "params": [ + { + "id": "window", + "value": "2m" + } + ] + } + ], + "group_by": [ + "bk_target_ip" + ], + "interval": "auto", + "interval_unit": "s", + "method": "avg_without_time", + "metric_field": "cur_tcp_estab", + "refId": "a", + "result_table_id": "dbm_system.netstat", + "result_table_label": "os", + "time_field": "", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" + ] + } + ] + } + ], + "refId": "A", + "source": "avg by(bk_target_ip) (irate(bkmonitor:dbm_system:netstat:cur_tcp_estab{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"}[2m]))", + "step": "", + "type": "range" + } + ], + "title": "New TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 108 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" ], - "title": "连接数", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { + "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip $tag_device_name", + "query_configs": [ + { + "alias": "$tag_bk_target_ip $tag_device_name", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "bk_target_ip", + "device_name" + ], + "interval": "auto", + "interval_unit": "s", + "method": "AVG", + "metric_field": "wkb_s", + "refId": "a", + "result_table_id": "dbm_system.io", + "result_table_label": "os", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" + ] } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 84 + ] + } + ], + "refId": "A", + "source": "avg by (device_name, bk_target_ip) (avg_over_time(bkmonitor:dbm_system:io:util{app=\"$app\",cluster_domain=\"$cluster_domain\",device_name!~\"nbd*|ram*|loop*\",instance_role=\"es_master\"}[1m]))", + "step": "", + "type": "range" + } + ], + "title": "Disk IO Util", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip", - "query_configs": [ - { - "alias": "$tag_bk_target_ip", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [ - { - "id": "irate", - "params": [ - { - "id": "window", - "value": "2m" - } - ] - } - ], - "group_by": [ - "bk_target_ip" - ], - "interval": "auto", - "interval_unit": "s", - "method": "avg_without_time", - "metric_field": "cur_tcp_estab", - "refId": "a", - "result_table_id": "dbm_system.netstat", - "result_table_label": "os", - "time_field": "", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 108 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip$tag_mount_point", + "query_configs": [ + { + "alias": "$tag_bk_target_ip$tag_mount_point", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "bk_target_ip", + "mount_point" + ], + "interval": 60, + "interval_unit": "s", + "method": "AVG", + "metric_field": "in_use", + "refId": "a", + "result_table_id": "dbm_system.disk", + "result_table_label": "os", + "time_field": "", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" + ] + }, + { + "condition": "and", + "key": "mount_point", + "method": "neq", + "value": [ + "/" + ] + }, + { + "condition": "and", + "key": "mount_point", + "method": "neq", + "value": [ + "/usr/local" ] } - ], - "refId": "A", - "source": "avg by(bk_target_ip) (irate(bkmonitor:dbm_system:netstat:cur_tcp_estab{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"}[2m]))", - "step": "", - "type": "range" + ] } ], - "title": "新建连接数", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "refId": "A", + "source": "avg by (bk_target_ip, mount_point) (avg_over_time(bkmonitor:dbm_system:disk:in_use{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\",mount_point!=\"/\",mount_point!=\"/usr/local\"}[1m]))", + "step": "", + "type": "range" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 92 - }, - "id": null, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "title": "打开fd数", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 92 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip$tag_mount_point", - "query_configs": [ - { - "alias": "$tag_bk_target_ip$tag_mount_point", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "bk_target_ip", - "mount_point" - ], - "interval": 60, - "interval_unit": "s", - "method": "AVG", - "metric_field": "in_use", - "refId": "a", - "result_table_id": "dbm_system.disk", - "result_table_label": "os", - "time_field": "", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] - }, - { - "condition": "and", - "key": "mount_point", - "method": "neq", - "value": [ - "/" - ] - }, - { - "condition": "and", - "key": "mount_point", - "method": "neq", - "value": [ - "/usr/local" - ] - } - ] - } - ], - "refId": "A", - "source": "avg by (bk_target_ip, mount_point) (avg_over_time(bkmonitor:dbm_system:disk:in_use{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\",mount_point!=\"/\",mount_point!=\"/usr/local\"}[1m]))", - "step": "", - "type": "range" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "KBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 116 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" ], - "title": "分区使用率", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { + "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "KBs" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 100 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "uid": "P3XSxkgSz" }, - "targets": [ + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip $tag_device_name", + "query_configs": [ { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip $tag_device_name", - "query_configs": [ - { - "alias": "$tag_bk_target_ip $tag_device_name", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "device_name", - "bk_target_ip" - ], - "interval": 60, - "interval_unit": "s", - "method": "AVG", - "metric_field": "rkb_s", - "refId": "a", - "result_table_id": "dbm_system.io", - "result_table_label": "os", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] - } + "alias": "$tag_bk_target_ip $tag_device_name", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "device_name", + "bk_target_ip" + ], + "interval": 60, + "interval_unit": "s", + "method": "AVG", + "metric_field": "rkb_s", + "refId": "a", + "result_table_id": "dbm_system.io", + "result_table_label": "os", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" ] } - ], - "refId": "A", - "source": "avg by (device_name, bk_target_ip) (avg_over_time(bkmonitor:dbm_system:io:rkb_s{device_name!~\"nbd*|ram*|loop*\",app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"}[1m]))", - "step": "", - "type": "range" + ] } ], - "title": "磁盘读取", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "refId": "A", + "source": "avg by (device_name, bk_target_ip) (avg_over_time(bkmonitor:dbm_system:io:rkb_s{device_name!~\"nbd*|ram*|loop*\",app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"}[1m]))", + "step": "", + "type": "range" + } + ], + "title": "Disk Read Speed ", + "type": "timeseries" + }, + { + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "KBs" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 100 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "desc" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip $tag_device_name", - "query_configs": [ - { - "alias": "$tag_bk_target_ip $tag_device_name", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "bk_target_ip", - "device_name" - ], - "interval": "auto", - "interval_unit": "s", - "method": "AVG", - "metric_field": "wkb_s", - "refId": "a", - "result_table_id": "dbm_system.io", - "result_table_label": "os", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] - } - ] - } - ], - "refId": "A", - "source": "avg by (bk_target_ip, device_name) (avg_over_time(bkmonitor:dbm_system:io:wkb_s{device_name!~\"nbd*|ram*|loop*\",app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"}[1m]))", - "step": "", - "type": "range" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "KBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 116 + }, + "id": 65, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" ], - "title": "磁盘写入", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { + "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 108 - }, - "id": null, - "options": { - "legend": { - "calcs": [ - "max", - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "uid": "P3XSxkgSz" }, - "targets": [ + "expressionList": [], + "format": "time_series", + "host": [], + "mode": "code", + "module": [], + "promqlAlias": "$tag_bk_target_ip $tag_device_name", + "query_configs": [ { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "host": [], - "mode": "code", - "module": [], - "promqlAlias": "$tag_bk_target_ip $tag_device_name", - "query_configs": [ - { - "alias": "$tag_bk_target_ip $tag_device_name", - "data_source_label": "bk_monitor", - "data_type_label": "time_series", - "display": true, - "filter_dict": {}, - "functions": [], - "group_by": [ - "bk_target_ip", - "device_name" - ], - "interval": "auto", - "interval_unit": "s", - "method": "AVG", - "metric_field": "wkb_s", - "refId": "a", - "result_table_id": "dbm_system.io", - "result_table_label": "os", - "time_field": "time", - "where": [ - { - "key": "app", - "method": "eq", - "value": [ - "$app" - ] - }, - { - "condition": "and", - "key": "cluster_domain", - "method": "eq", - "value": [ - "$cluster_domain" - ] - }, - { - "condition": "and", - "key": "instance_role", - "method": "eq", - "value": [ - "es_master" - ] - } + "alias": "$tag_bk_target_ip $tag_device_name", + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "display": true, + "filter_dict": {}, + "functions": [], + "group_by": [ + "bk_target_ip", + "device_name" + ], + "interval": "auto", + "interval_unit": "s", + "method": "AVG", + "metric_field": "wkb_s", + "refId": "a", + "result_table_id": "dbm_system.io", + "result_table_label": "os", + "time_field": "time", + "where": [ + { + "key": "app", + "method": "eq", + "value": [ + "$app" + ] + }, + { + "condition": "and", + "key": "cluster_domain", + "method": "eq", + "value": [ + "$cluster_domain" + ] + }, + { + "condition": "and", + "key": "instance_role", + "method": "eq", + "value": [ + "es_master" ] } - ], - "refId": "A", - "source": "avg by (device_name, bk_target_ip) (avg_over_time(bkmonitor:dbm_system:io:util{app=\"$app\",cluster_domain=\"$cluster_domain\",device_name!~\"nbd*|ram*|loop*\",instance_role=\"es_master\"}[1m]))", - "step": "", - "type": "range" + ] } ], - "title": "磁盘ioutil", - "type": "timeseries" + "refId": "A", + "source": "avg by (bk_target_ip, device_name) (avg_over_time(bkmonitor:dbm_system:io:wkb_s{device_name!~\"nbd*|ram*|loop*\",app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_master\"}[1m]))", + "step": "", + "type": "range" } ], - "title": "master", - "type": "row" + "title": "Disk Write Speed", + "type": "timeseries" }, { "collapsed": false, @@ -7464,17 +7416,17 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 124 }, - "id": null, + "id": 67, "panels": [], - "title": "热节点", + "title": "Hot Nodes Info", "type": "row" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -7482,6 +7434,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -7495,6 +7448,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -7531,9 +7485,9 @@ "h": 8, "w": 12, "x": 0, - "y": 53 + "y": 125 }, - "id": null, + "id": 69, "options": { "legend": { "calcs": [ @@ -7556,7 +7510,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -7635,13 +7589,13 @@ "type": "range" } ], - "title": "每秒indexing", + "title": "Indexing Rating", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -7649,6 +7603,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -7662,6 +7617,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -7698,9 +7654,9 @@ "h": 8, "w": 12, "x": 12, - "y": 53 + "y": 125 }, - "id": null, + "id": 99, "options": { "legend": { "calcs": [ @@ -7721,7 +7677,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -7792,13 +7748,13 @@ "type": "range" } ], - "title": "每秒search", + "title": "Searching Rating", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -7806,6 +7762,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -7819,6 +7776,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -7855,9 +7813,9 @@ "h": 8, "w": 12, "x": 0, - "y": 61 + "y": 133 }, - "id": null, + "id": 150, "options": { "legend": { "calcs": [ @@ -7878,7 +7836,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -7955,13 +7913,13 @@ "type": "range" } ], - "title": "Write Active", + "title": "Write Active Threads", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -7969,6 +7927,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -7982,6 +7941,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8018,9 +7978,9 @@ "h": 8, "w": 12, "x": 12, - "y": 61 + "y": 133 }, - "id": null, + "id": 151, "options": { "legend": { "calcs": [ @@ -8041,7 +8001,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -8118,13 +8078,13 @@ "type": "range" } ], - "title": "Search Active", + "title": "Search Active Threads", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -8132,6 +8092,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -8145,6 +8106,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8181,9 +8143,9 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 141 }, - "id": null, + "id": 96, "options": { "legend": { "calcs": [ @@ -8204,7 +8166,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -8287,7 +8249,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -8295,6 +8257,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -8308,6 +8271,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8344,9 +8308,9 @@ "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 141 }, - "id": null, + "id": 94, "options": { "legend": { "calcs": [ @@ -8367,7 +8331,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -8450,7 +8414,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -8458,6 +8422,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -8471,6 +8436,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8507,9 +8473,9 @@ "h": 8, "w": 12, "x": 0, - "y": 77 + "y": 149 }, - "id": null, + "id": 95, "options": { "legend": { "calcs": [ @@ -8530,7 +8496,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -8621,7 +8587,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -8629,6 +8595,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -8642,6 +8609,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8678,9 +8646,9 @@ "h": 8, "w": 12, "x": 12, - "y": 77 + "y": 149 }, - "id": null, + "id": 93, "options": { "legend": { "calcs": [ @@ -8701,7 +8669,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -8784,7 +8752,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -8792,6 +8760,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -8805,6 +8774,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8842,9 +8812,9 @@ "h": 8, "w": 12, "x": 0, - "y": 85 + "y": 157 }, - "id": null, + "id": 92, "options": { "legend": { "calcs": [ @@ -8865,7 +8835,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -8880,13 +8850,13 @@ "type": "range" } ], - "title": "jvm heap使用率", + "title": "JVM Heap Usage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -8894,6 +8864,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -8907,6 +8878,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8943,9 +8915,9 @@ "h": 8, "w": 12, "x": 12, - "y": 85 + "y": 157 }, - "id": null, + "id": 91, "options": { "legend": { "calcs": [ @@ -8966,7 +8938,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -9064,7 +9036,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -9159,13 +9131,13 @@ "type": "range" } ], - "title": "jvm gc次数（1m）", + "title": "JVM GC Times", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -9173,6 +9145,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -9186,6 +9159,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9223,9 +9197,9 @@ "h": 8, "w": 12, "x": 0, - "y": 93 + "y": 165 }, - "id": null, + "id": 90, "options": { "legend": { "calcs": [ @@ -9246,7 +9220,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -9393,13 +9367,13 @@ "type": "range" } ], - "title": "jvm gc（STW）时间占比", + "title": "JVM GC（STW）Time Percatage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -9407,6 +9381,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -9420,6 +9395,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9457,9 +9433,9 @@ "h": 8, "w": 12, "x": 12, - "y": 93 + "y": 165 }, - "id": null, + "id": 89, "options": { "legend": { "calcs": [ @@ -9480,7 +9456,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -9549,13 +9525,13 @@ "type": "range" } ], - "title": "segment memory", + "title": "Segment Memory", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -9563,6 +9539,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -9576,6 +9553,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9613,9 +9591,9 @@ "h": 8, "w": 12, "x": 0, - "y": 101 + "y": 173 }, - "id": null, + "id": 88, "options": { "legend": { "calcs": [ @@ -9636,7 +9614,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -9696,13 +9674,13 @@ "type": "range" } ], - "title": "CPU使用率", + "title": "CPU Usage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -9710,6 +9688,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -9723,6 +9702,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9760,9 +9740,9 @@ "h": 8, "w": 12, "x": 12, - "y": 101 + "y": 173 }, - "id": null, + "id": 87, "options": { "legend": { "calcs": [ @@ -9783,7 +9763,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -9842,13 +9822,13 @@ "type": "range" } ], - "title": "系统负载（1m）", + "title": "Load(1m)", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -9856,6 +9836,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -9869,6 +9850,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9906,9 +9888,9 @@ "h": 8, "w": 12, "x": 0, - "y": 109 + "y": 181 }, - "id": null, + "id": 84, "options": { "legend": { "calcs": [ @@ -9929,7 +9911,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -9988,13 +9970,13 @@ "type": "range" } ], - "title": "入流量(单位：bits/s)", + "title": "Received Bits in Second (Unit：bits/s)", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -10002,6 +9984,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -10015,6 +9998,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10052,9 +10036,9 @@ "h": 8, "w": 12, "x": 12, - "y": 109 + "y": 181 }, - "id": null, + "id": 83, "options": { "legend": { "calcs": [ @@ -10075,7 +10059,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -10134,13 +10118,13 @@ "type": "range" } ], - "title": "出流量", + "title": "Sent Bits in Second ( Unit: bits/s )", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -10148,6 +10132,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -10161,6 +10146,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10197,9 +10183,9 @@ "h": 8, "w": 12, "x": 0, - "y": 117 + "y": 189 }, - "id": null, + "id": 82, "options": { "legend": { "calcs": [ @@ -10220,7 +10206,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -10279,13 +10265,13 @@ "type": "range" } ], - "title": "连接数", + "title": "Established TCP Connections", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -10293,6 +10279,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -10306,6 +10293,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10342,9 +10330,9 @@ "h": 8, "w": 12, "x": 12, - "y": 117 + "y": 189 }, - "id": null, + "id": 81, "options": { "legend": { "calcs": [ @@ -10365,7 +10353,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -10434,13 +10422,13 @@ "type": "range" } ], - "title": "新建连接数", + "title": "New TCP Connections", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -10448,6 +10436,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -10461,6 +10450,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10497,88 +10487,9 @@ "h": 8, "w": 12, "x": 0, - "y": 125 - }, - "id": null, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "title": "打开fd数", - "type": "timeseries" - }, - { - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] + "y": 197 }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 125 - }, - "id": null, + "id": 78, "options": { "legend": { "calcs": [ @@ -10590,8 +10501,8 @@ "showLegend": true }, "tooltip": { - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, "targets": [ @@ -10599,16 +10510,10 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, - "expressionList": [ - { - "active": true, - "alias": "使用率", - "expression": "a /b", - "functions": [] - } - ], + "enableDownSampling": true, + "expressionList": [], "format": "time_series", "host": [], "mode": "ui", @@ -10616,10 +10521,10 @@ "promqlAlias": "", "query_configs": [ { - "alias": "", + "alias": "Disk Used", "data_source_label": "bk_monitor", "data_type_label": "time_series", - "display": false, + "display": true, "filter_dict": {}, "functions": [], "group_by": [], @@ -10630,25 +10535,9 @@ "refId": "a", "result_table_id": "dbm_system.disk", "result_table_label": "os", - "time_field": "", + "time_field": "time", "where": [ { - "key": "mount_point", - "method": "neq", - "value": [ - "/" - ] - }, - { - "condition": "and", - "key": "mount_point", - "method": "neq", - "value": [ - "/usr/local" - ] - }, - { - "condition": "and", "key": "app", "method": "eq", "value": [ @@ -10672,40 +10561,46 @@ ] } ] - }, + } + ], + "refId": "A", + "source": "sum by(bk_target_ip) (bkmonitor:dbm_system:disk:used)", + "step": "", + "type": "range" + }, + { + "cluster": [], + "datasource": { + "type": "bkmonitor-timeseries-datasource", + "uid": "P3XSxkgSz" + }, + "enableDownSampling": true, + "expressionList": [], + "format": "time_series", + "hide": false, + "host": [], + "mode": "ui", + "module": [], + "promqlAlias": "", + "query_configs": [ { - "alias": "", + "alias": "Disk in Total", "data_source_label": "bk_monitor", "data_type_label": "time_series", + "display": true, "filter_dict": {}, "functions": [], - "group_by": [], - "interval": 60, - "interval_unit": "s", - "method": "sum_without_time", - "metric_field": "total", - "refId": "b", - "result_table_id": "dbm_system.disk", - "result_table_label": "os", - "time_field": "", - "where": [ - { - "key": "mount_point", - "method": "neq", - "value": [ - "/" - ] - }, - { - "condition": "and", - "key": "mount_point", - "method": "neq", - "value": [ - "/usr/local" - ] - }, + "group_by": [], + "interval": "auto", + "interval_unit": "s", + "method": "sum_without_time", + "metric_field": "total", + "refId": "a", + "result_table_id": "dbm_system.disk", + "result_table_label": "os", + "time_field": "time", + "where": [ { - "condition": "and", "key": "app", "method": "eq", "value": [ @@ -10731,19 +10626,19 @@ ] } ], - "refId": "A", - "source": "sum by(bk_target_ip, mount_point) (bkmonitor:dbm_system:disk:used{mount_point!=\"/\",mount_point!=\"/usr/local\"}) /sum by(bk_target_ip, mount_point) (bkmonitor:dbm_system:disk:total{mount_point!=\"/\",mount_point!=\"/usr/local\"})", + "refId": "B", + "source": "", "step": "", "type": "range" } ], - "title": "总磁盘使用率", + "title": "Disk Capacity", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -10751,6 +10646,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -10764,6 +10660,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10792,17 +10689,18 @@ "value": 80 } ] - } + }, + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 133 + "x": 12, + "y": 197 }, - "id": null, + "id": 79, "options": { "legend": { "calcs": [ @@ -10814,8 +10712,8 @@ "showLegend": true }, "tooltip": { - "mode": "multi", - "sort": "desc" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -10823,9 +10721,16 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, - "expressionList": [], + "expressionList": [ + { + "active": true, + "alias": "使用率", + "expression": "a /b", + "functions": [] + } + ], "format": "time_series", "host": [], "mode": "ui", @@ -10833,10 +10738,10 @@ "promqlAlias": "", "query_configs": [ { - "alias": "已使用容量", + "alias": "", "data_source_label": "bk_monitor", "data_type_label": "time_series", - "display": true, + "display": false, "filter_dict": {}, "functions": [], "group_by": [], @@ -10850,6 +10755,22 @@ "time_field": "", "where": [ { + "key": "mount_point", + "method": "neq", + "value": [ + "/" + ] + }, + { + "condition": "and", + "key": "mount_point", + "method": "neq", + "value": [ + "/usr/local" + ] + }, + { + "condition": "and", "key": "app", "method": "eq", "value": [ @@ -10873,45 +10794,40 @@ ] } ] - } - ], - "refId": "A", - "source": "sum by(bk_target_ip) (bkmonitor:dbm_system:disk:used)", - "step": "", - "type": "range" - }, - { - "cluster": [], - "datasource": { - "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" - }, - "expressionList": [], - "format": "time_series", - "hide": false, - "host": [], - "mode": "ui", - "module": [], - "promqlAlias": "", - "query_configs": [ + }, { - "alias": "总容量", + "alias": "", "data_source_label": "bk_monitor", "data_type_label": "time_series", - "display": true, "filter_dict": {}, "functions": [], "group_by": [], - "interval": "auto", + "interval": 60, "interval_unit": "s", "method": "sum_without_time", "metric_field": "total", - "refId": "a", + "refId": "b", "result_table_id": "dbm_system.disk", "result_table_label": "os", "time_field": "", "where": [ { + "key": "mount_point", + "method": "neq", + "value": [ + "/" + ] + }, + { + "condition": "and", + "key": "mount_point", + "method": "neq", + "value": [ + "/usr/local" + ] + }, + { + "condition": "and", "key": "app", "method": "eq", "value": [ @@ -10937,19 +10853,19 @@ ] } ], - "refId": "B", - "source": "", + "refId": "A", + "source": "sum by(bk_target_ip, mount_point) (bkmonitor:dbm_system:disk:used{mount_point!=\"/\",mount_point!=\"/usr/local\"}) /sum by(bk_target_ip, mount_point) (bkmonitor:dbm_system:disk:total{mount_point!=\"/\",mount_point!=\"/usr/local\"})", "step": "", "type": "range" } ], - "title": "磁盘容量", + "title": "Disk Usage in Total", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -10957,6 +10873,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -10970,6 +10887,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10999,22 +10917,22 @@ } ] }, - "unit": "percent" + "unit": "dtdurations" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 133 + "x": 0, + "y": 205 }, - "id": null, + "id": 72, "options": { "legend": { "calcs": [ "max", - "lastNotNull" + "last" ], "displayMode": "table", "placement": "right", @@ -11030,33 +10948,34 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", "host": [], "mode": "code", "module": [], - "promqlAlias": "$tag_bk_target_ip $tag_mount_point", + "promqlAlias": "$tag_name", "query_configs": [ { - "alias": "$tag_bk_target_ip $tag_mount_point", + "alias": "$tag___name", + "data_label": "exporter_dbm_elasticsearch_exporter", "data_source_label": "bk_monitor", "data_type_label": "time_series", + "display": true, "filter_dict": {}, "functions": [], "group_by": [ - "bk_target_ip", - "mount_point" + "__name" ], "interval": 60, "interval_unit": "s", - "method": "AVG", - "metric_field": "in_use", + "method": "max_without_time", + "metric_field": "elasticsearch_jvm_uptime_seconds", "refId": "a", - "result_table_id": "dbm_system.disk", - "result_table_label": "os", - "time_field": "", + "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", + "result_table_label": "component", + "time_field": "time", "where": [ { "key": "app", @@ -11075,44 +10994,34 @@ }, { "condition": "and", - "key": "instance_role", + "key": "es_data_node", "method": "eq", - "value": [ - "es_datanode_hot" - ] - }, - { - "condition": "and", - "key": "mount_point", - "method": "neq", - "value": [ - "/" - ] + "value": [] }, { "condition": "and", - "key": "mount_point", - "method": "neq", + "key": "__name", + "method": "reg", "value": [ - "/usr/local" + "dn-.*" ] } ] } ], "refId": "A", - "source": "avg by (bk_target_ip, mount_point) (avg_over_time(bkmonitor:dbm_system:disk:in_use{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_datanode_hot\",mount_point!=\"/\",mount_point!=\"/usr/local\"}[1m]))", + "source": "max by(name) (bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_uptime_seconds{name=~\"dn-.*|hot-.*\",app=\"$app\",cluster_domain=\"$cluster_domain\"})", "step": "", "type": "range" } ], - "title": "分区使用率", + "title": "Node Uptime", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -11120,6 +11029,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -11133,6 +11043,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11162,17 +11073,17 @@ } ] }, - "unit": "KBs" + "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 141 + "x": 12, + "y": 205 }, - "id": null, + "id": 77, "options": { "legend": { "calcs": [ @@ -11193,33 +11104,33 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", "host": [], "mode": "code", "module": [], - "promqlAlias": "$tag_bk_target_ip $tag_device_name", + "promqlAlias": "$tag_bk_target_ip $tag_mount_point", "query_configs": [ { - "alias": "$tag_bk_target_ip $tag_device_name", + "alias": "$tag_bk_target_ip $tag_mount_point", "data_source_label": "bk_monitor", "data_type_label": "time_series", "filter_dict": {}, "functions": [], "group_by": [ - "device_name", - "bk_target_ip" + "bk_target_ip", + "mount_point" ], "interval": 60, "interval_unit": "s", "method": "AVG", - "metric_field": "rkb_s", + "metric_field": "in_use", "refId": "a", - "result_table_id": "dbm_system.io", + "result_table_id": "dbm_system.disk", "result_table_label": "os", - "time_field": "time", + "time_field": "", "where": [ { "key": "app", @@ -11243,23 +11154,39 @@ "value": [ "es_datanode_hot" ] + }, + { + "condition": "and", + "key": "mount_point", + "method": "neq", + "value": [ + "/" + ] + }, + { + "condition": "and", + "key": "mount_point", + "method": "neq", + "value": [ + "/usr/local" + ] } ] } ], "refId": "A", - "source": "avg by (device_name, bk_target_ip) (avg_over_time(bkmonitor:dbm_system:io:rkb_s{device_name!~\"nbd*|ram*|loop*\",app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_datanode_hot\"}[1m]))", + "source": "avg by (bk_target_ip, mount_point) (avg_over_time(bkmonitor:dbm_system:disk:in_use{app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_datanode_hot\",mount_point!=\"/\",mount_point!=\"/usr/local\"}[1m]))", "step": "", "type": "range" } ], - "title": "磁盘读取", + "title": "Disk Usage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -11267,6 +11194,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -11280,6 +11208,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11316,10 +11245,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 141 + "x": 0, + "y": 213 }, - "id": null, + "id": 76, "options": { "legend": { "calcs": [ @@ -11340,7 +11269,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -11362,7 +11291,7 @@ "interval": 60, "interval_unit": "s", "method": "AVG", - "metric_field": "wkb_s", + "metric_field": "rkb_s", "refId": "a", "result_table_id": "dbm_system.io", "result_table_label": "os", @@ -11395,18 +11324,18 @@ } ], "refId": "A", - "source": "avg by (device_name, bk_target_ip) (avg_over_time(bkmonitor:dbm_system:io:wkb_s{device_name!~\"nbd*|ram*|loop*\",app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_datanode_hot\"}[1m]))", + "source": "avg by (device_name, bk_target_ip) (avg_over_time(bkmonitor:dbm_system:io:rkb_s{device_name!~\"nbd*|ram*|loop*\",app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_datanode_hot\"}[1m]))", "step": "", "type": "range" } ], - "title": "磁盘写入", + "title": "Disk Read Speed ", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -11414,6 +11343,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -11427,6 +11357,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11456,22 +11387,22 @@ } ] }, - "unit": "dtdurations" + "unit": "KBs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 149 + "x": 12, + "y": 213 }, - "id": null, + "id": 75, "options": { "legend": { "calcs": [ "max", - "last" + "lastNotNull" ], "displayMode": "table", "placement": "right", @@ -11487,33 +11418,32 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", "host": [], "mode": "code", "module": [], - "promqlAlias": "$tag_name", + "promqlAlias": "$tag_bk_target_ip $tag_device_name", "query_configs": [ { - "alias": "$tag___name", - "data_label": "exporter_dbm_elasticsearch_exporter", + "alias": "$tag_bk_target_ip $tag_device_name", "data_source_label": "bk_monitor", "data_type_label": "time_series", - "display": true, "filter_dict": {}, "functions": [], "group_by": [ - "__name" + "device_name", + "bk_target_ip" ], "interval": 60, "interval_unit": "s", - "method": "max_without_time", - "metric_field": "elasticsearch_jvm_uptime_seconds", + "method": "AVG", + "metric_field": "wkb_s", "refId": "a", - "result_table_id": "exporter_dbm_elasticsearch_exporter.__default__", - "result_table_label": "component", + "result_table_id": "dbm_system.io", + "result_table_label": "os", "time_field": "time", "where": [ { @@ -11533,34 +11463,28 @@ }, { "condition": "and", - "key": "es_data_node", + "key": "instance_role", "method": "eq", - "value": [] - }, - { - "condition": "and", - "key": "__name", - "method": "reg", "value": [ - "dn-.*" + "es_datanode_hot" ] } ] } ], "refId": "A", - "source": "max by(name) (bkmonitor:exporter_dbm_elasticsearch_exporter:__default__:elasticsearch_jvm_uptime_seconds{name=~\"dn-.*|hot-.*\",app=\"$app\",cluster_domain=\"$cluster_domain\"})", + "source": "avg by (device_name, bk_target_ip) (avg_over_time(bkmonitor:dbm_system:io:wkb_s{device_name!~\"nbd*|ram*|loop*\",app=\"$app\",cluster_domain=\"$cluster_domain\",instance_role=\"es_datanode_hot\"}[1m]))", "step": "", "type": "range" } ], - "title": "节点uptime", + "title": "Disk Write Speed", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -11568,6 +11492,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -11581,6 +11506,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11617,10 +11543,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 149 + "x": 0, + "y": 221 }, - "id": null, + "id": 152, "options": { "legend": { "calcs": [ @@ -11641,7 +11567,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -11701,7 +11627,7 @@ "type": "range" } ], - "title": "磁盘ioutil", + "title": "Disk IO Util", "type": "timeseries" }, { @@ -11710,17 +11636,17 @@ "h": 1, "w": 24, "x": 0, - "y": 157 + "y": 229 }, - "id": null, + "id": 101, "panels": [], - "title": "冷节点", + "title": "Cold Nodes Info", "type": "row" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -11728,6 +11654,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -11741,6 +11668,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11777,9 +11705,9 @@ "h": 8, "w": 12, "x": 0, - "y": 158 + "y": 230 }, - "id": null, + "id": 103, "options": { "legend": { "calcs": [ @@ -11800,7 +11728,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -11878,13 +11806,13 @@ "type": "range" } ], - "title": "每秒indexing", + "title": "Indexing Rating", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -11892,6 +11820,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -11905,6 +11834,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -11941,9 +11871,9 @@ "h": 8, "w": 12, "x": 12, - "y": 158 + "y": 230 }, - "id": null, + "id": 129, "options": { "legend": { "calcs": [ @@ -11964,7 +11894,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -11979,13 +11909,13 @@ "type": "range" } ], - "title": "每秒search", + "title": "Searching Rating", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -11993,6 +11923,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -12006,6 +11937,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12043,9 +11975,9 @@ "h": 8, "w": 12, "x": 0, - "y": 166 + "y": 238 }, - "id": null, + "id": 122, "options": { "legend": { "calcs": [ @@ -12066,7 +11998,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -12081,13 +12013,13 @@ "type": "range" } ], - "title": "jvm heap使用率", + "title": "JVM Heap Usage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -12095,6 +12027,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -12108,6 +12041,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12144,9 +12078,9 @@ "h": 8, "w": 12, "x": 12, - "y": 166 + "y": 238 }, - "id": null, + "id": 121, "options": { "legend": { "calcs": [ @@ -12167,7 +12101,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -12263,7 +12197,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -12357,13 +12291,13 @@ "type": "range" } ], - "title": "jvm gc次数（1m）", + "title": "JVM GC Times", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -12371,6 +12305,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -12384,6 +12319,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12421,9 +12357,9 @@ "h": 8, "w": 12, "x": 0, - "y": 174 + "y": 246 }, - "id": null, + "id": 120, "options": { "legend": { "calcs": [ @@ -12444,7 +12380,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -12591,13 +12527,13 @@ "type": "range" } ], - "title": "jvm gc（STW）时间占比", + "title": "JVM GC（STW）Time Percatage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -12605,6 +12541,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -12618,6 +12555,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12655,9 +12593,9 @@ "h": 8, "w": 12, "x": 12, - "y": 174 + "y": 246 }, - "id": null, + "id": 119, "options": { "legend": { "calcs": [ @@ -12678,7 +12616,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -12746,13 +12684,13 @@ "type": "range" } ], - "title": "segment memory", + "title": "Segment Memory", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -12760,6 +12698,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -12773,6 +12712,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12810,9 +12750,9 @@ "h": 8, "w": 12, "x": 0, - "y": 182 + "y": 254 }, - "id": null, + "id": 118, "options": { "legend": { "calcs": [ @@ -12833,7 +12773,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -12892,13 +12832,13 @@ "type": "range" } ], - "title": "CPU使用率", + "title": "CPU Usage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -12906,6 +12846,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -12919,6 +12860,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -12955,9 +12897,9 @@ "h": 8, "w": 12, "x": 12, - "y": 182 + "y": 254 }, - "id": null, + "id": 117, "options": { "legend": { "calcs": [ @@ -12978,7 +12920,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -13037,13 +12979,13 @@ "type": "range" } ], - "title": "系统负载（1m）", + "title": "Load(1m)", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -13051,6 +12993,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -13064,6 +13007,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13100,9 +13044,9 @@ "h": 8, "w": 12, "x": 0, - "y": 190 + "y": 262 }, - "id": null, + "id": 153, "options": { "legend": { "calcs": [ @@ -13123,7 +13067,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -13200,13 +13144,13 @@ "type": "range" } ], - "title": "Write Active", + "title": "Write Active Threads", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -13214,6 +13158,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -13227,6 +13172,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13263,9 +13209,9 @@ "h": 8, "w": 12, "x": 12, - "y": 190 + "y": 262 }, - "id": null, + "id": 156, "options": { "legend": { "calcs": [ @@ -13286,7 +13232,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -13363,13 +13309,13 @@ "type": "range" } ], - "title": "Search Active", + "title": "Search Active Threads", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -13377,6 +13323,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -13390,6 +13337,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13426,9 +13374,9 @@ "h": 8, "w": 12, "x": 0, - "y": 198 + "y": 270 }, - "id": null, + "id": 154, "options": { "legend": { "calcs": [ @@ -13449,7 +13397,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -13532,7 +13480,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -13540,6 +13488,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -13553,6 +13502,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13589,9 +13539,9 @@ "h": 8, "w": 12, "x": 12, - "y": 198 + "y": 270 }, - "id": null, + "id": 157, "options": { "legend": { "calcs": [ @@ -13612,7 +13562,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -13695,7 +13645,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -13703,6 +13653,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -13716,6 +13667,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13752,9 +13704,9 @@ "h": 8, "w": 12, "x": 0, - "y": 206 + "y": 278 }, - "id": null, + "id": 155, "options": { "legend": { "calcs": [ @@ -13775,7 +13727,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -13866,7 +13818,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -13874,6 +13826,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -13887,6 +13840,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -13923,9 +13877,9 @@ "h": 8, "w": 12, "x": 12, - "y": 206 + "y": 278 }, - "id": null, + "id": 158, "options": { "legend": { "calcs": [ @@ -13946,7 +13900,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -14037,7 +13991,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -14045,6 +13999,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -14058,6 +14013,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14095,9 +14051,9 @@ "h": 8, "w": 12, "x": 0, - "y": 214 + "y": 286 }, - "id": null, + "id": 114, "options": { "legend": { "calcs": [ @@ -14118,7 +14074,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -14177,13 +14133,13 @@ "type": "range" } ], - "title": "入流量(单位：bits/s)", + "title": "Received Bits in Second (Unit：bits/s)", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -14191,6 +14147,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -14204,6 +14161,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14241,9 +14199,9 @@ "h": 8, "w": 12, "x": 12, - "y": 214 + "y": 286 }, - "id": null, + "id": 113, "options": { "legend": { "calcs": [ @@ -14264,7 +14222,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -14323,13 +14281,13 @@ "type": "range" } ], - "title": "出流量(单位：bits/s)", + "title": "Sent Bits in Second ( Unit: bits/s )", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -14337,6 +14295,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -14350,6 +14309,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14386,9 +14346,9 @@ "h": 8, "w": 12, "x": 0, - "y": 222 + "y": 294 }, - "id": null, + "id": 112, "options": { "legend": { "calcs": [ @@ -14409,7 +14369,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -14468,13 +14428,13 @@ "type": "range" } ], - "title": "连接数", + "title": "Established TCP Connections", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -14482,6 +14442,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -14495,6 +14456,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14531,9 +14493,9 @@ "h": 8, "w": 12, "x": 12, - "y": 222 + "y": 294 }, - "id": null, + "id": 111, "options": { "legend": { "calcs": [ @@ -14554,7 +14516,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -14623,13 +14585,13 @@ "type": "range" } ], - "title": "新建连接数", + "title": "New Connections", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -14637,6 +14599,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -14650,6 +14613,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14687,9 +14651,9 @@ "h": 8, "w": 12, "x": 0, - "y": 230 + "y": 302 }, - "id": null, + "id": 108, "options": { "legend": { "calcs": [ @@ -14710,14 +14674,15 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, + "enableDownSampling": false, "expressionList": [], "format": "time_series", "host": [], "mode": "code", "module": [], - "promqlAlias": "总容量", + "promqlAlias": "Disk in Total", "query_configs": [ { "alias": "总容量", @@ -14771,15 +14736,16 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, + "enableDownSampling": false, "expressionList": [], "format": "time_series", "hide": false, "host": [], "mode": "code", "module": [], - "promqlAlias": "已使用容量", + "promqlAlias": "Disk Used", "query_configs": [ { "alias": "已使用容量", @@ -14830,13 +14796,13 @@ "type": "range" } ], - "title": "总磁盘容量", + "title": "Disk Capacity", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -14844,6 +14810,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -14857,6 +14824,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -14894,9 +14862,9 @@ "h": 8, "w": 12, "x": 12, - "y": 230 + "y": 302 }, - "id": null, + "id": 109, "options": { "legend": { "calcs": [ @@ -14917,7 +14885,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -15054,13 +15022,13 @@ "type": "range" } ], - "title": "总磁盘使用率", + "title": "Disk Usage in Total", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -15068,6 +15036,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -15081,6 +15050,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15118,9 +15088,9 @@ "h": 8, "w": 12, "x": 0, - "y": 238 + "y": 310 }, - "id": null, + "id": 106, "options": { "legend": { "calcs": [ @@ -15141,7 +15111,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -15201,13 +15171,13 @@ "type": "range" } ], - "title": "磁盘读取", + "title": "Disk Read Speed ", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -15215,6 +15185,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -15228,6 +15199,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15265,9 +15237,9 @@ "h": 8, "w": 12, "x": 12, - "y": 238 + "y": 310 }, - "id": null, + "id": 105, "options": { "legend": { "calcs": [ @@ -15288,7 +15260,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -15348,13 +15320,13 @@ "type": "range" } ], - "title": "磁盘写入", + "title": "Disk Write Speed", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -15362,6 +15334,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -15375,6 +15348,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15412,9 +15386,9 @@ "h": 8, "w": 12, "x": 0, - "y": 246 + "y": 318 }, - "id": null, + "id": 104, "options": { "legend": { "calcs": [ @@ -15435,7 +15409,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -15496,13 +15470,13 @@ "type": "range" } ], - "title": "节点uptime", + "title": "Node Uptime", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -15510,6 +15484,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -15523,6 +15498,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15560,9 +15536,9 @@ "h": 8, "w": 12, "x": 12, - "y": 246 + "y": 318 }, - "id": null, + "id": 107, "options": { "legend": { "calcs": [ @@ -15583,7 +15559,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -15659,13 +15635,13 @@ "type": "range" } ], - "title": "分区使用率", + "title": "Disk Usage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -15673,6 +15649,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -15686,6 +15663,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15723,9 +15701,9 @@ "h": 8, "w": 12, "x": 0, - "y": 254 + "y": 326 }, - "id": null, + "id": 159, "options": { "legend": { "calcs": [ @@ -15746,7 +15724,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -15806,7 +15784,7 @@ "type": "range" } ], - "title": "磁盘ioutil", + "title": "Disk IO Util", "type": "timeseries" }, { @@ -15815,17 +15793,17 @@ "h": 1, "w": 24, "x": 0, - "y": 262 + "y": 334 }, - "id": null, + "id": 131, "panels": [], - "title": "client", + "title": "Client Nodes Info", "type": "row" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -15833,6 +15811,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -15846,6 +15825,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15883,9 +15863,9 @@ "h": 8, "w": 12, "x": 0, - "y": 263 + "y": 335 }, - "id": null, + "id": 133, "options": { "legend": { "calcs": [ @@ -15906,7 +15886,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -15921,13 +15901,13 @@ "type": "range" } ], - "title": "jvm heap使用率", + "title": "JVM Heap Usage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -15935,6 +15915,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -15948,6 +15929,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -15984,9 +15966,9 @@ "h": 8, "w": 12, "x": 12, - "y": 263 + "y": 335 }, - "id": null, + "id": 148, "options": { "legend": { "calcs": [ @@ -16007,7 +15989,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -16095,7 +16077,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [ { @@ -16181,13 +16163,13 @@ "type": "range" } ], - "title": "jvm gc次数（1m）", + "title": "JVM GC Times", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -16195,6 +16177,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -16208,6 +16191,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16245,9 +16229,9 @@ "h": 8, "w": 12, "x": 0, - "y": 271 + "y": 343 }, - "id": null, + "id": 147, "options": { "legend": { "calcs": [ @@ -16268,7 +16252,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -16283,13 +16267,13 @@ "type": "range" } ], - "title": "jvm gc时间占比", + "title": "JVM GC（STW）Time Percatage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -16297,6 +16281,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -16310,6 +16295,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16347,9 +16333,9 @@ "h": 8, "w": 12, "x": 12, - "y": 271 + "y": 343 }, - "id": null, + "id": 146, "options": { "legend": { "calcs": [ @@ -16370,7 +16356,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -16430,13 +16416,13 @@ "type": "range" } ], - "title": "节点uptime", + "title": "Node Uptime", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -16444,6 +16430,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -16457,6 +16444,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16494,9 +16482,9 @@ "h": 8, "w": 12, "x": 0, - "y": 279 + "y": 351 }, - "id": null, + "id": 145, "options": { "legend": { "calcs": [ @@ -16517,7 +16505,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -16576,13 +16564,13 @@ "type": "range" } ], - "title": "CPU使用率", + "title": "CPU Usage", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -16590,6 +16578,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -16603,6 +16592,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16639,9 +16629,9 @@ "h": 8, "w": 12, "x": 12, - "y": 279 + "y": 351 }, - "id": null, + "id": 144, "options": { "legend": { "calcs": [ @@ -16662,7 +16652,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -16721,13 +16711,13 @@ "type": "range" } ], - "title": "系统负载（1m）", + "title": "Load(1m)", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -16735,6 +16725,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -16748,6 +16739,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16785,9 +16777,9 @@ "h": 8, "w": 12, "x": 0, - "y": 287 + "y": 359 }, - "id": null, + "id": 141, "options": { "legend": { "calcs": [ @@ -16808,7 +16800,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -16867,13 +16859,13 @@ "type": "range" } ], - "title": "入流量(单位：bits/s)", + "title": "Received Bits in Second (Unit：bits/s)", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -16881,6 +16873,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -16894,6 +16887,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -16931,9 +16925,9 @@ "h": 8, "w": 12, "x": 12, - "y": 287 + "y": 359 }, - "id": null, + "id": 140, "options": { "legend": { "calcs": [ @@ -16954,7 +16948,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -17013,13 +17007,13 @@ "type": "range" } ], - "title": "出流量(单位：bits/s)", + "title": "Sent Bits in Second ( Unit: bits/s )", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -17027,6 +17021,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -17040,6 +17035,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17076,9 +17072,9 @@ "h": 8, "w": 12, "x": 0, - "y": 295 + "y": 367 }, - "id": null, + "id": 139, "options": { "legend": { "calcs": [ @@ -17099,7 +17095,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -17158,13 +17154,13 @@ "type": "range" } ], - "title": "连接数", + "title": "Established TCP Connections", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -17172,6 +17168,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -17185,6 +17182,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17221,9 +17219,9 @@ "h": 8, "w": 12, "x": 12, - "y": 295 + "y": 367 }, - "id": null, + "id": 138, "options": { "legend": { "calcs": [ @@ -17244,7 +17242,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -17313,13 +17311,13 @@ "type": "range" } ], - "title": "新建连接数", + "title": "New Connections", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -17327,6 +17325,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -17340,6 +17339,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17376,9 +17376,9 @@ "h": 8, "w": 12, "x": 0, - "y": 303 + "y": 375 }, - "id": null, + "id": 160, "options": { "legend": { "calcs": [ @@ -17399,7 +17399,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -17476,13 +17476,13 @@ "type": "range" } ], - "title": "Write Active", + "title": "Write Active Threads", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -17490,6 +17490,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -17503,6 +17504,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17539,9 +17541,9 @@ "h": 8, "w": 12, "x": 12, - "y": 303 + "y": 375 }, - "id": null, + "id": 161, "options": { "legend": { "calcs": [ @@ -17562,7 +17564,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -17639,13 +17641,13 @@ "type": "range" } ], - "title": "Search Active", + "title": "Search Active Threads", "type": "timeseries" }, { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -17653,6 +17655,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -17666,6 +17669,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17702,9 +17706,9 @@ "h": 8, "w": 12, "x": 0, - "y": 311 + "y": 383 }, - "id": null, + "id": 162, "options": { "legend": { "calcs": [ @@ -17725,7 +17729,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -17808,7 +17812,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -17816,6 +17820,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -17829,6 +17834,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -17865,9 +17871,9 @@ "h": 8, "w": 12, "x": 12, - "y": 311 + "y": 383 }, - "id": null, + "id": 163, "options": { "legend": { "calcs": [ @@ -17888,7 +17894,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -17971,7 +17977,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -17979,6 +17985,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -17992,6 +17999,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18028,9 +18036,9 @@ "h": 8, "w": 12, "x": 0, - "y": 319 + "y": 391 }, - "id": null, + "id": 164, "options": { "legend": { "calcs": [ @@ -18051,7 +18059,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -18142,7 +18150,7 @@ { "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "fieldConfig": { "defaults": { @@ -18150,6 +18158,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -18163,6 +18172,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -18199,9 +18209,9 @@ "h": 8, "w": 12, "x": 12, - "y": 319 + "y": 391 }, - "id": null, + "id": 165, "options": { "legend": { "calcs": [ @@ -18222,7 +18232,7 @@ "cluster": [], "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "expressionList": [], "format": "time_series", @@ -18312,19 +18322,22 @@ } ], "refresh": false, - "schemaVersion": 37, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [ { - "current": {}, + "current": { + "selected": false, + "text": "jlsy", + "value": "jlsy" + }, "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "definition": "- Blueking Monitor - 维度", - "hide": 2, + "hide": 0, "includeAll": false, "label": "app", "multi": false, @@ -18359,10 +18372,14 @@ "type": "query" }, { - "current": {}, + "current": { + "selected": false, + "text": "es.b2-log.jlsy.db", + "value": "es.b2-log.jlsy.db" + }, "datasource": { "type": "bkmonitor-timeseries-datasource", - "uid": "bkmonitor_timeseries" + "uid": "P3XSxkgSz" }, "definition": "- Blueking Monitor - 维度", "hide": 0, @@ -18412,10 +18429,12 @@ "from": "now-30m", "to": "now" }, - "timepicker": {}, + "timepicker": { + "hidden": false + }, "timezone": "default", "title": "ES", "uid": "aVri53FIk", - "version": 75, + "version": 83, "weekStart": "" } diff --git a/dbm-ui/backend/bk_dataview/dashboards/json/tendis-predixyrediscluster.json b/dbm-ui/backend/bk_dataview/dashboards/json/tendis-predixyrediscluster.json index 0d7dca563b..6c689078bc 100644 --- a/dbm-ui/backend/bk_dataview/dashboards/json/tendis-predixyrediscluster.json +++ b/dbm-ui/backend/bk_dataview/dashboards/json/tendis-predixyrediscluster.json @@ -9745,10 +9745,10 @@ "include": { "names": [ "ip", - "Value 1", - "Value 2", - "Value 3", - "Value 4" + "Value #A", + "Value #B", + "Value #C", + "Value #D" ] } } @@ -9759,10 +9759,10 @@ "excludeByName": {}, "indexByName": {}, "renameByName": { - "Value 1": "TotalMemory", - "Value 2": "UsedMemory", - "Value 3": "UsedPct%", - "Value 4": "ShardNum", + "Value #A": "TotalMemory", + "Value #B": "UsedMemory", + "Value #C": "UsedPct%", + "Value #D": "ShardNum", "ip": "" } } @@ -10152,10 +10152,10 @@ "include": { "names": [ "instance", - "Value 1", - "Value 2", - "Value 3", - "Value 4" + "Value #A", + "Value #B", + "Value #C", + "Value #D" ] } } @@ -10165,10 +10165,10 @@ "options": { "alias": "MemUsedRate", "binary": { - "left": "Value 1", + "left": "Value #A", "operator": "/", "reducer": "sum", - "right": "Value 3" + "right": "Value #C" }, "mode": "binary", "reduce": { @@ -10184,10 +10184,10 @@ "indexByName": {}, "renameByName": { "MemUsedRate": "", - "Value 1": "MemUsed", - "Value 2": "MemUsedRss", - "Value 3": "MaxMemory", - "Value 4": "MaxMemoryPolicy", + "Value #A": "MemUsed", + "Value #B": "MemUsedRss", + "Value #C": "MaxMemory", + "Value #D": "MaxMemoryPolicy", "instance": "" } } diff --git a/dbm-ui/backend/bk_dataview/dashboards/json/tendis-redisinstance.json b/dbm-ui/backend/bk_dataview/dashboards/json/tendis-redisinstance.json index 7b6f7254fe..e34669b6f9 100644 --- a/dbm-ui/backend/bk_dataview/dashboards/json/tendis-redisinstance.json +++ b/dbm-ui/backend/bk_dataview/dashboards/json/tendis-redisinstance.json @@ -6033,10 +6033,10 @@ "include": { "names": [ "ip", - "Value 1", - "Value 2", - "Value 3", - "Value 4" + "Value #A", + "Value #B", + "Value #C", + "Value #D" ] } } @@ -6047,10 +6047,10 @@ "excludeByName": {}, "indexByName": {}, "renameByName": { - "Value 1": "TotalMemory", - "Value 2": "UsedMemory", - "Value 3": "UsedPct%", - "Value 4": "ShardNum", + "Value #A": "TotalMemory", + "Value #B": "UsedMemory", + "Value #C": "UsedPct%", + "Value #D": "ShardNum", "ip": "" } } @@ -6440,10 +6440,10 @@ "include": { "names": [ "instance", - "Value 1", - "Value 2", - "Value 3", - "Value 4" + "Value #A", + "Value #B", + "Value #C", + "Value #D" ] } } @@ -6453,10 +6453,10 @@ "options": { "alias": "MemUsedRate", "binary": { - "left": "Value 1", + "left": "Value #A", "operator": "/", "reducer": "sum", - "right": "Value 3" + "right": "Value #C" }, "mode": "binary", "reduce": { @@ -6472,10 +6472,10 @@ "indexByName": {}, "renameByName": { "MemUsedRate": "", - "Value 1": "MemUsed", - "Value 2": "MemUsedRss", - "Value 3": "MaxMemory", - "Value 4": "MaxMemoryPolicy", + "Value #A": "MemUsed", + "Value #B": "MemUsedRss", + "Value #C": "MaxMemory", + "Value #D": "MaxMemoryPolicy", "instance": "" } } diff --git a/dbm-ui/backend/bk_dataview/dashboards/json/tendiscache.json b/dbm-ui/backend/bk_dataview/dashboards/json/tendiscache.json index c4f1b44324..d543e595eb 100644 --- a/dbm-ui/backend/bk_dataview/dashboards/json/tendiscache.json +++ b/dbm-ui/backend/bk_dataview/dashboards/json/tendiscache.json @@ -9905,10 +9905,10 @@ "include": { "names": [ "ip", - "Value 1", - "Value 2", - "Value 3", - "Value 4" + "Value #A", + "Value #B", + "Value #C", + "Value #D" ] } } @@ -9919,10 +9919,10 @@ "excludeByName": {}, "indexByName": {}, "renameByName": { - "Value 1": "TotalMemory", - "Value 2": "UsedMemory", - "Value 3": "UsedPct%", - "Value 4": "ShardNum", + "Value #A": "TotalMemory", + "Value #B": "UsedMemory", + "Value #C": "UsedPct%", + "Value #D": "ShardNum", "ip": "" } } @@ -10312,10 +10312,10 @@ "include": { "names": [ "instance", - "Value 1", - "Value 2", - "Value 3", - "Value 4" + "Value #A", + "Value #B", + "Value #C", + "Value #D" ] } } @@ -10325,10 +10325,10 @@ "options": { "alias": "MemUsedRate", "binary": { - "left": "Value 1", + "left": "Value #A", "operator": "/", "reducer": "sum", - "right": "Value 3" + "right": "Value #C" }, "mode": "binary", "reduce": { @@ -10344,10 +10344,10 @@ "indexByName": {}, "renameByName": { "MemUsedRate": "", - "Value 1": "MemUsed", - "Value 2": "MemUsedRss", - "Value 3": "MaxMemory", - "Value 4": "MaxMemoryPolicy", + "Value #A": "MemUsed", + "Value #B": "MemUsedRss", + "Value #C": "MaxMemory", + "Value #D": "MaxMemoryPolicy", "instance": "" } } diff --git a/dbm-ui/backend/bk_dataview/dashboards/readme.md b/dbm-ui/backend/bk_dataview/dashboards/readme.md index bf71f3570e..54d350d8df 100644 --- a/dbm-ui/backend/bk_dataview/dashboards/readme.md +++ b/dbm-ui/backend/bk_dataview/dashboards/readme.md @@ -47,12 +47,9 @@ cd backend/bk_dataview/dashboards/json find . -type f -name "*.json" -exec sed -i '' -e 's#"url": "\(.*\)/grafana/\(.*\)orgId=[0-9]*\(.*\)"#"url": "{BK_SAAS_HOST}/grafana/\2orgId=1\3"#' {} \; ``` - # 这里按需隐藏变量，不暴露所有变量 templating.list.hide 设置为 2，这里一般把需暴露出来的变量设置为 0，不暴露的设置为 2 - -# 替换 datasource 为 bkmonitor_timeseries 或者 bklog -templating.list.datasource.uid 设置为 bkmonitor_timeseries 或者 bklog +同步程序会自动把 app 变量隐藏 # 导入监控的方法，还原模板后导入 diff --git a/dbm-ui/backend/bk_dataview/grafana/provisioning.py b/dbm-ui/backend/bk_dataview/grafana/provisioning.py index d0acd4ebf0..852e08152e 100644 --- a/dbm-ui/backend/bk_dataview/grafana/provisioning.py +++ b/dbm-ui/backend/bk_dataview/grafana/provisioning.py @@ -157,6 +157,9 @@ def replace_dashboard(cls, dashboard: dict, used_index_name: list, index_name_id datasource_type, tpl_uid = cls.get_obj_datasource_type_uid(tpl) if datasource_type: dashboard["templating"]["list"][tpl_index]["datasource"]["uid"] = tpl_uid + # 隐藏掉 app（业务）选择器 + if tpl["name"] == "app": + dashboard["templating"]["list"][tpl_index]["hide"] = 2 return dashboard diff --git a/dbm-ui/backend/bk_dataview/grafana/views.py b/dbm-ui/backend/bk_dataview/grafana/views.py index 69953d98ce..ba41802d7e 100644 --- a/dbm-ui/backend/bk_dataview/grafana/views.py +++ b/dbm-ui/backend/bk_dataview/grafana/views.py @@ -21,6 +21,8 @@ from django.views.decorators.csrf import csrf_exempt from django.views.generic import View +from backend import env +from backend.bk_web.exceptions import ExternalClusterIdInvalidException from backend.configuration.constants import SystemSettingsEnum from backend.configuration.models import SystemSettings from backend.db_meta.enums import ClusterType @@ -412,6 +414,11 @@ def _auth(self, request): resource_meta = ResourceEnum.cluster_type_to_resource_meta(cluster.cluster_type) resources = [[resource] for resource in resource_meta.batch_create_instances([cluster.id])] result = IAMPermission(actions, resources).has_permission(request, "") + + # 针对外部查询，需在判断是否集群是否在允许的白名单内 + if env.ENABLE_EXTERNAL_PROXY and cluster.id not in SystemSettings.get_external_whitelist_cluster_ids(): + raise ExternalClusterIdInvalidException(cluster_id=cluster.id) + if not result: raise PermissionError diff --git a/dbm-ui/backend/bk_web/constants.py b/dbm-ui/backend/bk_web/constants.py index 5bba47b9b3..dbad96e890 100644 --- a/dbm-ui/backend/bk_web/constants.py +++ b/dbm-ui/backend/bk_web/constants.py @@ -56,8 +56,8 @@ r"/apis/cmdb/[0-9]+/list_modules/", r"/apis/mysql/bizs/[0-9]+/remote_service/show_cluster_databases/", r"/apis/dbbase/simple_query_cluster/", - # webconsole - r"/apis/dbbase/webconsole/", + # webconsole # TODO 暂时下架 webconsole 功能，待安全加固后再 + # r"/apis/dbbase/webconsole/", # 平台设置相关接口 "/apis/conf/biz_settings/simple/", # grafana和监控相关接口 diff --git a/dbm-ui/backend/bk_web/exceptions.py b/dbm-ui/backend/bk_web/exceptions.py index dfe0221339..fccec45e9f 100644 --- a/dbm-ui/backend/bk_web/exceptions.py +++ b/dbm-ui/backend/bk_web/exceptions.py @@ -27,3 +27,9 @@ class ExternalUserNotExistException(ExternalProxyBaseException): class ExternalRouteInvalidException(ExternalProxyBaseException): ERROR_CODE = "002" MESSAGE = _("转发路由非法") + + +class ExternalClusterIdInvalidException(ExternalProxyBaseException): + ERROR_CODE = "003" + MESSAGE = _("转发集群请求非法") + MESSAGE_TPL = _("转发集群[{cluster_id}]请求非法") diff --git a/dbm-ui/backend/bk_web/middleware.py b/dbm-ui/backend/bk_web/middleware.py index c988d045b7..6f03358786 100644 --- a/dbm-ui/backend/bk_web/middleware.py +++ b/dbm-ui/backend/bk_web/middleware.py @@ -30,8 +30,14 @@ NON_EXTERNAL_PROXY_ROUTING, ROUTING_WHITELIST_PATTERNS, ) -from backend.bk_web.exceptions import ExternalProxyBaseException, ExternalRouteInvalidException +from backend.bk_web.exceptions import ( + ExternalClusterIdInvalidException, + ExternalProxyBaseException, + ExternalRouteInvalidException, +) from backend.bk_web.handlers import _error +from backend.configuration.models import SystemSettings +from backend.db_services.dbbase.views import DBBaseViewSet from backend.ticket.views import TicketViewSet from backend.utils.local import local from backend.utils.string import str2bool @@ -166,6 +172,8 @@ def check_create_ticket(): # 目前只放开数据导出 if data["ticket_type"] not in EXTERNAL_TICKET_TYPE_WHITELIST: raise ExternalRouteInvalidException(_("单据类型[{}]非法，未开通白名单").format(data["ticket_type"])) + if data["details"]["cluster_id"] not in SystemSettings.get_external_whitelist_cluster_ids(): + raise ExternalClusterIdInvalidException(cluster_id=data["cluster_id"]) # 单据过滤校验函数 def check_list_ticket(): @@ -174,9 +182,16 @@ def check_list_ticket(): data["ticket_type__in"] = ",".join(EXTERNAL_TICKET_TYPE_WHITELIST) request.GET = data + def check_webconsole(): + data = json.loads(request.body.decode("utf-8")) + # 校验集群是否在白名单中 + if data["cluster_id"] not in SystemSettings.get_external_whitelist_cluster_ids(): + raise ExternalClusterIdInvalidException(cluster_id=data["cluster_id"]) + check_action_func_map = { f"{TicketViewSet.__name__}.{TicketViewSet.create.__name__}": check_create_ticket, f"{TicketViewSet.__name__}.{TicketViewSet.list.__name__}": check_list_ticket, + f"{DBBaseViewSet.__name__}.{DBBaseViewSet.webconsole.__name__}": check_webconsole, } # 根据请求的视图 + 动作判断是否特殊接口，以及接口参数是否合法 try: diff --git a/dbm-ui/backend/bk_web/viewsets.py b/dbm-ui/backend/bk_web/viewsets.py index d3c68495bf..5e46290db5 100644 --- a/dbm-ui/backend/bk_web/viewsets.py +++ b/dbm-ui/backend/bk_web/viewsets.py @@ -235,18 +235,11 @@ def after_response(self, request, response, *args, **kwargs): ): data = re.sub(IP_RE, "*.*.*.*", response.content.decode("utf-8")) return Response(json.loads(data)) - # for path in [ - # "/timeseries/time_series/unify_query/", - # "/timeseries/graph_promql_query/", - # "/timeseries/grafana/query/", - # "/timeseries/grafana/query_log/", - # ]: - # if request.path.endswith(path): - # # 外部 API 转发请求，把 IP 替换为 *.*.*.* - # data = re.sub(IP_RE, "*.*.*.*", response.content.decode("utf-8")) - # return Response(json.loads(data)) - - return HttpResponse(response) + + # 按原样补充响应头 + content_type = response.headers.get("Content-Type", "") + http_response = HttpResponse(response.content, status=response.status_code, content_type=content_type) + return http_response def external_proxy(self, request, *args, **kwargs): params = request.data or request.query_params diff --git a/dbm-ui/backend/configuration/constants.py b/dbm-ui/backend/configuration/constants.py index c493504c0b..0b8a8824bf 100644 --- a/dbm-ui/backend/configuration/constants.py +++ b/dbm-ui/backend/configuration/constants.py @@ -104,6 +104,7 @@ class SystemSettingsEnum(str, StructuredEnum): # 主机默认统一转移到 DBM 业务下托管，若业务 ID 属于这个列表，则转移到对应的业务下 INDEPENDENT_HOSTING_BIZS = EnumField("INDEPENDENT_HOSTING_BIZS", _("独立托管机器的业务列表")) BF_WHITELIST_BIZS = EnumField("BF_WHITELIST_BIZS", _("BF业务白名单")) + EXTERNAL_WHITELIST_CLUSTER_IDS = EnumField("EXTERNAL_WHITELIST_CLUSTER_IDS", _("外部访问集群ID白名单列表")) SPEC_OFFSET = EnumField("SPEC_OFFSET", _("默认的规格参数偏移量")) DEVICE_CLASSES = EnumField("DEVICE_CLASSES", _("机型列表")) BKM_DUTY_NOTICE = EnumField("BKM_DUTY_NOTICE", _("轮值通知设置")) diff --git a/dbm-ui/backend/configuration/models/system.py b/dbm-ui/backend/configuration/models/system.py index 78e58f3a96..90c00a42a5 100644 --- a/dbm-ui/backend/configuration/models/system.py +++ b/dbm-ui/backend/configuration/models/system.py @@ -33,7 +33,7 @@ class AbstractSettings(AuditedModel): desc = models.CharField(_("描述"), max_length=LEN_LONG) @classmethod - def get_setting_value(cls, key: dict, default: Optional[Any] = None) -> Union[str, Dict]: + def get_setting_value(cls, key: dict, default: Optional[Any] = None) -> Union[str, Dict, List]: """插入一条配置记录""" try: setting_value = cls.objects.get(**key).value @@ -103,7 +103,7 @@ def register_system_settings(cls): setattr(settings, system_setting.key, system_setting.value) @classmethod - def get_setting_value(cls, key: str, default: Optional[Any] = None) -> Union[str, Dict]: + def get_setting_value(cls, key: str, default: Optional[Any] = None) -> Union[str, Dict, List]: return super().get_setting_value(key={"key": key}, default=default) @classmethod @@ -116,6 +116,15 @@ def insert_setting_value(cls, key: str, value: Any, value_type: str = "str", use desc=constants.SystemSettingsEnum.get_choice_label(key), ) + @classmethod + def get_external_whitelist_cluster_ids(cls) -> List[int]: + return [ + conf["cluster_id"] + for conf in cls.get_setting_value( + key=constants.SystemSettingsEnum.EXTERNAL_WHITELIST_CLUSTER_IDS.value, default=[] + ) + ] + class BizSettings(AbstractSettings): """业务配置表""" diff --git a/dbm-ui/backend/db_meta/api/cluster/nosqlcomm/scale_proxy.py b/dbm-ui/backend/db_meta/api/cluster/nosqlcomm/scale_proxy.py index 225178326c..5b110b1237 100644 --- a/dbm-ui/backend/db_meta/api/cluster/nosqlcomm/scale_proxy.py +++ b/dbm-ui/backend/db_meta/api/cluster/nosqlcomm/scale_proxy.py @@ -18,7 +18,13 @@ from backend.db_meta.api import common from backend.db_meta.api.cluster.nosqlcomm.create_cluster import update_cluster_type -from backend.db_meta.enums import AccessLayer, ClusterMachineAccessTypeDefine, InstanceInnerRole +from backend.db_meta.enums import ( + AccessLayer, + ClusterEntryRole, + ClusterMachineAccessTypeDefine, + ClusterType, + InstanceInnerRole, +) from backend.db_meta.models import Cluster, ProxyInstance from backend.flow.utils.cc_manage import CcManage from backend.flow.utils.redis.redis_module_operate import RedisCCTopoOperator @@ -64,8 +70,19 @@ def add_proxies(cluster: Cluster, proxies: List[Dict]): cluster.proxyinstance_set.add(*proxy_objs) logger.info("cluster {} add proxyinstance {}".format(cluster.immute_domain, proxy_objs)) - # 修改表 db_meta_proxyinstance_bind_entry + # 修改表 db_meta_proxyinstance_bind_entry # 这里要去掉node 域名。 for cluster_entry_obj in cluster.clusterentry_set.all(): + if cluster.cluster_type in [ + ClusterType.TendisPredixyRedisCluster, + ClusterType.TendisPredixyTendisplusCluster, + ]: + if cluster_entry_obj.role == ClusterEntryRole.NODE_ENTRY: + logger.info( + "cluster {} entry {} ignore add proxyinstance {}".format( + cluster.immute_domain, cluster_entry_obj.entry, proxy_objs + ) + ) + continue cluster_entry_obj.proxyinstance_set.add(*proxy_objs) logger.info( "cluster {} entry {} add proxyinstance {}".format( diff --git a/dbm-ui/backend/db_meta/api/cluster/nosqlcomm/scale_tendis.py b/dbm-ui/backend/db_meta/api/cluster/nosqlcomm/scale_tendis.py index 837628f2d3..3dbe2a1f64 100644 --- a/dbm-ui/backend/db_meta/api/cluster/nosqlcomm/scale_tendis.py +++ b/dbm-ui/backend/db_meta/api/cluster/nosqlcomm/scale_tendis.py @@ -112,6 +112,11 @@ def redo_slaves(cluster: Cluster, tendisss: List[Dict], created_by: str = ""): # 修改表 db_meta_storageinstance_cluster cluster.storageinstance_set.add(*receiver_objs) + # 修改表 db_meta_storageinstance_proxyinstance + proxy_objs = cluster.proxyinstance_set.all() + for rece_obj in receiver_objs: + rece_obj.proxyinstance_set.add(*proxy_objs) + # 更新 nodes. 域名对应的 cluster_entry信息 cluster_entry = cluster.clusterentry_set.filter(role=ClusterEntryRole.NODE_ENTRY.value).first() if cluster_entry and cluster_entry.entry.startswith("nodes."): @@ -152,6 +157,13 @@ def make_sync(cluster: Cluster, tendisss: List[Dict]): cluster.storageinstance_set.add(*receiver_objs) logger.info("cluster {} add storageinstance {}".format(cluster.immute_domain, receiver_objs)) + # db_meta_proxyinstance_storageinstance ; 页面展示用 + proxy_objs = cluster.proxyinstance_set.all() + for receiver in receiver_objs: + receiver.proxyinstance_set.add(*proxy_objs) + for receiver in new_recevers_objs: + receiver.proxyinstance_set.add(*proxy_objs) + # 修改表 db_meta_storageinstance_cluster ; (仅仅为了DBHA能获取密码！) cluster.storageinstance_set.add(*new_recevers_objs) logger.info("cluster {} add storageinstance {}".format(cluster.immute_domain, new_recevers_objs)) diff --git a/dbm-ui/backend/db_monitor/management/commands/update_collect.py b/dbm-ui/backend/db_monitor/management/commands/update_collect.py index ab44135200..dd890267b6 100644 --- a/dbm-ui/backend/db_monitor/management/commands/update_collect.py +++ b/dbm-ui/backend/db_monitor/management/commands/update_collect.py @@ -12,6 +12,7 @@ from django.core.management.base import BaseCommand +from backend.configuration.constants import DBType from backend.db_monitor.models import CollectInstance logger = logging.getLogger("root") @@ -25,7 +26,7 @@ def add_arguments(self, parser): parser.add_argument( "-d", "--dbtype", - choices=["mysql", "redis", "es", "hdfs", "kafka", "pulsar", "influxdb", "all"], + choices=[*DBType.get_values(), "all"], type=str, help="db类型", ) diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL Proxy\347\253\257\345\217\243\346\216\242\346\265\213\345\274\202\345\270\270.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL Proxy\347\253\257\345\217\243\346\216\242\346\265\213\345\274\202\345\270\270.json" index 6321869ed8..75e3cb2781 100644 --- "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL Proxy\347\253\257\345\217\243\346\216\242\346\265\213\345\274\202\345\270\270.json" +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL Proxy\347\253\257\345\217\243\346\216\242\346\265\213\345\274\202\345\270\270.json" @@ -210,7 +210,7 @@ "app": "", "path": "", "priority": 0, - "priority_group_key": "", + "priority_group_key": "422d171193dd6c7b", "edit_allowed": true, "metric_type": "time_series", "data_source_type": "监控采集指标", @@ -218,8 +218,8 @@ }, "is_enabled": true, "monitor_indicator": "AVG(mysqlproxy_up)", - "version": 23, + "version": 24, "alert_source": "time_series", "custom_conditions": [], - "export_at": "2024-11-26T16:26:58+08:00" + "export_at": "2024-12-06T15:52:08+08:00" } \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL Slave \345\220\214\346\255\245\345\274\202\345\270\270.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL Slave \345\220\214\346\255\245\345\274\202\345\270\270.json" index 41bafe48a2..0588bf11a5 100644 --- "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL Slave \345\220\214\346\255\245\345\274\202\345\270\270.json" +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL Slave \345\220\214\346\255\245\345\274\202\345\270\270.json" @@ -98,8 +98,8 @@ "user_groups": [], "user_type": "main", "signal": [ - "abnormal", - "no_data" + "no_data", + "abnormal" ], "options": { "end_time": "23:59:59", @@ -208,7 +208,7 @@ "app": "", "path": "", "priority": 0, - "priority_group_key": "", + "priority_group_key": "894a024869ba958d", "edit_allowed": true, "metric_type": "event", "data_source_type": "自定义事件", @@ -216,8 +216,8 @@ }, "is_enabled": true, "monitor_indicator": "COUNT(slave-status-dbm_report_channel)", - "version": 24, + "version": 25, "alert_source": "event", "custom_conditions": [], - "export_at": "2024-11-26T16:26:58+08:00" + "export_at": "2024-12-06T15:52:08+08:00" } \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \344\270\273\346\234\272\347\243\201\347\233\230\347\251\272\351\227\264\344\275\277\347\224\250\347\216\207.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \344\270\273\346\234\272\347\243\201\347\233\230\347\251\272\351\227\264\344\275\277\347\224\250\347\216\207.json" index 30f91000aa..7822e5a52e 100644 --- "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \344\270\273\346\234\272\347\243\201\347\233\230\347\251\272\351\227\264\344\275\277\347\224\250\347\216\207.json" +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \344\270\273\346\234\272\347\243\201\347\233\230\347\251\272\351\227\264\344\275\277\347\224\250\347\216\207.json" @@ -33,7 +33,7 @@ "functions": [], "result_table_id": "dbm_system.disk", "agg_method": "AVG", - "agg_interval": 60, + "agg_interval": 300, "agg_dimension": [ "appid", "app", @@ -47,11 +47,12 @@ { "key": "db_type", "value": [ - "mysql", "tendbcluster" + "mysql", + "tendbcluster" ], "method": "eq", "condition": "and", - "dimension_name": "db_type" + "dimension_name": "dbm db type" } ], "metric_field": "in_use", @@ -68,6 +69,10 @@ { "method": "gte", "threshold": 95 + }, + { + "method": "lt", + "threshold": 98 } ] ], @@ -95,7 +100,7 @@ "level": 1, "expression": "", "trigger_config": { - "count": 1, + "count": 2, "uptime": { "calendars": [], "time_ranges": [ @@ -105,10 +110,10 @@ } ] }, - "check_window": 5 + "check_window": 3 }, "recovery_config": { - "check_window": 5, + "check_window": 2, "status_setter": "recovery" }, "connector": "and" @@ -117,7 +122,7 @@ "level": 2, "expression": "", "trigger_config": { - "count": 1, + "count": 2, "uptime": { "calendars": [], "time_ranges": [ @@ -127,10 +132,10 @@ } ] }, - "check_window": 5 + "check_window": 3 }, "recovery_config": { - "check_window": 5, + "check_window": 2, "status_setter": "recovery" }, "connector": "and" @@ -140,6 +145,7 @@ "notice": { "config_id": 40811, "user_groups": [], + "user_type": "main", "signal": [ "no_data", "abnormal" @@ -204,44 +210,7 @@ "timedelta": 60, "is_enabled": true, "converge_func": "collect", - "need_biz_converge": false, - "sub_converge_config": { - "timedelta": 60, - "count": 2, - "condition": [ - { - "dimension": "bk_biz_id", - "value": [ - "self" - ] - }, - { - "dimension": "notice_receiver", - "value": [ - "self" - ] - }, - { - "dimension": "notice_way", - "value": [ - "self" - ] - }, - { - "dimension": "alert_level", - "value": [ - "self" - ] - }, - { - "dimension": "signal", - "value": [ - "self" - ] - } - ], - "converge_func": "collect_alarm" - } + "need_biz_converge": false }, "chart_image_enabled": true, "exclude_notice_ways": { @@ -260,7 +229,7 @@ "relate_type": "NOTICE", "config": { "need_poll": true, - "notify_interval": 7200, + "notify_interval": 21600, "interval_notify_mode": "standard", "template": [ { @@ -289,14 +258,16 @@ "app": "", "path": "", "priority": 0, - "priority_group_key": "ba5f279a6c53b7c2", + "priority_group_key": "", + "edit_allowed": true, "metric_type": "time_series", "data_source_type": "监控采集指标", "is_enabled": true }, "is_enabled": true, "monitor_indicator": "AVG(磁盘空间使用率)", - "version": 24, + "version": 25, "alert_source": "time_series", - "custom_conditions": [] + "custom_conditions": [], + "export_at": "2024-12-06T15:52:08+08:00" } \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \344\270\273\346\234\272\347\243\201\347\233\230\347\251\272\351\227\264\344\275\277\347\224\250\347\216\207\346\236\201\351\231\220.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \344\270\273\346\234\272\347\243\201\347\233\230\347\251\272\351\227\264\344\275\277\347\224\250\347\216\207\346\236\201\351\231\220.json" new file mode 100644 index 0000000000..105c279529 --- /dev/null +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \344\270\273\346\234\272\347\243\201\347\233\230\347\251\272\351\227\264\344\275\277\347\224\250\347\216\207\346\236\201\351\231\220.json" @@ -0,0 +1,234 @@ +{ + "bk_biz_id": 0, + "name": "MySQL 主机磁盘空间使用率极限", + "db_type": "mysql", + "details": { + "bk_biz_id": "", + "name": "MySQL 主机磁盘空间使用率极限", + "source": "dbm", + "scenario": "os", + "type": "monitor", + "items": [ + { + "name": "MAX(磁盘空间使用率)", + "no_data_config": { + "level": 2, + "continuous": 10, + "is_enabled": false, + "agg_dimension": [ + "bk_target_ip", + "bk_target_cloud_id" + ] + }, + "target": [], + "expression": "a", + "functions": [], + "origin_sql": "", + "query_configs": [ + { + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "alias": "a", + "metric_id": "bk_monitor.dbm_system.disk.in_use", + "functions": [], + "result_table_id": "dbm_system.disk", + "agg_method": "MAX", + "agg_interval": 60, + "agg_dimension": [ + "appid", + "app", + "instance_role", + "cluster_domain", + "bk_target_ip", + "bk_target_cloud_id", + "mount_point" + ], + "agg_condition": [ + { + "key": "db_type", + "value": [ + "mysql", + "tendbcluster" + ], + "method": "eq", + "condition": "and", + "dimension_name": "dbm db type" + } + ], + "metric_field": "in_use", + "unit": "percent", + "name": "磁盘空间使用率" + } + ], + "algorithms": [ + { + "type": "Threshold", + "level": 1, + "config": [ + [ + { + "method": "gte", + "threshold": 98 + } + ] + ], + "unit_prefix": "%" + } + ], + "metric_type": "time_series" + } + ], + "detects": [ + { + "level": 1, + "expression": "", + "trigger_config": { + "count": 1, + "uptime": { + "calendars": [], + "time_ranges": [ + { + "end": "23:59", + "start": "00:00" + } + ] + }, + "check_window": 5 + }, + "recovery_config": { + "check_window": 5, + "status_setter": "recovery" + }, + "connector": "and" + } + ], + "actions": [], + "notice": { + "config_id": 48116, + "user_groups": [], + "user_type": "main", + "signal": [ + "abnormal", + "no_data" + ], + "options": { + "end_time": "23:59:59", + "start_time": "00:00:00", + "assign_mode": [ + "by_rule" + ], + "upgrade_config": { + "is_enabled": false, + "user_groups": [], + "upgrade_interval": 1440 + }, + "converge_config": { + "count": 1, + "condition": [ + { + "dimension": "strategy_id", + "value": [ + "self" + ] + }, + { + "dimension": "dimensions", + "value": [ + "self" + ] + }, + { + "dimension": "alert_level", + "value": [ + "self" + ] + }, + { + "dimension": "signal", + "value": [ + "self" + ] + }, + { + "dimension": "bk_biz_id", + "value": [ + "self" + ] + }, + { + "dimension": "notice_receiver", + "value": [ + "self" + ] + }, + { + "dimension": "notice_way", + "value": [ + "self" + ] + } + ], + "timedelta": 60, + "is_enabled": true, + "converge_func": "collect", + "need_biz_converge": false + }, + "chart_image_enabled": true, + "exclude_notice_ways": { + "ack": [], + "closed": [], + "recovered": [] + }, + "noise_reduce_config": { + "unit": "percent", + "count": 10, + "timedelta": 5, + "dimensions": [], + "is_enabled": false + } + }, + "relate_type": "NOTICE", + "config": { + "need_poll": true, + "notify_interval": 7200, + "interval_notify_mode": "standard", + "template": [ + { + "signal": "abnormal", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n通知人:{{alarm.receivers}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "recovered", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n通知人:{{alarm.receivers}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "closed", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n通知人:{{alarm.receivers}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + } + ] + } + }, + "labels": [ + "DBM", + "DBM_MYSQL", + "MySQL" + ], + "app": "", + "path": "", + "priority": 0, + "priority_group_key": "", + "edit_allowed": true, + "metric_type": "time_series", + "data_source_type": "监控采集指标", + "is_enabled": true + }, + "is_enabled": true, + "monitor_indicator": "MAX(磁盘空间使用率)", + "version": 1, + "alert_source": "time_series", + "custom_conditions": [], + "export_at": "2024-12-06T15:52:08+08:00" +} \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \346\205\242\346\237\245\350\257\242\346\225\260\351\207\217.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \346\205\242\346\237\245\350\257\242\346\225\260\351\207\217.json" index 2c556b6b90..e963abba23 100644 --- "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \346\205\242\346\237\245\350\257\242\346\225\260\351\207\217.json" +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \346\205\242\346\237\245\350\257\242\346\225\260\351\207\217.json" @@ -293,7 +293,7 @@ "app": "", "path": "", "priority": 0, - "priority_group_key": "", + "priority_group_key": "a47ba1274c93c964", "edit_allowed": true, "metric_type": "time_series", "data_source_type": "监控采集指标", @@ -301,8 +301,8 @@ }, "is_enabled": true, "monitor_indicator": "AVG(mysql_global_status_slow_queries)", - "version": 1, + "version": 2, "alert_source": "time_series", "custom_conditions": [], - "export_at": "2024-11-26T16:26:58+08:00" + "export_at": "2024-12-06T15:52:08+08:00" } \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \347\253\257\345\217\243\346\216\242\346\265\213\345\274\202\345\270\270.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \347\253\257\345\217\243\346\216\242\346\265\213\345\274\202\345\270\270.json" index b4a937206d..438e640061 100644 --- "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \347\253\257\345\217\243\346\216\242\346\265\213\345\274\202\345\270\270.json" +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \347\253\257\345\217\243\346\216\242\346\265\213\345\274\202\345\270\270.json" @@ -100,8 +100,8 @@ "user_groups": [], "user_type": "main", "signal": [ - "no_data", - "abnormal" + "abnormal", + "no_data" ], "options": { "end_time": "23:59:59", @@ -211,7 +211,7 @@ "app": "", "path": "", "priority": 0, - "priority_group_key": "", + "priority_group_key": "f8605997f317d4ba", "edit_allowed": true, "metric_type": "time_series", "data_source_type": "监控采集指标", @@ -219,8 +219,8 @@ }, "is_enabled": true, "monitor_indicator": "AVG(mysql_up)", - "version": 23, + "version": 24, "alert_source": "time_series", "custom_conditions": [], - "export_at": "2024-11-26T16:26:58+08:00" + "export_at": "2024-12-06T15:52:08+08:00" } \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \350\277\233\347\250\213\345\217\221\347\224\237\351\207\215\345\220\257.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \350\277\233\347\250\213\345\217\221\347\224\237\351\207\215\345\220\257.json" new file mode 100644 index 0000000000..6bee052780 --- /dev/null +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/MySQL \350\277\233\347\250\213\345\217\221\347\224\237\351\207\215\345\220\257.json" @@ -0,0 +1,222 @@ +{ + "bk_biz_id": 0, + "name": "MySQL 进程发生重启", + "db_type": "mysql", + "details": { + "bk_biz_id": "", + "name": "MySQL 进程发生重启", + "source": "dbm", + "scenario": "component", + "type": "monitor", + "items": [ + { + "name": "AVG(mysql_global_status_uptime)", + "no_data_config": { + "level": 2, + "continuous": 10, + "is_enabled": false, + "agg_dimension": [ + "bk_target_service_instance_id" + ] + }, + "target": [], + "expression": "a", + "functions": [], + "origin_sql": "", + "query_configs": [ + { + "data_source_label": "bk_monitor", + "data_type_label": "time_series", + "alias": "a", + "metric_id": "bk_monitor.exporter_dbm_mysqld_exporter.__default__.mysql_global_status_uptime", + "functions": [], + "result_table_id": "exporter_dbm_mysqld_exporter.__default__", + "data_label": "exporter_dbm_mysqld_exporter", + "agg_method": "AVG", + "agg_interval": 60, + "agg_dimension": [ + "bk_target_service_instance_id", + "app", + "cluster_domain", + "instance_role", + "instance", + "appid" + ], + "agg_condition": [], + "metric_field": "mysql_global_status_uptime", + "unit": "", + "name": "mysql_global_status_uptime" + } + ], + "algorithms": [ + { + "type": "Threshold", + "level": 2, + "config": [ + [ + { + "method": "lt", + "threshold": 60 + } + ] + ], + "unit_prefix": "" + } + ], + "metric_type": "time_series" + } + ], + "detects": [ + { + "level": 2, + "expression": "", + "trigger_config": { + "count": 1, + "uptime": { + "calendars": [], + "time_ranges": [ + { + "end": "23:59", + "start": "00:00" + } + ] + }, + "check_window": 5 + }, + "recovery_config": { + "check_window": 5, + "status_setter": "recovery" + }, + "connector": "and" + } + ], + "actions": [], + "notice": { + "config_id": 40762, + "user_groups": [], + "user_type": "main", + "signal": [ + "no_data", + "abnormal" + ], + "options": { + "end_time": "23:59:59", + "start_time": "00:00:00", + "assign_mode": [ + "by_rule" + ], + "upgrade_config": { + "is_enabled": false, + "user_groups": [], + "upgrade_interval": 1440 + }, + "converge_config": { + "count": 1, + "condition": [ + { + "dimension": "strategy_id", + "value": [ + "self" + ] + }, + { + "dimension": "dimensions", + "value": [ + "self" + ] + }, + { + "dimension": "alert_level", + "value": [ + "self" + ] + }, + { + "dimension": "signal", + "value": [ + "self" + ] + }, + { + "dimension": "bk_biz_id", + "value": [ + "self" + ] + }, + { + "dimension": "notice_receiver", + "value": [ + "self" + ] + }, + { + "dimension": "notice_way", + "value": [ + "self" + ] + } + ], + "timedelta": 60, + "is_enabled": true, + "converge_func": "collect", + "need_biz_converge": false + }, + "chart_image_enabled": true, + "exclude_notice_ways": { + "ack": [], + "closed": [], + "recovered": [] + }, + "noise_reduce_config": { + "unit": "percent", + "count": 10, + "timedelta": 5, + "dimensions": [], + "is_enabled": false + } + }, + "relate_type": "NOTICE", + "config": { + "need_poll": true, + "notify_interval": 7200, + "interval_notify_mode": "standard", + "template": [ + { + "signal": "abnormal", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n通知人:{{alarm.receivers}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "recovered", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n通知人:{{alarm.receivers}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "closed", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n通知人:{{alarm.receivers}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + } + ] + } + }, + "labels": [ + "DBM", + "DBM_MYSQL", + "MySQL" + ], + "app": "", + "path": "", + "priority": 0, + "priority_group_key": "", + "edit_allowed": true, + "metric_type": "time_series", + "data_source_type": "监控采集指标", + "is_enabled": true + }, + "is_enabled": true, + "monitor_indicator": "AVG(mysql_global_status_uptime)", + "version": 1, + "alert_source": "time_series", + "custom_conditions": [], + "export_at": "2024-12-06T15:52:08+08:00" +} \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/Mysql monitor \344\270\273\344\273\216\345\277\203\350\267\263\345\274\202\345\270\270.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/Mysql monitor \344\270\273\344\273\216\345\277\203\350\267\263\345\274\202\345\270\270.json" new file mode 100644 index 0000000000..5bd06cfedc --- /dev/null +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/Mysql monitor \344\270\273\344\273\216\345\277\203\350\267\263\345\274\202\345\270\270.json" @@ -0,0 +1,255 @@ +{ + "bk_biz_id": 0, + "name": "Mysql monitor 主从心跳异常", + "db_type": "mysql", + "details": { + "bk_biz_id": "", + "name": "Mysql monitor 主从心跳异常", + "source": "dbm", + "scenario": "component", + "type": "monitor", + "items": [ + { + "name": "MAX(master_slave_heartbeat)", + "no_data_config": { + "level": 2, + "continuous": 10, + "is_enabled": false, + "agg_dimension": [] + }, + "target": [], + "expression": "a", + "functions": [], + "origin_sql": "", + "query_configs": [ + { + "data_source_label": "custom", + "data_type_label": "time_series", + "alias": "a", + "metric_id": "custom.bkmonitor_time_series_{metric_data_id}.__default__.master_slave_heartbeat", + "functions": [], + "result_table_id": "bkmonitor_time_series_{metric_data_id}", + "data_label": "dbm_report_channel", + "agg_method": "MAX", + "agg_interval": 300, + "agg_dimension": [ + "appid", + "cluster_domain", + "instance_host", + "instance_port" + ], + "agg_condition": [], + "metric_field": "master_slave_heartbeat", + "unit": "", + "name": "master_slave_heartbeat" + } + ], + "algorithms": [ + { + "type": "Threshold", + "level": 2, + "config": [ + [ + { + "method": "gte", + "threshold": 300 + } + ] + ], + "unit_prefix": "" + } + ], + "metric_type": "time_series" + } + ], + "detects": [ + { + "level": 2, + "expression": "", + "trigger_config": { + "count": 1, + "uptime": { + "calendars": [], + "time_ranges": [ + { + "end": "23:59", + "start": "00:00" + } + ] + }, + "check_window": 5 + }, + "recovery_config": { + "check_window": 5, + "status_setter": "recovery" + }, + "connector": "and" + } + ], + "actions": [], + "notice": { + "config_id": 145174, + "user_groups": [], + "user_type": "main", + "signal": [ + "no_data", + "abnormal" + ], + "options": { + "end_time": "23:59:59", + "start_time": "00:00:00", + "assign_mode": [ + "by_rule" + ], + "upgrade_config": { + "is_enabled": false, + "user_groups": [], + "upgrade_interval": 1440 + }, + "converge_config": { + "count": 1, + "condition": [ + { + "dimension": "strategy_id", + "value": [ + "self" + ] + }, + { + "dimension": "dimensions", + "value": [ + "self" + ] + }, + { + "dimension": "alert_level", + "value": [ + "self" + ] + }, + { + "dimension": "signal", + "value": [ + "self" + ] + }, + { + "dimension": "bk_biz_id", + "value": [ + "self" + ] + }, + { + "dimension": "notice_receiver", + "value": [ + "self" + ] + }, + { + "dimension": "notice_way", + "value": [ + "self" + ] + } + ], + "timedelta": 60, + "is_enabled": true, + "converge_func": "collect", + "need_biz_converge": true, + "sub_converge_config": { + "timedelta": 60, + "count": 2, + "condition": [ + { + "dimension": "bk_biz_id", + "value": [ + "self" + ] + }, + { + "dimension": "notice_receiver", + "value": [ + "self" + ] + }, + { + "dimension": "notice_way", + "value": [ + "self" + ] + }, + { + "dimension": "alert_level", + "value": [ + "self" + ] + }, + { + "dimension": "signal", + "value": [ + "self" + ] + } + ], + "converge_func": "collect_alarm" + } + }, + "chart_image_enabled": true, + "exclude_notice_ways": { + "ack": [], + "closed": [], + "recovered": [] + }, + "noise_reduce_config": { + "unit": "percent", + "count": 10, + "timedelta": 5, + "dimensions": [], + "is_enabled": false + } + }, + "relate_type": "NOTICE", + "config": { + "need_poll": true, + "notify_interval": 21600, + "interval_notify_mode": "standard", + "template": [ + { + "signal": "abnormal", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "recovered", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "closed", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + } + ] + } + }, + "labels": [ + "DBM", + "DBM_MYSQL", + "MySQL" + ], + "app": "", + "path": "", + "priority": 0, + "priority_group_key": "", + "edit_allowed": true, + "metric_type": "time_series", + "data_source_type": "自定义指标", + "is_enabled": true + }, + "is_enabled": true, + "monitor_indicator": "MAX(master_slave_heartbeat)", + "version": 1, + "alert_source": "time_series", + "custom_conditions": [], + "export_at": "2024-12-06T15:54:01+08:00" +} \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/TendbCluster tdbctl \344\270\255\346\216\247\345\220\214\346\255\245\345\274\202\345\270\270.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/TendbCluster tdbctl \344\270\255\346\216\247\345\220\214\346\255\245\345\274\202\345\270\270.json" new file mode 100644 index 0000000000..0112885d41 --- /dev/null +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/TendbCluster tdbctl \344\270\255\346\216\247\345\220\214\346\255\245\345\274\202\345\270\270.json" @@ -0,0 +1,253 @@ +{ + "bk_biz_id": 0, + "name": "TendbCluster tdbctl 中控同步异常", + "db_type": "mysql", + "details": { + "bk_biz_id": "", + "name": "TendbCluster tdbctl 中控同步异常", + "source": "dbm", + "scenario": "component", + "type": "monitor", + "items": [ + { + "name": "COUNT(ctl-replicate-dbm_report_channel)", + "no_data_config": { + "level": 2, + "continuous": 10, + "is_enabled": false, + "agg_dimension": [] + }, + "target": [], + "expression": "a", + "functions": [], + "origin_sql": "", + "query_configs": [ + { + "data_source_label": "custom", + "data_type_label": "event", + "alias": "a", + "metric_id": "custom.event.bkmonitor_event_{event_data_id}.ctl-replicate", + "functions": [], + "result_table_id": "bkmonitor_event_{event_data_id}", + "agg_method": "COUNT", + "agg_interval": 60, + "agg_dimension": [ + "appid", + "cluster_domain", + "instance_host" + ], + "agg_condition": [], + "custom_event_name": "ctl-replicate", + "query_string": "*", + "name": "ctl-replicate" + } + ], + "algorithms": [ + { + "type": "Threshold", + "level": 1, + "config": [ + [ + { + "method": "gte", + "threshold": 1 + } + ] + ], + "unit_prefix": "" + } + ], + "metric_type": "event" + } + ], + "detects": [ + { + "level": 1, + "expression": "", + "trigger_config": { + "count": 1, + "uptime": { + "calendars": [], + "time_ranges": [ + { + "end": "23:59", + "start": "10:10" + } + ] + }, + "check_window": 5 + }, + "recovery_config": { + "check_window": 5, + "status_setter": "recovery" + }, + "connector": "and" + } + ], + "actions": [], + "notice": { + "config_id": 145190, + "user_groups": [], + "user_type": "main", + "signal": [ + "abnormal", + "no_data" + ], + "options": { + "end_time": "23:59:59", + "start_time": "00:00:00", + "assign_mode": [ + "by_rule" + ], + "upgrade_config": { + "is_enabled": false, + "user_groups": [], + "upgrade_interval": 1440 + }, + "converge_config": { + "count": 1, + "condition": [ + { + "dimension": "strategy_id", + "value": [ + "self" + ] + }, + { + "dimension": "dimensions", + "value": [ + "self" + ] + }, + { + "dimension": "alert_level", + "value": [ + "self" + ] + }, + { + "dimension": "signal", + "value": [ + "self" + ] + }, + { + "dimension": "bk_biz_id", + "value": [ + "self" + ] + }, + { + "dimension": "notice_receiver", + "value": [ + "self" + ] + }, + { + "dimension": "notice_way", + "value": [ + "self" + ] + } + ], + "timedelta": 60, + "is_enabled": true, + "converge_func": "collect", + "need_biz_converge": true, + "sub_converge_config": { + "timedelta": 60, + "count": 2, + "condition": [ + { + "dimension": "bk_biz_id", + "value": [ + "self" + ] + }, + { + "dimension": "notice_receiver", + "value": [ + "self" + ] + }, + { + "dimension": "notice_way", + "value": [ + "self" + ] + }, + { + "dimension": "alert_level", + "value": [ + "self" + ] + }, + { + "dimension": "signal", + "value": [ + "self" + ] + } + ], + "converge_func": "collect_alarm" + } + }, + "chart_image_enabled": true, + "exclude_notice_ways": { + "ack": [], + "closed": [], + "recovered": [] + }, + "noise_reduce_config": { + "unit": "percent", + "count": 10, + "timedelta": 5, + "dimensions": [], + "is_enabled": false + } + }, + "relate_type": "NOTICE", + "config": { + "need_poll": true, + "notify_interval": 21600, + "interval_notify_mode": "standard", + "template": [ + { + "signal": "abnormal", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "recovered", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "closed", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + } + ] + } + }, + "labels": [ + "DBM", + "DBM_MYSQL", + "MySQL" + ], + "app": "", + "path": "", + "priority": 0, + "priority_group_key": "", + "edit_allowed": true, + "metric_type": "event", + "data_source_type": "自定义事件", + "is_enabled": true + }, + "is_enabled": true, + "monitor_indicator": "COUNT(ctl-replicate-dbm_report_channel)", + "version": 1, + "alert_source": "event", + "custom_conditions": [], + "export_at": "2024-12-06T15:54:01+08:00" +} \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/mysql/Tendbcluster tdbctl \344\270\255\346\216\247 get primary \345\274\202\345\270\270.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/Tendbcluster tdbctl \344\270\255\346\216\247 get primary \345\274\202\345\270\270.json" new file mode 100644 index 0000000000..7516db1378 --- /dev/null +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/mysql/Tendbcluster tdbctl \344\270\255\346\216\247 get primary \345\274\202\345\270\270.json" @@ -0,0 +1,254 @@ +{ + "bk_biz_id": 0, + "name": "Tendbcluster tdbctl 中控 get primary 异常", + "db_type": "mysql", + "details": { + "bk_biz_id": "", + "name": "Tendbcluster tdbctl 中控 get primary 异常", + "source": "dbm", + "scenario": "component", + "type": "monitor", + "items": [ + { + "name": "COUNT(get-ctl-primary-dbm_report_channel)", + "no_data_config": { + "level": 2, + "continuous": 10, + "is_enabled": false, + "agg_dimension": [] + }, + "target": [], + "expression": "a", + "functions": [], + "origin_sql": "", + "query_configs": [ + { + "data_source_label": "custom", + "data_type_label": "event", + "alias": "a", + "metric_id": "custom.event.bkmonitor_event_{event_data_id}.get-ctl-primary", + "functions": [], + "result_table_id": "bkmonitor_event_{event_data_id}", + "agg_method": "COUNT", + "agg_interval": 60, + "agg_dimension": [ + "appid", + "cluster_domain", + "instance_host", + "instance_port" + ], + "agg_condition": [], + "custom_event_name": "get-ctl-primary", + "query_string": "*", + "name": "get-ctl-primary" + } + ], + "algorithms": [ + { + "type": "Threshold", + "level": 1, + "config": [ + [ + { + "method": "gte", + "threshold": 1 + } + ] + ], + "unit_prefix": "" + } + ], + "metric_type": "event" + } + ], + "detects": [ + { + "level": 1, + "expression": "", + "trigger_config": { + "count": 1, + "uptime": { + "calendars": [], + "time_ranges": [ + { + "end": "23:59", + "start": "11:00" + } + ] + }, + "check_window": 5 + }, + "recovery_config": { + "check_window": 5, + "status_setter": "recovery" + }, + "connector": "and" + } + ], + "actions": [], + "notice": { + "config_id": 145173, + "user_groups": [], + "user_type": "main", + "signal": [ + "abnormal", + "no_data" + ], + "options": { + "end_time": "23:59:59", + "start_time": "00:00:00", + "assign_mode": [ + "by_rule" + ], + "upgrade_config": { + "is_enabled": false, + "user_groups": [], + "upgrade_interval": 1440 + }, + "converge_config": { + "count": 1, + "condition": [ + { + "dimension": "strategy_id", + "value": [ + "self" + ] + }, + { + "dimension": "dimensions", + "value": [ + "self" + ] + }, + { + "dimension": "alert_level", + "value": [ + "self" + ] + }, + { + "dimension": "signal", + "value": [ + "self" + ] + }, + { + "dimension": "bk_biz_id", + "value": [ + "self" + ] + }, + { + "dimension": "notice_receiver", + "value": [ + "self" + ] + }, + { + "dimension": "notice_way", + "value": [ + "self" + ] + } + ], + "timedelta": 60, + "is_enabled": true, + "converge_func": "collect", + "need_biz_converge": true, + "sub_converge_config": { + "timedelta": 60, + "count": 2, + "condition": [ + { + "dimension": "bk_biz_id", + "value": [ + "self" + ] + }, + { + "dimension": "notice_receiver", + "value": [ + "self" + ] + }, + { + "dimension": "notice_way", + "value": [ + "self" + ] + }, + { + "dimension": "alert_level", + "value": [ + "self" + ] + }, + { + "dimension": "signal", + "value": [ + "self" + ] + } + ], + "converge_func": "collect_alarm" + } + }, + "chart_image_enabled": true, + "exclude_notice_ways": { + "ack": [], + "closed": [], + "recovered": [] + }, + "noise_reduce_config": { + "unit": "percent", + "count": 10, + "timedelta": 5, + "dimensions": [], + "is_enabled": false + } + }, + "relate_type": "NOTICE", + "config": { + "need_poll": true, + "notify_interval": 21600, + "interval_notify_mode": "standard", + "template": [ + { + "signal": "abnormal", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "recovered", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + }, + { + "signal": "closed", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", + "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" + } + ] + } + }, + "labels": [ + "DBM", + "DBM_MYSQL", + "MySQL" + ], + "app": "", + "path": "", + "priority": 0, + "priority_group_key": "", + "edit_allowed": true, + "metric_type": "event", + "data_source_type": "自定义事件", + "is_enabled": true + }, + "is_enabled": true, + "monitor_indicator": "COUNT(get-ctl-primary-dbm_report_channel)", + "version": 1, + "alert_source": "event", + "custom_conditions": [], + "export_at": "2024-12-06T15:54:01+08:00" +} \ No newline at end of file diff --git "a/dbm-ui/backend/db_monitor/tpls/alarm/sqlserver/Sqlserver-\350\277\233\347\250\213\345\255\230\346\264\273.json" "b/dbm-ui/backend/db_monitor/tpls/alarm/sqlserver/Sqlserver-\350\277\233\347\250\213\345\255\230\346\264\273.json" index 65333359c8..4a47467586 100644 --- "a/dbm-ui/backend/db_monitor/tpls/alarm/sqlserver/Sqlserver-\350\277\233\347\250\213\345\255\230\346\264\273.json" +++ "b/dbm-ui/backend/db_monitor/tpls/alarm/sqlserver/Sqlserver-\350\277\233\347\250\213\345\255\230\346\264\273.json" @@ -14,8 +14,10 @@ "no_data_config": { "level": 2, "continuous": 10, - "is_enabled": false, - "agg_dimension": [] + "is_enabled": true, + "agg_dimension": [ + "bk_target_service_instance_id" + ] }, "target": [], "expression": "a", @@ -88,14 +90,81 @@ "connector": "and" } ], - "actions": [], + "actions": [ + { + "id": 168888, + "config_id": 137317, + "user_groups": [], + "user_type": "main", + "signal": [ + "abnormal" + ], + "options": { + "end_time": "23:59:59", + "start_time": "00:00:00", + "converge_config": { + "count": 1, + "condition": [ + { + "value": [ + "self" + ], + "dimension": "action_info" + } + ], + "timedelta": 60, + "is_enabled": false, + "converge_func": "skip_when_success", + "need_biz_converge": true + } + }, + "relate_type": "ACTION", + "config": { + "id": 137317, + "name": "dbm_autofix_http_callback", + "desc": "", + "bk_biz_id": "5005578", + "plugin_id": "2", + "execute_config": { + "template_detail": { + "need_poll": false, + "notify_interval": 60, + "interval_notify_mode": "standard", + "method": "POST", + "url": "", + "headers": [], + "authorize": { + "auth_type": "bearer_token", + "auth_config": { + "token": "" + } + }, + "body": { + "data_type": "raw", + "params": [], + "content": "{\"callback_message\": {{alarm.callback_message}},\"appointees\": \"{{alarm.appointees}}\"}", + "content_type": "json" + }, + "query_params": [], + "failed_retry": { + "is_enabled": true, + "timeout": 10, + "max_retry_times": 2, + "retry_interval": 2 + } + }, + "timeout": 600 + } + } + } + ], "notice": { - "config_id": 47942, + "config_id": 118363, "user_groups": [], "user_type": "main", "signal": [ - "no_data", - "abnormal" + "abnormal", + "no_data" ], "options": { "end_time": "23:59:59", @@ -218,17 +287,17 @@ "template": [ { "signal": "abnormal", - "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" }, { "signal": "recovered", - "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" }, { "signal": "closed", - "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n{{content.related_info}}", + "message_tmpl": "{{content.level}}\n{{content.begin_time}}\n{{content.time}}\n{{content.duration}}\n{{content.target_type}}\n{{content.data_source}}\n{{content.content}}\n{{content.current_value}}\n{{content.biz}}\n{{content.target}}\n{{content.dimension}}\n{{content.detail}}\n{{content.assign_detail}}\n通知人:{{alarm.receivers}}\n{{content.related_info}}", "title_tmpl": "{{business.bk_biz_name}} - {{alarm.name}}{{alarm.display_type}}" } ] @@ -250,8 +319,8 @@ }, "is_enabled": true, "monitor_indicator": "MAX(mssql_serveice_available)", - "version": 2, + "version": 3, "alert_source": "time_series", "custom_conditions": [], - "export_at": "2024-04-02T16:01:58+08:00" + "export_at": "2024-11-29T16:28:30+08:00" } \ No newline at end of file diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/db_monitor.py b/dbm-ui/backend/db_periodic_task/local_tasks/db_monitor.py index 38bdbcefa2..d209197a34 100644 --- a/dbm-ui/backend/db_periodic_task/local_tasks/db_monitor.py +++ b/dbm-ui/backend/db_periodic_task/local_tasks/db_monitor.py @@ -28,14 +28,14 @@ logger = logging.getLogger("celery") -@register_periodic_task(run_every=crontab(hour="*/6", minute="0")) +@register_periodic_task(run_every=crontab(hour="0", minute="0")) def update_local_notice_group(): """同步告警组""" dba_ids = DBAdministrator.objects.values_list("id", flat=True) count = len(dba_ids) # 同步 DBA 内置告警组 for index, dba_id in enumerate(dba_ids): - countdown = calculate_countdown(count=count, index=index, duration=6 * TimeUnit.HOUR) + countdown = calculate_countdown(count=count, index=index, duration=8 * TimeUnit.HOUR) logger.info("dba_id({}) update notice group will be run after {} seconds.".format(dba_id, countdown)) with start_new_span(update_dba_notice_group): update_dba_notice_group.apply_async(kwargs={"dba_id": dba_id}, countdown=countdown) @@ -81,7 +81,7 @@ def sync_plat_monitor_policy(action_id=None, db_type=None, force=False): MonitorPolicy.sync_plat_monitor_policy(action_id=action_id, db_type=db_type, force=force) -@register_periodic_task(run_every=crontab(minute=0, hour="*/1")) +@register_periodic_task(run_every=crontab(minute=0, hour="*/2")) def sync_plat_dispatch_policy(): """同步平台分派通知策略 按照app_id->db_type来拆分策略： @@ -94,7 +94,7 @@ def sync_plat_dispatch_policy(): count = len(biz_ids) # 同步平台/业务分派策略 for index, bk_biz_id in enumerate(biz_ids): - countdown = calculate_countdown(count=count, index=index, duration=TimeUnit.HOUR) + countdown = calculate_countdown(count=count, index=index, duration=2 * TimeUnit.HOUR) logger.info("biz({}) sync dispatch policy will be run after {} seconds.".format(bk_biz_id, countdown)) with start_new_span(sync_biz_dispatch_policy): sync_biz_dispatch_policy.apply_async(kwargs={"bk_biz_id": bk_biz_id}, countdown=countdown) diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/__init__.py b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/check_app_setting_data.py b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/check_app_setting_data.py new file mode 100644 index 0000000000..380c1ea2db --- /dev/null +++ b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/check_app_setting_data.py @@ -0,0 +1,267 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from backend.db_meta.enums import ClusterPhase, ClusterType, InstanceInnerRole, InstanceRole, InstanceStatus +from backend.db_meta.models import Cluster, StorageInstance +from backend.db_meta.models.storage_set_dtl import SqlserverClusterSyncMode +from backend.db_report.models.sqlserver_check_report import ( + SqlserverCheckAppSettingReport, + SqlserverCheckJobSyncReport, + SqlserverCheckLinkServerReport, + SqlserverCheckSysJobStatuReport, + SqlserverCheckUserSyncReport, +) +from backend.flow.utils.sqlserver.sqlserver_bk_config import ( + get_module_infos, + get_sqlserver_alarm_config, + get_sqlserver_backup_config, +) +from backend.flow.utils.sqlserver.sqlserver_db_function import ( + check_ha_config, + check_sys_job_status, + fix_app_setting_data, + get_app_setting_data, + insert_sqlserver_config, +) + + +class CheckAppSettingData(object): + """ + 已dbm元数据为准 + 检查实例的app_setting表的信息是否符合预期，如果存在信息不一致，则需要已某种方式输出告知相关DBA + """ + + def __init__(self): + # 获取所有的online状态的cluster + self.clusters = Cluster.objects.prefetch_related( + "storageinstance_set", + "storageinstance_set__machine", + ).filter(phase=ClusterPhase.ONLINE, cluster_type__in=[ClusterType.SqlserverHA, ClusterType.SqlserverSingle]) + + def check_task(self): + """ + 定义巡检逻辑 + """ + for cluster in self.clusters: + print(cluster.name) + self.check_app_setting_data(cluster) + self.check_job_is_disabled(cluster) + if cluster.cluster_type == ClusterType.SqlserverHA: + master = cluster.storageinstance_set.get(instance_inner_role=InstanceInnerRole.MASTER) + for s in cluster.storageinstance_set.filter( + status=InstanceStatus.RUNNING, instance_inner_role=InstanceInnerRole.SLAVE + ): + self.check_user(master_instance=master, slave_instance=s, cluster=cluster) + self.check_job(master_instance=master, slave_instance=s, cluster=cluster) + self.check_link_server(master_instance=master, slave_instance=s, cluster=cluster) + + @staticmethod + def fix_app_setting_data(cluster: Cluster, instance: StorageInstance, sync_mode: str, master: StorageInstance): + """ + 存在不一致元数据，进行修复 + """ + is_fix = 0 + status, msg = fix_app_setting_data(cluster=cluster, instance=instance, sync_mode=sync_mode, master=master) + if status: + is_fix = 1 + SqlserverCheckAppSettingReport.objects.create( + cluster=cluster.name, + cluster_type=cluster.cluster_type, + instance_host=instance.machine.ip, + instance_port=instance.port, + is_inconsistent=1, + is_fix=is_fix, + status=status, + msg=msg, + ) + return True + + @staticmethod + def add_app_setting_data(cluster: Cluster, instance: StorageInstance): + """ + 插入app_setting数据 + """ + is_fix = 0 + fix_status = False + msg = "fix failed" + # 获取集群字符集配置 + charset = get_module_infos( + bk_biz_id=cluster.bk_biz_id, + db_module_id=cluster.db_module_id, + cluster_type=ClusterType(cluster.cluster_type), + )["charset"] + + # 获取集群的备份配置 + backup_config = get_sqlserver_backup_config( + bk_biz_id=cluster.bk_biz_id, + db_module_id=cluster.db_module_id, + cluster_domain=cluster.immute_domain, + ) + + # 获取集群的个性化配置 + alarm_config = get_sqlserver_alarm_config( + bk_biz_id=cluster.bk_biz_id, + db_module_id=cluster.db_module_id, + cluster_domain=cluster.immute_domain, + ) + + # 配置数据 + try: + fix_status = insert_sqlserver_config( + cluster=cluster, + storages=[instance], + charset=charset, + backup_config=backup_config, + alarm_config=alarm_config, + ) + except Exception: + is_fix = 0 + + if fix_status: + is_fix = 1 + msg = "fix successfully" + + SqlserverCheckAppSettingReport.objects.create( + cluster=cluster.name, + cluster_type=cluster.cluster_type, + instance_host=instance.machine.ip, + instance_port=instance.port, + is_inconsistent=1, + is_fix=is_fix, + status=fix_status, + msg=msg, + ) + return True + + def check_app_setting_data(self, cluster: Cluster): + master = cluster.storageinstance_set.get(instance_role__in=[InstanceRole.ORPHAN, InstanceRole.BACKEND_MASTER]) + if cluster.cluster_type == ClusterType.SqlserverHA: + sync_mode = SqlserverClusterSyncMode.objects.get(cluster_id=cluster.id).sync_mode + else: + sync_mode = "" + + # 按照集群维度查询所有的实例，状态running中的 + for instance in cluster.storageinstance_set.filter(status=InstanceStatus.RUNNING): + data, err = get_app_setting_data(instance=instance, bk_cloud_id=cluster.bk_cloud_id) + if data is None: + # 如果返回是空则,则大概率是访问异常，录入异常信息,跳过这次的校验 + SqlserverCheckAppSettingReport.objects.create( + cluster=cluster.name, + cluster_type=cluster.cluster_type, + instance_host=instance.machine.ip, + instance_port=instance.port, + is_inconsistent=1, + is_fix=0, + status=False, + msg=err, + ) + continue + + if len(data) == 0: + # 则说明没有配置app_setting,需要重新执行 + self.add_app_setting_data(cluster=cluster, instance=instance) + + elif ( + int(data["APP"]) != cluster.bk_biz_id + or int(data["BK_BIZ_ID"]) != cluster.bk_biz_id + or int(data["BK_CLOUD_ID"]) != cluster.bk_cloud_id + or int(data["CLUSTER_ID"]) != cluster.id + or data["CLUSTER_DOMAIN"] != cluster.immute_domain + or int(data["PORT"]) != instance.port + or data["ROLE"] != instance.instance_inner_role + or data["SYNCHRONOUS_MODE"] != sync_mode + or data["MASTER_IP"] != master.machine.ip + or int(data["MASTER_PORT"]) != master.port + ): + # 尝试修复数据 + self.fix_app_setting_data(cluster=cluster, instance=instance, sync_mode=sync_mode, master=master) + + @staticmethod + def check_user(master_instance: StorageInstance, slave_instance: StorageInstance, cluster: Cluster): + """ + 检查主从的用户是否一致 + """ + status, msg = check_ha_config( + master_instance=master_instance, + slave_instance=slave_instance, + bk_cloud_id=cluster.bk_cloud_id, + check_tag="user", + ) + if not status: + SqlserverCheckUserSyncReport.objects.create( + cluster=cluster.name, + cluster_type=cluster.cluster_type, + instance_host=slave_instance.machine.ip, + instance_port=slave_instance.port, + is_user_inconsistent=1, + status=status, + msg=msg, + ) + + @staticmethod + def check_job(master_instance: StorageInstance, slave_instance: StorageInstance, cluster: Cluster): + """ + 检测主从的业务作业是否一致 + """ + status, msg = check_ha_config( + master_instance=master_instance, + slave_instance=slave_instance, + bk_cloud_id=cluster.bk_cloud_id, + check_tag="job", + ) + if not status: + SqlserverCheckJobSyncReport.objects.create( + cluster=cluster.name, + cluster_type=cluster.cluster_type, + instance_host=slave_instance.machine.ip, + instance_port=slave_instance.port, + is_job_inconsistent=1, + status=status, + msg=msg, + ) + + @staticmethod + def check_link_server(master_instance: StorageInstance, slave_instance: StorageInstance, cluster: Cluster): + """ + 检测主从的link_server是否一致 + """ + status, msg = check_ha_config( + master_instance=master_instance, + slave_instance=slave_instance, + bk_cloud_id=cluster.bk_cloud_id, + check_tag="job", + ) + if not status: + SqlserverCheckLinkServerReport.objects.create( + cluster=cluster.name, + cluster_type=cluster.cluster_type, + instance_host=slave_instance.machine.ip, + instance_port=slave_instance.port, + is_link_server_inconsistent=1, + status=status, + msg=msg, + ) + + @staticmethod + def check_job_is_disabled(cluster: Cluster): + # 按照集群维度查询所有的实例，状态running中的 + for instance in cluster.storageinstance_set.filter(status=InstanceStatus.RUNNING): + status, msg = check_sys_job_status(cluster=cluster, instance=instance) + if not status: + # 只有异常才记录 + SqlserverCheckSysJobStatuReport.objects.create( + cluster=cluster.name, + cluster_type=cluster.cluster_type, + instance_host=instance.machine.ip, + instance_port=instance.port, + is_job_disable=1, + status=status, + msg=msg, + ) diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/task.py b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/task.py new file mode 100644 index 0000000000..a11088b228 --- /dev/null +++ b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/task.py @@ -0,0 +1,27 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from celery.schedules import crontab + +from backend.db_periodic_task.local_tasks.register import register_periodic_task +from backend.db_periodic_task.local_tasks.sqlserver.check_app_setting_data import CheckAppSettingData + +logger = logging.getLogger("celery") + + +@register_periodic_task(run_every=crontab(minute=30, hour=6)) +def check_instance_app_setting(): + """ + 检查实例的元数据表(app_setting)是否正常 + 每条凌晨7点执行 + """ + CheckAppSettingData().check_task() diff --git a/dbm-ui/backend/db_report/migrations/0007_sqlservercheckappsettingreport_sqlservercheckjobsyncreport_sqlserverchecklinkserverreport_sqlserverc.py b/dbm-ui/backend/db_report/migrations/0007_sqlservercheckappsettingreport_sqlservercheckjobsyncreport_sqlserverchecklinkserverreport_sqlserverc.py new file mode 100644 index 0000000000..f9c1879921 --- /dev/null +++ b/dbm-ui/backend/db_report/migrations/0007_sqlservercheckappsettingreport_sqlservercheckjobsyncreport_sqlserverchecklinkserverreport_sqlserverc.py @@ -0,0 +1,161 @@ +# Generated by Django 3.2.25 on 2024-11-25 07:00 + +from django.db import migrations, models + +from backend.db_meta.enums import ClusterType + + +class Migration(migrations.Migration): + + dependencies = [ + ("db_report", "0006_auto_20231121_1621"), + ] + + operations = [ + migrations.CreateModel( + name="SqlserverCheckAppSettingReport", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("creator", models.CharField(max_length=64, verbose_name="创建人")), + ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("updater", models.CharField(max_length=64, verbose_name="修改人")), + ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")), + ("bk_biz_id", models.IntegerField(default=0, help_text="业务的 cmdb id")), + ("bk_cloud_id", models.IntegerField(default=0, help_text="云区域 id")), + ("status", models.BooleanField(default=True, help_text="巡检结果状态, 默认正常")), + ("msg", models.TextField(default="", help_text="备注信息")), + ("cluster", models.CharField(default="", max_length=255, verbose_name="集群名称")), + ( + "cluster_type", + models.CharField( + choices=ClusterType.get_choices(), + default="", + max_length=64, + ), + ), + ("instance_host", models.CharField(default="", max_length=255, verbose_name="实例IP")), + ("instance_port", models.IntegerField(default=48322, verbose_name="实例端口")), + ("is_inconsistent", models.IntegerField(default=0, verbose_name="元信息是否不一致")), + ("is_fix", models.IntegerField(default=0, verbose_name="元数据是否自动修复成功")), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="SqlserverCheckJobSyncReport", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("creator", models.CharField(max_length=64, verbose_name="创建人")), + ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("updater", models.CharField(max_length=64, verbose_name="修改人")), + ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")), + ("bk_biz_id", models.IntegerField(default=0, help_text="业务的 cmdb id")), + ("bk_cloud_id", models.IntegerField(default=0, help_text="云区域 id")), + ("status", models.BooleanField(default=True, help_text="巡检结果状态, 默认正常")), + ("msg", models.TextField(default="", help_text="备注信息")), + ("cluster", models.CharField(default="", max_length=255, verbose_name="集群名称")), + ( + "cluster_type", + models.CharField( + choices=ClusterType.get_choices(), + default="", + max_length=64, + ), + ), + ("instance_host", models.CharField(default="", max_length=255, verbose_name="实例IP")), + ("instance_port", models.IntegerField(default=48322, verbose_name="实例端口")), + ("is_job_inconsistent", models.IntegerField(default=2, verbose_name="业务Job数量是否不一致")), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="SqlserverCheckLinkServerReport", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("creator", models.CharField(max_length=64, verbose_name="创建人")), + ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("updater", models.CharField(max_length=64, verbose_name="修改人")), + ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")), + ("bk_biz_id", models.IntegerField(default=0, help_text="业务的 cmdb id")), + ("bk_cloud_id", models.IntegerField(default=0, help_text="云区域 id")), + ("status", models.BooleanField(default=True, help_text="巡检结果状态, 默认正常")), + ("msg", models.TextField(default="", help_text="备注信息")), + ("cluster", models.CharField(default="", max_length=255, verbose_name="集群名称")), + ( + "cluster_type", + models.CharField( + choices=ClusterType.get_choices(), + default="", + max_length=64, + ), + ), + ("instance_host", models.CharField(default="", max_length=255, verbose_name="实例IP")), + ("instance_port", models.IntegerField(default=48322, verbose_name="实例端口")), + ("is_link_server_inconsistent", models.IntegerField(default=2, verbose_name="业务LinkServer数量是否不一致")), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="SqlserverCheckSysJobStatuReport", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("creator", models.CharField(max_length=64, verbose_name="创建人")), + ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("updater", models.CharField(max_length=64, verbose_name="修改人")), + ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")), + ("bk_biz_id", models.IntegerField(default=0, help_text="业务的 cmdb id")), + ("bk_cloud_id", models.IntegerField(default=0, help_text="云区域 id")), + ("status", models.BooleanField(default=True, help_text="巡检结果状态, 默认正常")), + ("msg", models.TextField(default="", help_text="备注信息")), + ("cluster", models.CharField(default="", max_length=255, verbose_name="集群名称")), + ( + "cluster_type", + models.CharField( + choices=ClusterType.get_choices(), + default="", + max_length=64, + ), + ), + ("instance_host", models.CharField(default="", max_length=255, verbose_name="实例IP")), + ("instance_port", models.IntegerField(default=48322, verbose_name="实例端口")), + ("is_job_disable", models.IntegerField(default=0, verbose_name="是否存在disable状态的系统job")), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="SqlserverCheckUserSyncReport", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("creator", models.CharField(max_length=64, verbose_name="创建人")), + ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("updater", models.CharField(max_length=64, verbose_name="修改人")), + ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")), + ("bk_biz_id", models.IntegerField(default=0, help_text="业务的 cmdb id")), + ("bk_cloud_id", models.IntegerField(default=0, help_text="云区域 id")), + ("status", models.BooleanField(default=True, help_text="巡检结果状态, 默认正常")), + ("msg", models.TextField(default="", help_text="备注信息")), + ("cluster", models.CharField(default="", max_length=255, verbose_name="集群名称")), + ( + "cluster_type", + models.CharField( + choices=ClusterType.get_choices(), + default="", + max_length=64, + ), + ), + ("instance_host", models.CharField(default="", max_length=255, verbose_name="实例IP")), + ("instance_port", models.IntegerField(default=48322, verbose_name="实例端口")), + ("is_user_inconsistent", models.IntegerField(default=0, verbose_name="业务账号是否不一致")), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/dbm-ui/backend/db_report/migrations/0008_auto_20241129_1754.py b/dbm-ui/backend/db_report/migrations/0008_auto_20241129_1754.py new file mode 100644 index 0000000000..9f57056a79 --- /dev/null +++ b/dbm-ui/backend/db_report/migrations/0008_auto_20241129_1754.py @@ -0,0 +1,256 @@ +# Generated by Django 3.2.25 on 2024-11-29 09:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "db_report", + "0007_sqlservercheckappsettingreport_sqlservercheckjobsyncreport_sqlserverchecklinkserverreport_sqlserverc", + ), + ] + + operations = [ + migrations.AlterField( + model_name="sqlservercheckappsettingreport", + name="cluster_type", + field=models.CharField( + choices=[ + ("tendbsingle", "MySQL单节点集群"), + ("tendbha", "MySQL高可用集群"), + ("tendbcluster", "TendbCluster集群"), + ("tbinlogdumper", "TBinlogDumper"), + ("redis", "Redis"), + ("PredixyRedisCluster", "RedisCluster集群"), + ("PredixyTendisplusCluster", "Tendisplus存储版集群"), + ("TwemproxyRedisInstance", "TendisCache集群"), + ("TwemproxyTendisSSDInstance", "TendisSSD集群"), + ("TwemproxyTendisplusInstance", "Tendis存储版集群"), + ("RedisInstance", "RedisCache主从版"), + ("TendisSSDInstance", "TendisSSD主从版"), + ("TendisplusInstance", "Tendisplus主从版"), + ("RedisCluster", "RedisCluster集群"), + ("TendisplusCluster", "TendisplusCluster集群"), + ("TendisplusInstance", "Tendisplus存储版集群"), + ("RedisInstance", "TendisCache集群"), + ("TendisSSDInstance", "TendisSSD集群"), + ("es", "ES集群"), + ("kafka", "Kafka集群"), + ("hdfs", "Hdfs集群"), + ("influxdb", "Influxdb实例"), + ("pulsar", "Pulsar集群"), + ("doris", "Doris集群"), + ("vm", "vm集群"), + ("dbmon", "redis监控"), + ("MongoReplicaSet", "Mongo副本集"), + ("MongoShardedCluster", "Mongo分片集群"), + ("riak", "Riak集群"), + ("sqlserver_single", "sqlserver单节点版"), + ("sqlserver_ha", "sqlserver主从版"), + ], + default="", + max_length=64, + verbose_name="集群类型", + ), + ), + migrations.AlterField( + model_name="sqlservercheckappsettingreport", + name="is_fix", + field=models.BooleanField(default=False, verbose_name="元数据是否自动修复成功"), + ), + migrations.AlterField( + model_name="sqlservercheckappsettingreport", + name="is_inconsistent", + field=models.BooleanField(default=False, verbose_name="元信息是否不一致"), + ), + migrations.AlterField( + model_name="sqlservercheckjobsyncreport", + name="cluster_type", + field=models.CharField( + choices=[ + ("tendbsingle", "MySQL单节点集群"), + ("tendbha", "MySQL高可用集群"), + ("tendbcluster", "TendbCluster集群"), + ("tbinlogdumper", "TBinlogDumper"), + ("redis", "Redis"), + ("PredixyRedisCluster", "RedisCluster集群"), + ("PredixyTendisplusCluster", "Tendisplus存储版集群"), + ("TwemproxyRedisInstance", "TendisCache集群"), + ("TwemproxyTendisSSDInstance", "TendisSSD集群"), + ("TwemproxyTendisplusInstance", "Tendis存储版集群"), + ("RedisInstance", "RedisCache主从版"), + ("TendisSSDInstance", "TendisSSD主从版"), + ("TendisplusInstance", "Tendisplus主从版"), + ("RedisCluster", "RedisCluster集群"), + ("TendisplusCluster", "TendisplusCluster集群"), + ("TendisplusInstance", "Tendisplus存储版集群"), + ("RedisInstance", "TendisCache集群"), + ("TendisSSDInstance", "TendisSSD集群"), + ("es", "ES集群"), + ("kafka", "Kafka集群"), + ("hdfs", "Hdfs集群"), + ("influxdb", "Influxdb实例"), + ("pulsar", "Pulsar集群"), + ("doris", "Doris集群"), + ("vm", "vm集群"), + ("dbmon", "redis监控"), + ("MongoReplicaSet", "Mongo副本集"), + ("MongoShardedCluster", "Mongo分片集群"), + ("riak", "Riak集群"), + ("sqlserver_single", "sqlserver单节点版"), + ("sqlserver_ha", "sqlserver主从版"), + ], + default="", + max_length=64, + verbose_name="集群类型", + ), + ), + migrations.AlterField( + model_name="sqlservercheckjobsyncreport", + name="is_job_inconsistent", + field=models.BooleanField(default=False, verbose_name="业务Job数量是否不一致"), + ), + migrations.AlterField( + model_name="sqlserverchecklinkserverreport", + name="cluster_type", + field=models.CharField( + choices=[ + ("tendbsingle", "MySQL单节点集群"), + ("tendbha", "MySQL高可用集群"), + ("tendbcluster", "TendbCluster集群"), + ("tbinlogdumper", "TBinlogDumper"), + ("redis", "Redis"), + ("PredixyRedisCluster", "RedisCluster集群"), + ("PredixyTendisplusCluster", "Tendisplus存储版集群"), + ("TwemproxyRedisInstance", "TendisCache集群"), + ("TwemproxyTendisSSDInstance", "TendisSSD集群"), + ("TwemproxyTendisplusInstance", "Tendis存储版集群"), + ("RedisInstance", "RedisCache主从版"), + ("TendisSSDInstance", "TendisSSD主从版"), + ("TendisplusInstance", "Tendisplus主从版"), + ("RedisCluster", "RedisCluster集群"), + ("TendisplusCluster", "TendisplusCluster集群"), + ("TendisplusInstance", "Tendisplus存储版集群"), + ("RedisInstance", "TendisCache集群"), + ("TendisSSDInstance", "TendisSSD集群"), + ("es", "ES集群"), + ("kafka", "Kafka集群"), + ("hdfs", "Hdfs集群"), + ("influxdb", "Influxdb实例"), + ("pulsar", "Pulsar集群"), + ("doris", "Doris集群"), + ("vm", "vm集群"), + ("dbmon", "redis监控"), + ("MongoReplicaSet", "Mongo副本集"), + ("MongoShardedCluster", "Mongo分片集群"), + ("riak", "Riak集群"), + ("sqlserver_single", "sqlserver单节点版"), + ("sqlserver_ha", "sqlserver主从版"), + ], + default="", + max_length=64, + verbose_name="集群类型", + ), + ), + migrations.AlterField( + model_name="sqlserverchecklinkserverreport", + name="is_link_server_inconsistent", + field=models.BooleanField(default=False, verbose_name="业务LinkServer数量是否不一致"), + ), + migrations.AlterField( + model_name="sqlserverchecksysjobstatureport", + name="cluster_type", + field=models.CharField( + choices=[ + ("tendbsingle", "MySQL单节点集群"), + ("tendbha", "MySQL高可用集群"), + ("tendbcluster", "TendbCluster集群"), + ("tbinlogdumper", "TBinlogDumper"), + ("redis", "Redis"), + ("PredixyRedisCluster", "RedisCluster集群"), + ("PredixyTendisplusCluster", "Tendisplus存储版集群"), + ("TwemproxyRedisInstance", "TendisCache集群"), + ("TwemproxyTendisSSDInstance", "TendisSSD集群"), + ("TwemproxyTendisplusInstance", "Tendis存储版集群"), + ("RedisInstance", "RedisCache主从版"), + ("TendisSSDInstance", "TendisSSD主从版"), + ("TendisplusInstance", "Tendisplus主从版"), + ("RedisCluster", "RedisCluster集群"), + ("TendisplusCluster", "TendisplusCluster集群"), + ("TendisplusInstance", "Tendisplus存储版集群"), + ("RedisInstance", "TendisCache集群"), + ("TendisSSDInstance", "TendisSSD集群"), + ("es", "ES集群"), + ("kafka", "Kafka集群"), + ("hdfs", "Hdfs集群"), + ("influxdb", "Influxdb实例"), + ("pulsar", "Pulsar集群"), + ("doris", "Doris集群"), + ("vm", "vm集群"), + ("dbmon", "redis监控"), + ("MongoReplicaSet", "Mongo副本集"), + ("MongoShardedCluster", "Mongo分片集群"), + ("riak", "Riak集群"), + ("sqlserver_single", "sqlserver单节点版"), + ("sqlserver_ha", "sqlserver主从版"), + ], + default="", + max_length=64, + verbose_name="集群类型", + ), + ), + migrations.AlterField( + model_name="sqlserverchecksysjobstatureport", + name="is_job_disable", + field=models.BooleanField(default=False, verbose_name="是否存在disable状态的系统job"), + ), + migrations.AlterField( + model_name="sqlservercheckusersyncreport", + name="cluster_type", + field=models.CharField( + choices=[ + ("tendbsingle", "MySQL单节点集群"), + ("tendbha", "MySQL高可用集群"), + ("tendbcluster", "TendbCluster集群"), + ("tbinlogdumper", "TBinlogDumper"), + ("redis", "Redis"), + ("PredixyRedisCluster", "RedisCluster集群"), + ("PredixyTendisplusCluster", "Tendisplus存储版集群"), + ("TwemproxyRedisInstance", "TendisCache集群"), + ("TwemproxyTendisSSDInstance", "TendisSSD集群"), + ("TwemproxyTendisplusInstance", "Tendis存储版集群"), + ("RedisInstance", "RedisCache主从版"), + ("TendisSSDInstance", "TendisSSD主从版"), + ("TendisplusInstance", "Tendisplus主从版"), + ("RedisCluster", "RedisCluster集群"), + ("TendisplusCluster", "TendisplusCluster集群"), + ("TendisplusInstance", "Tendisplus存储版集群"), + ("RedisInstance", "TendisCache集群"), + ("TendisSSDInstance", "TendisSSD集群"), + ("es", "ES集群"), + ("kafka", "Kafka集群"), + ("hdfs", "Hdfs集群"), + ("influxdb", "Influxdb实例"), + ("pulsar", "Pulsar集群"), + ("doris", "Doris集群"), + ("vm", "vm集群"), + ("dbmon", "redis监控"), + ("MongoReplicaSet", "Mongo副本集"), + ("MongoShardedCluster", "Mongo分片集群"), + ("riak", "Riak集群"), + ("sqlserver_single", "sqlserver单节点版"), + ("sqlserver_ha", "sqlserver主从版"), + ], + default="", + max_length=64, + verbose_name="集群类型", + ), + ), + migrations.AlterField( + model_name="sqlservercheckusersyncreport", + name="is_user_inconsistent", + field=models.BooleanField(default=False, verbose_name="业务账号是否不一致"), + ), + ] diff --git a/dbm-ui/backend/db_report/models/__init__.py b/dbm-ui/backend/db_report/models/__init__.py index 3e5d95b1e1..e1d3032db8 100644 --- a/dbm-ui/backend/db_report/models/__init__.py +++ b/dbm-ui/backend/db_report/models/__init__.py @@ -13,3 +13,10 @@ from .meta_check_report import MetaCheckReport from .mysqlbackup_check_report import MysqlBackupCheckReport from .redisbackup_check_report import RedisBackupCheckReport +from .sqlserver_check_report import ( + SqlserverCheckAppSettingReport, + SqlserverCheckJobSyncReport, + SqlserverCheckLinkServerReport, + SqlserverCheckSysJobStatuReport, + SqlserverCheckUserSyncReport, +) diff --git a/dbm-ui/backend/db_report/models/sqlserver_check_report.py b/dbm-ui/backend/db_report/models/sqlserver_check_report.py new file mode 100644 index 0000000000..424b810ad5 --- /dev/null +++ b/dbm-ui/backend/db_report/models/sqlserver_check_report.py @@ -0,0 +1,67 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.db import models +from django.utils.translation import ugettext as _ + +from backend.db_meta.enums import ClusterType +from backend.db_report.report_basemodel import BaseReportABS + + +class BaseSqlserverReportABS(BaseReportABS): + cluster = models.CharField(max_length=255, default="", verbose_name=_("集群名称")) + cluster_type = models.CharField( + max_length=64, choices=ClusterType.get_choices(), default="", verbose_name=_("集群类型") + ) + instance_host = models.CharField(max_length=255, default="", verbose_name=_("实例IP")) + instance_port = models.IntegerField(default=48322, verbose_name=_("实例端口")) + + class Meta: + abstract = True + + +class SqlserverCheckAppSettingReport(BaseSqlserverReportABS): + """ + 检测实例的app_setting表信息是否正常 + """ + + is_inconsistent = models.BooleanField(default=False, verbose_name=_("元信息是否不一致")) + is_fix = models.BooleanField(default=False, verbose_name=_("元数据是否自动修复成功")) + + +class SqlserverCheckSysJobStatuReport(BaseSqlserverReportABS): + """ + 检测主从集群的系统作业的启动情况 + """ + + is_job_disable = models.BooleanField(default=False, verbose_name=_("是否存在disable状态的系统job")) + + +class SqlserverCheckUserSyncReport(BaseSqlserverReportABS): + """ + 检测主从集群的账号同步状态 + """ + + is_user_inconsistent = models.BooleanField(default=False, verbose_name=_("业务账号是否不一致")) + + +class SqlserverCheckJobSyncReport(BaseSqlserverReportABS): + """ + 检测主从集群的业务作业的JOB同步状态 + """ + + is_job_inconsistent = models.BooleanField(default=False, verbose_name=_("业务Job数量是否不一致")) + + +class SqlserverCheckLinkServerReport(BaseSqlserverReportABS): + """ + 检测主从集群的linkserver配置的同步状态 + """ + + is_link_server_inconsistent = models.BooleanField(default=False, verbose_name=_("业务LinkServer数量是否不一致")) diff --git a/dbm-ui/backend/db_services/dbpermission/constants.py b/dbm-ui/backend/db_services/dbpermission/constants.py index 423483d7fe..1ff7b1e961 100644 --- a/dbm-ui/backend/db_services/dbpermission/constants.py +++ b/dbm-ui/backend/db_services/dbpermission/constants.py @@ -63,7 +63,7 @@ class GLOBAL(str, StructuredEnum): class MongoDB: class USER(str, StructuredEnum): - READ = EnumField("Read", _("Read")) + READ = EnumField("read", _("Read")) READ_write = EnumField("readWrite", _("readWrite")) READ_ANY_DATABASE = EnumField("readAnyDatabase", _("readAnyDatabase")) READ_WRITE_ANY_DATABASE = EnumField("readWriteAnyDatabase", _("readWriteAnyDatabase")) diff --git a/dbm-ui/backend/db_services/dbresource/handlers.py b/dbm-ui/backend/db_services/dbresource/handlers.py index 807b39e421..b8ac4fa23d 100644 --- a/dbm-ui/backend/db_services/dbresource/handlers.py +++ b/dbm-ui/backend/db_services/dbresource/handlers.py @@ -90,10 +90,14 @@ def custom_filter(self): pass def get_target_specs(self): + if not self.specs: + return [] + self.calc_machine_pair() self.calc_cluster_shard_num() self.system_filter() self.custom_filter() + return self.specs @@ -196,6 +200,11 @@ class RedisClusterSpecFilter(RedisSpecFilter): # 支持简单阔缩容倍数（非DTS方式/Slot迁移扩容方式） SCALE_MULITPLE = 4 + def __init__(self, capacity, future_capacity, spec_cluster_type, spec_machine_type, qps=None, shard_num=0): + # 这里的规格类型要转为tendis cache, redis cluster的规格类型同cache + spec_machine_type = SpecMachineType.TendisTwemproxyRedisInstance + super().__init__(capacity, future_capacity, spec_cluster_type, spec_machine_type, qps, shard_num) + def calc_machine_pair(self): """计算每种规格所需的机器组数和集群总容量: 目标容量 / 规格容量""" for spec in self.specs: diff --git a/dbm-ui/backend/db_services/dbresource/serializers.py b/dbm-ui/backend/db_services/dbresource/serializers.py index b37bd30464..c391fc9219 100644 --- a/dbm-ui/backend/db_services/dbresource/serializers.py +++ b/dbm-ui/backend/db_services/dbresource/serializers.py @@ -256,6 +256,7 @@ class ResourceSummarySerializer(serializers.Serializer): help_text=_("集群类型"), choices=SpecClusterType.get_choices(), required=False, default="" ) spec_id_list = serializers.CharField(help_text=_("规格ID"), required=False, default="") + enable_spec = serializers.BooleanField(help_text=_("仅聚合启用规格"), required=False, default=False) for_biz = serializers.IntegerField(help_text=_("专用业务ID"), required=False, default=0) city = serializers.CharField(help_text=_("城市名"), required=False, allow_blank=True) diff --git a/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py b/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py index b5a4259731..0debb593bf 100644 --- a/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py +++ b/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py @@ -151,6 +151,7 @@ def insert_log_into_node(_backup_node, _log): _backup_node["host"], _backup_node["port"] = _log["backup_host"], _log["backup_port"] _backup_node["file_list_details"] = [] _backup_node["binlog_info"] = _log.get("binlog_info") + _backup_node["backup_type"] = _log["backup_type"] # 更新备份时间，并插入文件列表信息 insert_time_field(_backup_node, _log) diff --git a/dbm-ui/backend/db_services/redis/redis_modules/apps.py b/dbm-ui/backend/db_services/redis/redis_modules/apps.py new file mode 100644 index 0000000000..c7a9d684df --- /dev/null +++ b/dbm-ui/backend/db_services/redis/redis_modules/apps.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging + +from django.apps import AppConfig +from django.db.models.signals import post_migrate + +logger = logging.getLogger("root") + + +def init_default_modules(sender, **kwargs): + from backend.db_services.redis.redis_modules.models import TbRedisModuleSupport + + try: + TbRedisModuleSupport.init_default_modules() + except Exception as err: # pylint: disable=broad-except: + logger.warning(f"init_default_modules occur error, {err}") + + +class InstanceConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "backend.db_services.redis.redis_modules" + + def ready(self): + post_migrate.connect(init_default_modules, sender=self) diff --git a/dbm-ui/backend/db_services/redis/redis_modules/migrations/0004_clusterredismoduleassociate.py b/dbm-ui/backend/db_services/redis/redis_modules/migrations/0004_clusterredismoduleassociate.py new file mode 100644 index 0000000000..b3c90aecbf --- /dev/null +++ b/dbm-ui/backend/db_services/redis/redis_modules/migrations/0004_clusterredismoduleassociate.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.25 on 2024-11-07 09:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("redis_modules", "0003_alter_tbredismodulesupport_so_file"), + ] + + operations = [ + migrations.CreateModel( + name="ClusterRedisModuleAssociate", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("cluster_id", models.IntegerField(default=0, verbose_name="集群ID")), + ("module_names", models.JSONField(default=list, verbose_name="module名称列表")), + ], + options={ + "verbose_name": "Cluster Redis module关联", + "verbose_name_plural": "Cluster Redis module关联", + "db_table": "tb_cluster_redis_module_associate", + "unique_together": {("cluster_id",)}, + }, + ), + ] diff --git a/dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py b/dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py index ad6932d7cd..654106c581 100644 --- a/dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py +++ b/dbm-ui/backend/db_services/redis/redis_modules/models/redis_module_support.py @@ -19,3 +19,63 @@ class Meta: verbose_name_plural = _("Redis module支持") db_table = "tb_redis_module_support" unique_together = (("major_version", "module_name", "so_file"),) + + @classmethod + def init_default_modules(cls, *args, **kwargs): + """初始化module 默认数据""" + default_modules = [ + {"major_version": "Redis-4", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-4", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-4.0.9", "module_name": "fo4_lock", "so_file": "redis_fo4lock.so"}, + {"major_version": "Redis-4.0.9", "module_name": "fo4_matchmaker", "so_file": "redis_fo4matchmaker.so"}, + {"major_version": "Redis-4.0.9", "module_name": "fo4_util", "so_file": "redis_fo4util.so"}, + { + "major_version": "Redis-4.0.9", + "module_name": "jlsy-b2", + "so_file": "libB2RedisModule_linux64_service30000.so", + }, + {"major_version": "Redis-4.0.9", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-4.0.9", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-5", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-5", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-6", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-6", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-6", "module_name": "redisjson", "so_file": "librejson-2.6.6.so"}, + {"major_version": "Redis-7", "module_name": "redisbloom", "so_file": "redisbloom-2.6.13.so"}, + {"major_version": "Redis-7", "module_name": "rediscell", "so_file": "libredis_cell_0.3.1.so"}, + {"major_version": "Redis-7", "module_name": "redisjson", "so_file": "librejson-2.6.11.so"}, + ] + + # 获取已存在的记录集合 + existing_set = list( + cls.objects.filter( + major_version__in=[module["major_version"] for module in default_modules], + module_name__in=[module["module_name"] for module in default_modules], + so_file__in=[module["so_file"] for module in default_modules], + ) + .values_list("major_version", "module_name", "so_file") + .distinct() + ) + + # 准备要批量创建的新记录 + modules_to_create = [ + cls(major_version=module["major_version"], module_name=module["module_name"], so_file=module["so_file"]) + for module in default_modules + if (module["major_version"], module["module_name"], module["so_file"]) not in existing_set + ] + + # 批量创建新记录 + cls.objects.bulk_create(modules_to_create) + + +class ClusterRedisModuleAssociate(models.Model): + """redis集群-module关联表""" + + cluster_id = models.IntegerField(_("集群ID"), default=0) + module_names = models.JSONField(_("module名称列表"), default=list) + + class Meta: + verbose_name = _("Cluster Redis module关联") + verbose_name_plural = _("Cluster Redis module关联") + db_table = "tb_cluster_redis_module_associate" + unique_together = (("cluster_id"),) diff --git a/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py b/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py index ea53fa100c..60f16b244a 100644 --- a/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py +++ b/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py @@ -29,6 +29,7 @@ from backend.db_services.dbbase.resources.query import ResourceList from backend.db_services.dbbase.resources.register import register_resource_decorator from backend.db_services.ipchooser.query.resource import ResourceQueryHelper +from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate from backend.db_services.redis.resources.constants import SQL_QUERY_MASTER_SLAVE_STATUS from backend.utils.basic import dictfetchall @@ -152,6 +153,10 @@ def _to_cluster_representation( forward_to__cluster_entry_type=ClusterEntryType.CLB.value, ).exists() + # 获取集群module名称 + cluster_module = ClusterRedisModuleAssociate.objects.filter(cluster_id=cluster.id).first() + module_names = cluster_module.module_names if cluster_module is not None else [] + # 集群额外信息 cluster_extra_info = { "cluster_spec": cluster_spec, @@ -162,6 +167,7 @@ def _to_cluster_representation( "redis_slave": redis_slave, "cluster_shard_num": len(redis_master), "machine_pair_cnt": machine_pair_cnt, + "module_names": module_names, } cluster_info = super()._to_cluster_representation( cluster, diff --git a/dbm-ui/backend/db_services/redis/toolbox/handlers.py b/dbm-ui/backend/db_services/redis/toolbox/handlers.py index 7433fa84f6..33f08f8b2f 100644 --- a/dbm-ui/backend/db_services/redis/toolbox/handlers.py +++ b/dbm-ui/backend/db_services/redis/toolbox/handlers.py @@ -20,6 +20,8 @@ from backend.db_services.dbbase.cluster.handlers import ClusterServiceHandler from backend.db_services.ipchooser.handlers.host_handler import HostHandler from backend.db_services.ipchooser.query.resource import ResourceQueryHelper +from backend.db_services.redis.redis_modules.models import TbRedisModuleSupport +from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate from backend.db_services.redis.resources.constants import SQL_QUERY_COUNT_INSTANCES, SQL_QUERY_INSTANCES from backend.db_services.redis.resources.redis_cluster.query import RedisListRetrieveResource from backend.exceptions import ApiResultError @@ -167,3 +169,19 @@ def webconsole_rpc(cls, cluster_id: int, cmd: str, db_num: int = 0, raw: bool = return {"query": "", "error_msg": err.message} return {"query": rpc_results[0]["result"], "error_msg": ""} + + @classmethod + def get_cluster_module_info(cls, cluster_id: int, version: str): + """ + 获取集群module信息 + """ + # 获取版本支持的module名称列表 + support_modules = TbRedisModuleSupport.objects.filter(major_version=version).values_list( + "module_name", flat=True + ) + # 获取集群已安装的module名称列表 + cluster_module_associate = ClusterRedisModuleAssociate.objects.filter(cluster_id=cluster_id).first() + cluster_modules = getattr(cluster_module_associate, "module_names", []) + # 字典输出集群是否安装的module列表 + results = {item: (item in cluster_modules) for item in support_modules} + return {"results": results} diff --git a/dbm-ui/backend/db_services/redis/toolbox/serializers.py b/dbm-ui/backend/db_services/redis/toolbox/serializers.py index 6e438cb98c..a33ff474d6 100644 --- a/dbm-ui/backend/db_services/redis/toolbox/serializers.py +++ b/dbm-ui/backend/db_services/redis/toolbox/serializers.py @@ -236,3 +236,8 @@ class Meta: "err_msg": "", } } + + +class GetClusterModuleInfoSerializer(serializers.Serializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + version = serializers.CharField(help_text=_("版本")) diff --git a/dbm-ui/backend/db_services/redis/toolbox/views.py b/dbm-ui/backend/db_services/redis/toolbox/views.py index c546063c6c..0ebe13ea02 100644 --- a/dbm-ui/backend/db_services/redis/toolbox/views.py +++ b/dbm-ui/backend/db_services/redis/toolbox/views.py @@ -23,6 +23,7 @@ from backend.db_services.redis.toolbox.handlers import ToolboxHandler from backend.db_services.redis.toolbox.serializers import ( GetClusterCapacityInfoSerializer, + GetClusterModuleInfoSerializer, GetClusterVersionSerializer, QueryByOneClusterSerializer, QueryClusterIpsSerializer, @@ -95,3 +96,14 @@ def get_cluster_capacity_update_info(self, request, bk_biz_id, **kwargs): update_info = get_cluster_capacity_update_required_info(**data) update_fields = ["capacity_update_type", "require_spec_id", "require_machine_group_num", "err_msg"] return Response(dict(zip(update_fields, update_info))) + + @common_swagger_auto_schema( + operation_summary=_("获取集群module信息"), + query_serializer=GetClusterModuleInfoSerializer(), + tags=[SWAGGER_TAG], + ) + @action(methods=["GET"], detail=False, serializer_class=GetClusterModuleInfoSerializer, pagination_class=None) + def get_cluster_module_info(self, request, bk_biz_id, **kwargs): + data = self.params_validate(self.get_serializer_class()) + cluster_id, version = data["cluster_id"], data["version"] + return Response(ToolboxHandler.get_cluster_module_info(cluster_id, version)) diff --git a/dbm-ui/backend/db_services/taskflow/handlers.py b/dbm-ui/backend/db_services/taskflow/handlers.py index 3c68321bb0..0a973257f2 100644 --- a/dbm-ui/backend/db_services/taskflow/handlers.py +++ b/dbm-ui/backend/db_services/taskflow/handlers.py @@ -34,7 +34,7 @@ RevokePipelineException, SkipNodeException, ) -from backend.flow.consts import StateType +from backend.flow.consts import PENDING_STATES, StateType from backend.flow.engine.bamboo.engine import BambooEngine from backend.flow.models import FlowNode, FlowTree from backend.utils.string import format_json_string @@ -52,7 +52,7 @@ def revoke_pipeline(self): # 如果当前的pipeline未被创建，则直接更新FlowTree的状态为撤销态 tree = FlowTree.objects.get(root_id=self.root_id) - if tree.status in [StateType.CREATED, StateType.READY]: + if tree.status in PENDING_STATES: tree.status = StateType.REVOKED tree.save() return EngineAPIResult(result=True, message=_("pipeline未创建，仅更新FlowTree")) @@ -209,15 +209,20 @@ def bklog_esquery_search(indices, query_string, start_time, end_time): def get_version_logs(self, node_id: str, version_id: str) -> List[Dict[str, Dict[str, str]]]: """获取节点的日志信息""" - try: - flow_node = FlowNode.objects.get(root_id=self.root_id, node_id=node_id) - except FlowNode.DoesNotExist: + if not FlowNode.objects.filter(root_id=self.root_id, node_id=node_id).count(): return [self.generate_log_record(message=_("节点尚未运行，请稍后查看"))] - if flow_node.updated_at < timezone.now() - timedelta(days=env.BKLOG_DEFAULT_RETENTION): + + try: + node_histories_map = {h["version"]: h for h in self.get_node_histories(node_id)} + history = node_histories_map[version_id] + except KeyError: + return [self.generate_log_record(message=_("无法找到当前版本{}的节点日志").format(version_id))] + + if history["finished_time"] < timezone.now() - timedelta(days=env.BKLOG_DEFAULT_RETENTION): return [self.generate_log_record(message=_("节点日志仅保留{}天").format(env.BKLOG_DEFAULT_RETENTION))] - start_time = datetime2str(flow_node.started_at) - end_time = datetime2str(flow_node.updated_at + timedelta(days=7)) + start_time = datetime2str(history["started_time"]) + end_time = datetime2str(history["finished_time"] + timedelta(days=1)) dbm_logs = self.bklog_esquery_search( indices=f"{env.DBA_APP_BK_BIZ_ID}_bklog.dbm_log", query_string=f"({self.root_id} AND {node_id} AND {version_id})" diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_db_table_size.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_db_table_size.json index 9acc0e7a99..847ce5fe65 100644 --- a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_db_table_size.json +++ b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_db_table_size.json @@ -16,7 +16,7 @@ }, "params": { "paths": [ - "/home/mysql/dbareport/mysql/dbsize/report.log.*" + "/home/mysql/dbareport/mysql/dbsize/report*.log" ], "conditions": { "separator": "", @@ -31,184 +31,592 @@ }, "fields": [ { + "field_name": "ext", + "type": "object", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "额外信息字段", + "unit": "", + "alias_name": "__ext", "option": { - "time_zone": "", - "time_format": "" + "es_type": "object" }, + "is_disabled": false, + "is_built_in": true, "is_time": false, - "is_delete": false, - "alias_name": "appid", - "field_name": "bk_biz_id", - "field_type": "int", - "description": "", - "field_index": 0, + "field_type": "object", "is_analyzed": false, - "is_built_in": false, - "is_dimension": true, - "is_case_sensitive": false, - "tokenize_on_chars": "" + "is_delete": false, + "is_dimension": true }, { + "field_name": "__parse_failure", + "type": "boolean", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "清洗失败标记", + "unit": "", + "alias_name": "", "option": { - "time_zone": "", - "time_format": "" + "real_path": "bk_separator_object.__parse_failure", + "es_type": "boolean" }, + "is_disabled": false, + "is_built_in": true, "is_time": false, - "is_delete": false, - "alias_name": "", - "field_name": "immute_domain", "field_type": "string", - "description": "", - "field_index": 1, "is_analyzed": false, - "is_built_in": false, - "is_dimension": true, - "is_case_sensitive": false, - "tokenize_on_chars": "" + "is_delete": false, + "is_dimension": true }, { + "field_name": "bk_host_id", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "主机ID", + "unit": "", + "alias_name": "", "option": { - "time_zone": "", - "time_format": "" + "es_type": "integer" }, + "is_disabled": false, + "is_built_in": true, "is_time": false, + "field_type": "int", + "is_analyzed": false, "is_delete": false, - "alias_name": "db_ip", - "field_name": "ip", - "field_type": "string", - "description": "", - "field_index": 2, + "is_dimension": true + }, + { + "field_name": "cloudid", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "云区域ID", + "unit": "", + "alias_name": "cloudId", + "option": { + "es_type": "integer" + }, + "is_disabled": false, + "is_built_in": true, + "is_time": false, + "field_type": "int", "is_analyzed": false, - "is_built_in": false, - "is_dimension": true, - "is_case_sensitive": false, - "tokenize_on_chars": "" + "is_delete": false, + "is_dimension": true }, { + "field_name": "gseindex", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "gse索引", + "unit": "", + "alias_name": "gseIndex", "option": { - "time_zone": "", - "time_format": "" + "es_type": "long" }, + "is_disabled": false, + "is_built_in": true, "is_time": false, + "field_type": "long", + "is_analyzed": false, "is_delete": false, - "alias_name": "", - "field_name": "port", + "is_dimension": true + }, + { + "field_name": "iterationindex", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "迭代ID", + "unit": "", + "alias_name": "iterationIndex", + "option": { + "es_type": "integer" + }, + "is_disabled": false, + "is_built_in": true, + "is_time": false, "field_type": "int", - "description": "", - "field_index": 3, "is_analyzed": false, - "is_built_in": false, - "is_dimension": true, + "is_delete": false, + "is_dimension": true + }, + { + "field_name": "data", + "type": "string", + "tag": "metric", + "default_value": null, + "is_config_by_user": true, + "description": "original_text", + "unit": "", + "alias_name": "log", + "option": { + "es_norms": false, + "es_type": "text" + }, + "is_disabled": false, "is_case_sensitive": false, - "tokenize_on_chars": "" + "tokenize_on_chars": "", + "is_built_in": true, + "is_time": false, + "field_type": "string", + "is_analyzed": true, + "is_dimension": false, + "is_delete": false }, { + "field_name": "filename", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "日志路径", + "unit": "", + "alias_name": "path", "option": { - "time_zone": "", - "time_format": "" + "es_type": "keyword" }, + "is_disabled": false, + "is_built_in": true, "is_time": false, - "is_delete": false, - "alias_name": "", - "field_name": "role", "field_type": "string", - "description": "", - "field_index": 4, "is_analyzed": false, - "is_built_in": false, - "is_dimension": true, - "is_case_sensitive": false, - "tokenize_on_chars": "" + "is_delete": false, + "is_dimension": true }, { + "field_name": "ip", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "ip", + "unit": "", + "alias_name": "serverIp", "option": { - "time_zone": "", - "time_format": "" + "es_type": "keyword" }, + "is_disabled": false, + "is_built_in": true, "is_time": false, + "field_type": "string", + "is_analyzed": false, "is_delete": false, + "is_dimension": true + }, + { + "field_name": "bk_cloud_id", "alias_name": "", - "field_name": "original_db_name", - "field_type": "string", + "field_type": "", "description": "", - "field_index": 5, "is_analyzed": false, + "is_dimension": false, + "is_time": false, + "is_delete": true, + "is_built_in": false + }, + { + "field_name": "bk_biz_id", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "appid", + "option": { + "field_index": 1, + "real_path": "bk_separator_object.bk_biz_id", + "es_type": "integer" + }, + "is_disabled": false, "is_built_in": false, - "is_dimension": true, - "is_case_sensitive": false, - "tokenize_on_chars": "" + "is_time": false, + "field_type": "int", + "is_analyzed": false, + "is_delete": false, + "is_dimension": true }, { + "field_name": "bk_target_service_instance_id", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "", "option": { - "time_zone": "", - "time_format": "" + "field_index": 2, + "real_path": "bk_separator_object.bk_target_service_instance_id", + "es_type": "integer" }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", + "is_built_in": false, "is_time": false, + "field_type": "int", + "is_analyzed": false, "is_delete": false, + "is_dimension": true + }, + { + "field_name": "cluster_domain", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", "alias_name": "", - "field_name": "db_name", + "option": { + "field_index": 3, + "real_path": "bk_separator_object.cluster_domain", + "es_type": "keyword" + }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", + "is_built_in": false, + "is_time": false, "field_type": "string", - "description": "", - "field_index": 6, "is_analyzed": false, - "is_built_in": false, - "is_dimension": true, - "is_case_sensitive": false, - "tokenize_on_chars": "" + "is_delete": false, + "is_dimension": true }, { + "field_name": "database_name", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "", "option": { - "time_zone": "", - "time_format": "" + "field_index": 4, + "real_path": "bk_separator_object.database_name", + "es_type": "keyword" }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", + "is_built_in": false, "is_time": false, + "field_type": "string", + "is_analyzed": false, "is_delete": false, + "is_dimension": true + }, + { + "field_name": "database_size", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", "alias_name": "", - "field_name": "db_size", + "option": { + "field_index": 5, + "real_path": "bk_separator_object.database_size", + "es_type": "long" + }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", + "is_built_in": false, + "is_time": false, "field_type": "long", - "description": "", - "field_index": 7, "is_analyzed": false, + "is_delete": false, + "is_dimension": true + }, + { + "field_name": "db_module", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "", + "option": { + "field_index": 6, + "real_path": "bk_separator_object.db_module", + "es_type": "integer" + }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", "is_built_in": false, - "is_dimension": true, + "is_time": false, + "field_type": "int", + "is_analyzed": false, + "is_delete": false, + "is_dimension": true + }, + { + "field_name": "instance_host", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "", + "option": { + "field_index": 7, + "real_path": "bk_separator_object.instance_host", + "es_type": "keyword" + }, + "is_disabled": false, "is_case_sensitive": false, - "tokenize_on_chars": "" + "tokenize_on_chars": "", + "is_built_in": false, + "is_time": false, + "field_type": "string", + "is_analyzed": false, + "is_delete": false, + "is_dimension": true }, { + "field_name": "instance_port", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "", "option": { - "time_zone": "", - "time_format": "" + "field_index": 8, + "real_path": "bk_separator_object.instance_port", + "es_type": "integer" }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", + "is_built_in": false, "is_time": false, + "field_type": "int", + "is_analyzed": false, "is_delete": false, + "is_dimension": true + }, + { + "field_name": "instance_role", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", "alias_name": "", - "field_name": "table_name", + "option": { + "field_index": 9, + "real_path": "bk_separator_object.instance_role", + "es_type": "keyword" + }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", + "is_built_in": false, + "is_time": false, "field_type": "string", + "is_analyzed": false, + "is_delete": false, + "is_dimension": true + }, + { + "field_name": "machine_type", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, "description": "", - "field_index": 8, + "unit": "", + "alias_name": "", + "option": { + "field_index": 10, + "real_path": "bk_separator_object.machine_type", + "es_type": "keyword" + }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", + "is_built_in": false, + "is_time": false, + "field_type": "string", "is_analyzed": false, + "is_delete": false, + "is_dimension": true + }, + { + "field_name": "original_database_name", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "", + "option": { + "field_index": 11, + "real_path": "bk_separator_object.original_database_name", + "es_type": "keyword" + }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", "is_built_in": false, + "is_time": false, + "field_type": "string", + "is_analyzed": false, + "is_delete": false, + "is_dimension": true + }, + { + "field_name": "report_time", + "type": "timestamp", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "数据时间", + "unit": "", + "alias_name": "dtEventTimeStamp", + "option": { + "time_format": "rfc3339", + "field_index": 12, + "timestamp_unit": "ms", + "default_function": "fn:timestamp_from_utctime", + "real_path": "bk_separator_object.report_time", + "time_zone": 8, + "es_format": "epoch_millis", + "es_type": "date" + }, + "is_disabled": false, + "is_built_in": true, + "is_time": true, "is_dimension": true, + "field_type": "string", + "is_analyzed": false, + "is_delete": false + }, + { + "field_name": "report_time", + "type": "string", + "tag": "metric", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "", + "option": { + "time_format": "rfc3339", + "field_index": 12, + "timestamp_unit": "ms", + "default_function": "fn:timestamp_from_utctime", + "real_path": "bk_separator_object.report_time", + "time_zone": 8, + "es_format": "epoch_millis", + "es_type": "date" + }, + "is_disabled": false, "is_case_sensitive": false, - "tokenize_on_chars": "" + "tokenize_on_chars": "", + "is_built_in": false, + "is_time": true, + "is_dimension": true, + "field_type": "string", + "is_analyzed": false, + "is_delete": false + }, + { + "field_name": "report_time", + "type": "timestamp", + "tag": "timestamp", + "default_value": "", + "is_config_by_user": true, + "description": "数据上报时间", + "unit": "", + "alias_name": "time", + "option": { + "time_format": "rfc3339", + "field_index": 12, + "timestamp_unit": "ms", + "default_function": "fn:timestamp_from_utctime", + "real_path": "bk_separator_object.report_time", + "time_zone": 8, + "es_format": "epoch_millis", + "es_type": "date" + }, + "is_disabled": false, + "is_built_in": true, + "is_time": true, + "is_dimension": true, + "field_type": "string", + "is_analyzed": false, + "is_delete": false }, { + "field_name": "table_name", + "type": "string", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, + "description": "", + "unit": "", + "alias_name": "", "option": { - "time_zone": "", - "time_format": "" + "field_index": 13, + "real_path": "bk_separator_object.table_name", + "es_type": "keyword" }, + "is_disabled": false, + "is_case_sensitive": false, + "tokenize_on_chars": "", + "is_built_in": false, "is_time": false, + "field_type": "string", + "is_analyzed": false, "is_delete": false, - "alias_name": "", + "is_dimension": true + }, + { "field_name": "table_size", - "field_type": "long", + "type": "float", + "tag": "dimension", + "default_value": null, + "is_config_by_user": true, "description": "", - "field_index": 9, - "is_analyzed": false, - "is_built_in": false, - "is_dimension": true, + "unit": "", + "alias_name": "", + "option": { + "field_index": 14, + "real_path": "bk_separator_object.table_size", + "es_type": "long" + }, + "is_disabled": false, "is_case_sensitive": false, - "tokenize_on_chars": "" + "tokenize_on_chars": "", + "is_built_in": false, + "is_time": false, + "field_type": "long", + "is_analyzed": false, + "is_delete": false, + "is_dimension": true } ] } \ No newline at end of file diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index ddc5f4fc9c..a5c984f802 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -24,7 +24,6 @@ DEPENDENCIES_PLUGINS = ["bkmonitorbeat", "bkunifylogbeat"] BIGDATA_DEPEND_PLUGINS = ["bkmonitorbeat"] - # 默认flow缓存数据过期时间：7天 DEFAULT_FLOW_CACHE_EXPIRE_TIME = 7 * 24 * 60 * 60 @@ -175,8 +174,9 @@ class StateType(str, StructuredEnum): EXPIRED = EnumField("EXPIRED", _("已过期")) +PENDING_STATES = [StateType.CREATED.value, StateType.READY.value] FAILED_STATES = [StateType.FAILED.value, StateType.REVOKED.value] -SUCCEED_STATES = [StateType.FINISHED] +SUCCEED_STATES = [StateType.FINISHED.value] # 备份系统文件TAG BACKUP_TAG = ( diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_enable_disable.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_enable_disable.py index bdd70e63c4..47f2842025 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_enable_disable.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_enable_disable.py @@ -42,10 +42,10 @@ def cluster_enable_disable( # 设置参数 sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] - cluster_type = sub_get_kwargs.payload["cluster_type"] # 获取集群信息 sub_get_kwargs.get_cluster_info_deinstall(cluster_id=cluster_id) + cluster_type = sub_get_kwargs.payload["cluster_type"] # 修改实例状态 kwargs = { diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py index 9f3cc1b430..1f1800a416 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py @@ -17,6 +17,7 @@ from backend.db_meta.enums.cluster_type import ClusterType from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs @@ -103,4 +104,16 @@ def cluster_increase_node(root_id: str, ticket_data: Optional[Dict], sub_kwargs: sub_pipelines.append(sub_sub_pipeline) sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + # 安装dbmon + ip_list = sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}增加节点".format(sub_get_kwargs.payload["cluster_name"]))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_monogs_autofix.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_monogs_autofix.py index 90ecc32241..8456e93c84 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_monogs_autofix.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_monogs_autofix.py @@ -20,6 +20,7 @@ from backend.flow.consts import MongoDBInstanceType, MongoDBManagerUser from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.add_domain_to_dns import ExecAddDomainToDnsOperationComponent from backend.flow.plugins.components.collections.mongodb.add_password_to_db import ( ExecAddPasswordToDBOperationComponent, @@ -229,6 +230,18 @@ def mongos_autofix(root_id: str, ticket_data: Optional[Dict], sub_sub_kwargs: Ac kwargs=kwargs, ) + # 安装dbmon + ip_list = sub_sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) + # 老实例提下架单 kwargs = { "infos": sub_sub_get_kwargs.payload["nodes"], diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_replace.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_replace.py index 7162bc65bb..d6962ba947 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_replace.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_replace.py @@ -18,6 +18,7 @@ from backend.flow.consts import MongoDBClusterRole, MongoDBInstanceType from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent from backend.flow.plugins.components.collections.mongodb.mongodb_cmr_4_meta import CMRMongoDBMetaComponent from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent @@ -143,4 +144,15 @@ def cluster_replace(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKw sub_pipeline.add_act( act_name=_("MongoDB-mongos修改meta"), act_component_code=CMRMongoDBMetaComponent.code, kwargs=kwargs ) + # 安装dbmon + ip_list = sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) return sub_pipeline.build_sub_process(sub_name=_("MongoDB--cluster整机替换")) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_scale.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_scale.py index f60df574b5..4bba4211f0 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_scale.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_scale.py @@ -18,6 +18,7 @@ from backend.flow.consts import MongoDBClusterRole from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs @@ -78,4 +79,16 @@ def cluster_scale(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwar sub_sub_pipelines.append(sub_sub_pipeline) sub_pipeline.add_parallel_sub_pipeline(sub_sub_pipelines) + # 安装dbmon + ip_list = sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--cluster:{}容量变更".format(info["cluster_id"]))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_shard_autofix.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_shard_autofix.py index 9a11600250..5b66ba5d49 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_shard_autofix.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_shard_autofix.py @@ -17,6 +17,7 @@ from backend.flow.consts import MongoDBClusterRole from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent from backend.flow.plugins.components.collections.mongodb.mongodb_cmr_4_meta import CMRMongoDBMetaComponent from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent @@ -97,4 +98,17 @@ def shard_autofix( sub_pipeline.add_act( act_name=_("MongoDB-mongod修改meta"), act_component_code=CMRMongoDBMetaComponent.code, kwargs=kwargs ) + + # 安装dbmon + ip_list = sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}自愈--ip:{}".format(name, info["ip"]))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/increase_mongos.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/increase_mongos.py index 878fb6029f..720b02e11c 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/increase_mongos.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/increase_mongos.py @@ -16,6 +16,7 @@ from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.add_domain_to_dns import ExecAddDomainToDnsOperationComponent from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent from backend.flow.plugins.components.collections.mongodb.mongos_scale_4_meta import MongosScaleMetaComponent @@ -110,4 +111,16 @@ def increase_mongos(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKw kwargs=kwargs, ) + # 安装dbmon + ip_list = sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}增加mongos".format(cluster_name))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py index 92c79146cf..97baf60a7a 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py @@ -18,6 +18,7 @@ from backend.flow.consts import MongoDBClusterRole from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent from backend.flow.plugins.components.collections.mongodb.mongodb_cmr_4_meta import CMRMongoDBMetaComponent from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent @@ -93,6 +94,17 @@ def replicaset_replace( act_component_code=CMRMongoDBMetaComponent.code, kwargs=kwargs, ) + # 安装dbmon 副本集 + ip_list = sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) else: if cluster_role == MongoDBClusterRole.ShardSvr.value: info["db_type"] = "cluster_mongodb" diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_scale.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_scale.py index 063345b678..4b13f5b3eb 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_scale.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_scale.py @@ -18,6 +18,7 @@ from backend.flow.consts import MongoDBClusterRole from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs @@ -90,6 +91,17 @@ def replicaset_scale( if not cluster_role: name = "replicaset" + # 安装dbmon + ip_list = sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) else: if cluster_role == MongoDBClusterRole.ShardSvr.value: name = "shard" diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py index 2966c85fe7..5fbc166a65 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py @@ -16,6 +16,7 @@ from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import add_install_dbmon from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs @@ -83,4 +84,16 @@ def replicaset_set_increase_node( sub_sub_pipelines.append(sub_sub_pipeline) sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_sub_pipelines) + # 安装dbmon 副本集 + ip_list = sub_get_kwargs.payload["plugin_hosts"] + exec_ips = [host["ip"] for host in ip_list] + add_install_dbmon( + root_id=root_id, + flow_data=ticket_data, + pipeline=sub_pipeline, + iplist=exec_ips, + bk_cloud_id=ip_list[0]["bk_cloud_id"], + allow_empty_instance=True, + ) + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}增加node".format("replicaset"))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/common/mysql_resotre_data_sub_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/common/mysql_resotre_data_sub_flow.py index 819b40701e..ddb2e5b7b9 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/common/mysql_resotre_data_sub_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/common/mysql_resotre_data_sub_flow.py @@ -19,26 +19,13 @@ from backend.flow.consts import MysqlChangeMasterType from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList -from backend.flow.engine.bamboo.scene.mysql.common.get_binlog_backup import get_backup_binlog from backend.flow.engine.bamboo.scene.mysql.common.get_local_backup import get_local_backup -from backend.flow.engine.bamboo.scene.spider.common.exceptions import ( - TendbGetBackupInfoFailedException, - TendbGetBinlogFailedException, -) +from backend.flow.engine.bamboo.scene.spider.common.exceptions import TendbGetBackupInfoFailedException from backend.flow.plugins.components.collections.mysql.exec_actuator_script import ExecuteDBActuatorScriptComponent -from backend.flow.plugins.components.collections.mysql.mysql_download_backupfile import ( - MySQLDownloadBackupfileComponent, -) from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent as MySQLTransFileComponent -from backend.flow.utils.mysql.mysql_act_dataclass import ( - DownloadBackupFileKwargs, - DownloadMediaKwargs, - ExecActuatorKwargs, - P2PFileKwargs, -) +from backend.flow.utils.mysql.mysql_act_dataclass import DownloadMediaKwargs, ExecActuatorKwargs, P2PFileKwargs from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload -from backend.utils.time import str2datetime logger = logging.getLogger("flow") @@ -150,115 +137,6 @@ def mysql_restore_data_sub_flow( return sub_pipeline.build_sub_process(sub_name=_("用本地备份恢复数据{}".format(exec_act_kwargs.exec_ip))) -def mysql_rollback_data_sub_flow( - root_id: str, ticket_data: dict, cluster: dict, cluster_model: Cluster, ins_list: list, is_rollback_binlog: bool -): - """ - @param root_id: 流程 root_id - @param ticket_data: 单据输入的tick_data - @param cluster: 流程信息 - @param cluster_model: 集群元数据 - @param ins_list: 查询实例列表 - @param is_rollback_binlog:是否前滚日志 - @return: - cluster: new_slave_ip,new_slave_port,master_ip,master_port,file_target_path,charset,change_master_force,backup_time - """ - rollback_time = str2datetime(cluster["rollback_time"], "%Y-%m-%d %H:%M:%S") - if is_rollback_binlog: - cluster["recover_binlog"] = True - else: - cluster["recover_binlog"] = False - sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) - cluster["change_master"] = False - backup_info = get_local_backup(ins_list, cluster_model, cluster["rollback_time"]) - logger.debug(backup_info) - if backup_info is None: - logger.error("cluster {} backup info not exists".format(cluster_model.id)) - raise TendbGetBackupInfoFailedException(message=_("获取集群 {} 的备份信息失败".format(cluster_model.id))) - cluster["backupinfo"] = backup_info - exec_act_kwargs = ExecActuatorKwargs( - bk_cloud_id=cluster_model.bk_cloud_id, - cluster_type=cluster_model.cluster_type, - cluster=cluster, - ) - exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.mysql_mkdir_dir.__name__ - exec_act_kwargs.exec_ip = cluster["rollback_ip"] - sub_pipeline.add_act( - act_name=_("创建目录 {}".format(cluster["file_target_path"])), - act_component_code=ExecuteDBActuatorScriptComponent.code, - kwargs=asdict(exec_act_kwargs), - ) - - # 下载备份文件 backup_info["file_list"] 需要确认 - task_ids = ["{}/{}".format(backup_info["backup_dir"], i["file_name"]) for i in backup_info["file_list"]] - if backup_info["backup_meta_file"] not in task_ids: - task_ids.append(backup_info["backup_meta_file"]) - sub_pipeline.add_act( - act_name=_("下载备份到{}".format(cluster["rollback_ip"])), - act_component_code=MySQLTransFileComponent.code, - kwargs=asdict( - P2PFileKwargs( - bk_cloud_id=cluster_model.bk_cloud_id, - file_list=task_ids, - file_target_path=cluster["file_target_path"], - source_ip_list=[backup_info["instance_ip"]], - exec_ip=cluster["rollback_ip"], - ) - ), - ) - - cluster["change_master"] = False - exec_act_kwargs.cluster = copy.deepcopy(cluster) - exec_act_kwargs.exec_ip = cluster["rollback_ip"] - exec_act_kwargs.job_timeout = MYSQL_DATA_RESTORE_TIME - exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.get_rollback_data_restore_payload.__name__ - sub_pipeline.add_act( - act_name=_("恢复新从节点数据 {}:{}".format(exec_act_kwargs.exec_ip, cluster["rollback_port"])), - act_component_code=ExecuteDBActuatorScriptComponent.code, - kwargs=asdict(exec_act_kwargs), - write_payload_var="change_master_info", - ) - - if is_rollback_binlog: - backup_time = str2datetime(backup_info["backup_time"], "%Y-%m-%d %H:%M:%S") - cluster["backup_time"] = backup_info["backup_time"] - binlog_result = get_backup_binlog( - cluster_id=cluster_model.id, - start_time=backup_time, - end_time=rollback_time, - binlog_info=backup_info["binlog_info"], - ) - if "query_binlog_error" in binlog_result.keys(): - raise TendbGetBinlogFailedException(message=binlog_result["query_binlog_error"]) - - cluster.update(binlog_result) - - download_kwargs = DownloadBackupFileKwargs( - bk_cloud_id=cluster_model.bk_cloud_id, - task_ids=binlog_result["binlog_task_ids"], - dest_ip=cluster["rollback_ip"], - dest_dir=cluster["file_target_path"], - reason="rollback node rollback binlog", - cluster=cluster, - ) - sub_pipeline.add_act( - act_name=_("下载定点恢复的binlog到{}").format(cluster["rollback_ip"]), - act_component_code=MySQLDownloadBackupfileComponent.code, - kwargs=asdict(download_kwargs), - ) - exec_act_kwargs.exec_ip = cluster["rollback_ip"] - exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.tendb_recover_binlog_payload.__name__ - exec_act_kwargs.cluster = copy.deepcopy(cluster) - sub_pipeline.add_act( - act_name=_("定点恢复之前滚binlog{}".format(exec_act_kwargs.exec_ip)), - act_component_code=ExecuteDBActuatorScriptComponent.code, - kwargs=asdict(exec_act_kwargs), - ) - return sub_pipeline.build_sub_process( - sub_name=_("用本地备份恢复数据{}:{}".format(exec_act_kwargs.exec_ip, cluster["rollback_port"])) - ) - - def mysql_restore_master_slave_sub_flow( root_id: str, ticket_data: dict, cluster: dict, cluster_model: Cluster, ins_list: list ): diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_fake_sql_semantic_check.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_fake_sql_semantic_check.py index 8ca20600b1..4dcdbec0e3 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_fake_sql_semantic_check.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_fake_sql_semantic_check.py @@ -14,6 +14,7 @@ from django.utils.translation import ugettext as _ from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.plugins.components.collections.common.pause import PauseComponent from backend.flow.plugins.components.collections.mysql.fake_semantic_check import FakeSemanticCheckComponent logger = logging.getLogger("flow") @@ -46,17 +47,17 @@ def fake_semantic_check(self): parallel_acts = [ { "act_name": _("并行1"), - "act_component_code": FakeSemanticCheckComponent.code, + "act_component_code": PauseComponent.code, "kwargs": {"parallel_acts": "1"}, }, { "act_name": _("并行2"), - "act_component_code": FakeSemanticCheckComponent.code, + "act_component_code": PauseComponent.code, "kwargs": {"parallel_acts": "2"}, }, { "act_name": _("错误并行3"), - "act_component_code": FakeSemanticCheckComponent.code, + "act_component_code": PauseComponent.code, "kwargs": {"is_error": True}, }, ] diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_upgrade.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_upgrade.py index a9cd6d1b99..7e602af542 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_upgrade.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_upgrade.py @@ -381,7 +381,8 @@ def tendbha_cluster_upgrade_subflow( bk_host_ids = [new_slave["bk_host_id"]] old_ro_slave_ip = old_ro_slave["ip"] old_ro_slave_ips.append(old_ro_slave_ip) - db_config = get_instance_config(cluster_cls.bk_cloud_id, old_ro_slave_ip, ports=ports) + origin_config = get_instance_config(cluster_cls.bk_cloud_id, old_ro_slave_ip, ports=ports) + db_config = deal_mycnf(pkg.name, db_version, origin_config) install_ro_slave_sub_pipeline = build_install_slave_sub_pipeline( uid, root_id, @@ -458,17 +459,8 @@ def tendbha_cluster_upgrade_subflow( ms_sub_pipeline = SubBuilder(root_id=root_id, data=parent_global_data) bk_host_ids = [new_master["bk_host_id"], new_slave["bk_host_id"]] master = cluster_cls.storageinstance_set.get(instance_inner_role=InstanceInnerRole.MASTER.value) - db_config = get_instance_config(cluster_cls.bk_cloud_id, master.machine.ip, ports) - if mysql_version_parse(db_version) >= mysql_version_parse("5.7.0"): - will_del_keys = ["slave_parallel_type", "replica_parallel_type"] - # 如果不是tmysql的话，需要删除一些配置 - if "tmysql" not in pkg.name: - will_del_keys.append("log_bin_compress") - will_del_keys.append("relay_log_uncompress") - for port in db_config: - for key in will_del_keys: - if db_config[port].get(key): - del db_config[port][key] + origin_config = get_instance_config(cluster_cls.bk_cloud_id, master.machine.ip, ports) + db_config = deal_mycnf(pkg.name, db_version, origin_config) install_ms_pair_subflow = build_install_ms_pair_sub_pipeline( uid=uid, root_id=root_id, @@ -583,6 +575,26 @@ def tendbha_cluster_upgrade_subflow( return sub_pipeline.build_sub_process(sub_name=_("{}:整体迁移升级").format(cluster_cls.immute_domain)) +def deal_mycnf(pkg_name, db_version: str, db_config: dict): + if mysql_version_parse(db_version) >= mysql_version_parse("5.7.0"): + will_del_keys = ["slave_parallel_type", "replica_parallel_type"] + # 如果不是tmysql的话，需要删除一些配置 + if "tmysql" not in pkg_name: + will_del_keys.append("log_bin_compress") + will_del_keys.append("relay_log_uncompress") + for port in db_config: + for key in will_del_keys: + if db_config[port].get(key): + del db_config[port][key] + if mysql_version_parse(db_version) >= mysql_version_parse("8.0.0"): + will_del_keys = ["innodb_large_prefix"] + for port in db_config: + for key in will_del_keys: + if db_config[port].get(key): + del db_config[port][key] + return db_config + + def non_standby_slaves_upgrade_subflow( uid: str, root_id: str, diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_rollback_data_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_rollback_data_flow.py index e6b0f8722e..6d94360468 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_rollback_data_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_rollback_data_flow.py @@ -23,26 +23,37 @@ from backend.db_meta.enums import ClusterType, InstanceInnerRole, InstanceRole from backend.db_meta.models import Cluster, StorageInstanceTuple from backend.db_package.models import Package -from backend.flow.consts import InstanceStatus, MediumEnum, MysqlChangeMasterType, RollbackType +from backend.db_services.mysql.fixpoint_rollback.handlers import FixPointRollbackHandler +from backend.flow.consts import InstanceStatus, MediumEnum, MySQLBackupTypeEnum, MysqlChangeMasterType, RollbackType from backend.flow.engine.bamboo.scene.common.builder import Builder, SubBuilder from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList from backend.flow.engine.bamboo.scene.mysql.common.exceptions import NormalTenDBFlowException -from backend.flow.engine.bamboo.scene.mysql.common.get_local_backup import check_storage_database -from backend.flow.engine.bamboo.scene.mysql.common.mysql_resotre_data_sub_flow import mysql_rollback_data_sub_flow +from backend.flow.engine.bamboo.scene.mysql.common.get_local_backup import check_storage_database, get_local_backup from backend.flow.engine.bamboo.scene.mysql.mysql_rollback_data_sub_flow import ( rollback_local_and_backupid, + rollback_local_and_time, rollback_remote_and_backupid, rollback_remote_and_time, ) from backend.flow.engine.bamboo.scene.mysql.mysql_single_apply_flow import MySQLSingleApplyFlow -from backend.flow.engine.bamboo.scene.spider.common.exceptions import NormalSpiderFlowException +from backend.flow.engine.bamboo.scene.spider.common.exceptions import ( + NormalSpiderFlowException, + TendbGetBackupInfoFailedException, +) from backend.flow.plugins.components.collections.mysql.exec_actuator_script import ExecuteDBActuatorScriptComponent +from backend.flow.plugins.components.collections.mysql.mysql_crond_control import MysqlCrondMonitorControlComponent from backend.flow.plugins.components.collections.mysql.mysql_rds_execute import MySQLExecuteRdsComponent from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent from backend.flow.utils.mysql.common.mysql_cluster_info import get_version_and_charset -from backend.flow.utils.mysql.mysql_act_dataclass import DownloadMediaKwargs, ExecActuatorKwargs, ExecuteRdsKwargs +from backend.flow.utils.mysql.mysql_act_dataclass import ( + CrondMonitorKwargs, + DownloadMediaKwargs, + ExecActuatorKwargs, + ExecuteRdsKwargs, +) from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload from backend.flow.utils.mysql.mysql_context_dataclass import ClusterInfoContext +from backend.utils.time import str2datetime logger = logging.getLogger("flow") @@ -171,19 +182,31 @@ def rollback_data_flow(self): inst_list.append( "{}{}{}".format(stand_by_slaves[0].machine.ip, IP_PORT_DIVIDER, stand_by_slaves[0].port) ) + + backupinfo = get_local_backup(inst_list, cluster_class, mycluster["rollback_time"]) + if backupinfo is None: + logger.error("cluster {} backup info not exists".format(cluster_class.id)) + raise TendbGetBackupInfoFailedException(message=_("获取集群 {} 的备份信息失败".format(cluster_class.id))) + mycluster["backupinfo"] = copy.deepcopy(backupinfo) + sub_pipeline.add_sub_pipeline( - sub_flow=mysql_rollback_data_sub_flow( + sub_flow=rollback_local_and_time( root_id=self.root_id, ticket_data=copy.deepcopy(self.data), - cluster=mycluster, + cluster_info=mycluster, cluster_model=cluster_class, - ins_list=inst_list, - is_rollback_binlog=True, ) ) # 远程备份+时间 elif self.data["rollback_type"] == RollbackType.REMOTE_AND_TIME.value: + rollback_handler = FixPointRollbackHandler(cluster_class.id) + backupinfo = rollback_handler.query_latest_backup_log(str2datetime(mycluster["rollback_time"])) + if backupinfo is None: + logger.error("cluster {} backup info not exists".format(cluster_class.id)) + raise TendbGetBackupInfoFailedException(message=_("获取集群 {} 的备份信息失败".format(cluster_class.id))) + mycluster["backupinfo"] = copy.deepcopy(backupinfo) + sub_pipeline.add_sub_pipeline( sub_flow=rollback_remote_and_time( root_id=self.root_id, ticket_data=copy.deepcopy(self.data), cluster_info=mycluster @@ -253,13 +276,35 @@ def rollback_to_cluster_flow(self): db_module_id=self.data["db_module_id"], cluster_type=self.data["cluster_type"], ) + # 为了回档的备份信息各个节点统一，获取备份的信息提前 + backupinfo = copy.deepcopy(self.data["backupinfo"]) + if self.data["rollback_type"] == RollbackType.LOCAL_AND_TIME: + inst_list = ["{}{}{}".format(master.machine.ip, IP_PORT_DIVIDER, master.port)] + stand_by_slaves = cluster_class.storageinstance_set.filter( + instance_inner_role=InstanceInnerRole.SLAVE.value, + is_stand_by=True, + status=InstanceStatus.RUNNING.value, + ) + if len(stand_by_slaves) > 0: + inst_list.append( + "{}{}{}".format(stand_by_slaves[0].machine.ip, IP_PORT_DIVIDER, stand_by_slaves[0].port) + ) + backupinfo = get_local_backup(inst_list, cluster_class, self.data["rollback_time"]) + if backupinfo is None: + logger.error("cluster {} backup info not exists".format(cluster_class.id)) + raise TendbGetBackupInfoFailedException(message=_("获取集群 {} 的备份信息失败".format(cluster_class.id))) - sub_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(self.data)) + elif self.data["rollback_type"] == RollbackType.REMOTE_AND_TIME.value: + rollback_handler = FixPointRollbackHandler(cluster_class.id) + backupinfo = rollback_handler.query_latest_backup_log(str2datetime(self.data["rollback_time"])) + if backupinfo is None: + logger.error("cluster {} backup info not exists".format(cluster_class.id)) + raise TendbGetBackupInfoFailedException(message=_("获取集群 {} 的备份信息失败".format(cluster_class.id))) + sub_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(self.data)) rollback_class = Cluster.objects.get(id=self.data["rollback_cluster_id"]) storages = rollback_class.storageinstance_set.all() rollback_pipeline_list = [] - repl_pipeline_list = [] for rollback_storage in storages: if not check_storage_database( rollback_class.bk_cloud_id, rollback_storage.machine.ip, rollback_storage.port @@ -285,7 +330,7 @@ def rollback_to_cluster_flow(self): "skip_local_exists": True, "name_regex": "^.+{}\\.\\d+(\\..*)*$".format(master.port), "rollback_time": self.data["rollback_time"], - "backupinfo": self.data["backupinfo"], + "backupinfo": backupinfo, "rollback_type": self.data["rollback_type"], "rollback_ip": rollback_storage.machine.ip, "rollback_port": rollback_storage.port, @@ -318,32 +363,45 @@ def rollback_to_cluster_flow(self): act_component_code=ExecuteDBActuatorScriptComponent.code, kwargs=asdict(exec_act_kwargs), ) - + # 屏蔽监控，停止从库备份 + rollback_pipeline.add_act( + act_name=_("屏蔽监控 {}").format(rollback_storage.ip_port), + act_component_code=MysqlCrondMonitorControlComponent.code, + kwargs=asdict( + CrondMonitorKwargs( + bk_cloud_id=cluster_class.bk_cloud_id, + exec_ips=[rollback_storage.machine.ip], + port=rollback_storage.port, + ) + ), + ) + if rollback_storage.instance_role in (InstanceRole.BACKEND_SLAVE, InstanceRole.BACKEND_REPEATER): + rollback_pipeline.add_act( + act_name=_("从库stop slave {}").format(rollback_storage.ip_port), + act_component_code=MySQLExecuteRdsComponent.code, + kwargs=asdict( + ExecuteRdsKwargs( + bk_cloud_id=cluster_class.bk_cloud_id, + instance_ip=rollback_storage.machine.ip, + instance_port=rollback_storage.port, + sqls=["stop slave"], + ) + ), + ) # 本地备份+时间 if self.data["rollback_type"] == RollbackType.LOCAL_AND_TIME: - inst_list = ["{}{}{}".format(master.machine.ip, IP_PORT_DIVIDER, master.port)] - stand_by_slaves = cluster_class.storageinstance_set.filter( - instance_inner_role=InstanceInnerRole.SLAVE.value, - is_stand_by=True, - status=InstanceStatus.RUNNING.value, - ).exclude(machine__ip__in=[rollback_storage.machine.ip]) - if len(stand_by_slaves) > 0: - inst_list.append( - "{}{}{}".format(stand_by_slaves[0].machine.ip, IP_PORT_DIVIDER, stand_by_slaves[0].port) - ) + mycluster["backupinfo"] = copy.deepcopy(backupinfo) rollback_pipeline.add_sub_pipeline( - sub_flow=mysql_rollback_data_sub_flow( + sub_flow=rollback_local_and_time( root_id=self.root_id, ticket_data=copy.deepcopy(self.data), - cluster=mycluster, + cluster_info=mycluster, cluster_model=cluster_class, - ins_list=inst_list, - is_rollback_binlog=True, ) ) - # 远程备份+时间 elif self.data["rollback_type"] == RollbackType.REMOTE_AND_TIME.value: + mycluster["backupinfo"] = copy.deepcopy(backupinfo) rollback_pipeline.add_sub_pipeline( sub_flow=rollback_remote_and_time( root_id=self.root_id, ticket_data=copy.deepcopy(self.data), cluster_info=mycluster @@ -367,62 +425,70 @@ def rollback_to_cluster_flow(self): else: raise NormalTenDBFlowException(message=_("rollback_type不存在")) - rollback_pipeline_list.append( - rollback_pipeline.build_sub_process( - sub_name=_("定点回档到{}:{}".format(rollback_storage.machine.ip, rollback_storage.port)) - ) - ) - # 针对slave repeater角色的从库。建立复制链路。重置slave>添加复制账号和获取位点>建立主从关系 + backup_type = backupinfo.get("backup_type", "") + # backup_type = MySQLBackupTypeEnum.PHYSICAL.value if rollback_storage.instance_role in (InstanceRole.BACKEND_SLAVE, InstanceRole.BACKEND_REPEATER): - repl_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(self.data)) - repl_pipeline.add_act( - act_name=_("从库reset slave {}").format(rollback_storage.ip_port), - act_component_code=MySQLExecuteRdsComponent.code, - kwargs=asdict( - ExecuteRdsKwargs( - bk_cloud_id=cluster_class.bk_cloud_id, - instance_ip=rollback_storage.machine.ip, - instance_port=rollback_storage.port, - sqls=["stop slave", "reset slave all"], - ) - ), - ) repl_master = StorageInstanceTuple.objects.get(receiver=rollback_storage) - repl_cluster = { - "target_ip": repl_master.ejector.machine.ip, - "target_port": repl_master.ejector.port, - "repl_ip": rollback_storage.machine.ip, - "repl_port": rollback_storage.port, - "change_master_type": MysqlChangeMasterType.MASTERSTATUS.value, - "change_master_force": True, - } - exec_act_kwargs.cluster = copy.deepcopy(repl_cluster) - exec_act_kwargs.exec_ip = repl_master.ejector.machine.ip - exec_act_kwargs.job_timeout = MYSQL_USUAL_JOB_TIME - exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.tendb_grant_remotedb_repl_user.__name__ - repl_pipeline.add_act( - act_name=_("新增repl帐户{}".format(exec_act_kwargs.exec_ip)), - act_component_code=ExecuteDBActuatorScriptComponent.code, - kwargs=asdict(exec_act_kwargs), - write_payload_var="show_master_status_info", - ) - exec_act_kwargs.exec_ip = rollback_storage.machine.ip - exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.tendb_remotedb_change_master.__name__ - repl_pipeline.add_act( - act_name=_("建立原主从关系{}".format(rollback_storage.ip_port)), - act_component_code=ExecuteDBActuatorScriptComponent.code, - kwargs=asdict(exec_act_kwargs), - ) - repl_pipeline_list.append( - repl_pipeline.build_sub_process( - sub_name=_("建立主从关系: {}->{}".format(repl_master.ejector.ip_port, rollback_storage.ip_port)) + if backup_type == MySQLBackupTypeEnum.PHYSICAL.value: + repl_cluster = { + "target_ip": repl_master.ejector.machine.ip, + "target_port": repl_master.ejector.port, + "repl_ip": rollback_storage.machine.ip, + "repl_port": rollback_storage.port, + "change_master_type": MysqlChangeMasterType.MASTERSTATUS.value, + "change_master_force": True, + } + exec_act_kwargs.cluster = copy.deepcopy(repl_cluster) + exec_act_kwargs.exec_ip = repl_master.ejector.machine.ip + exec_act_kwargs.job_timeout = MYSQL_USUAL_JOB_TIME + exec_act_kwargs.get_mysql_payload_func = ( + MysqlActPayload.tendb_grant_remotedb_repl_user.__name__ + ) + rollback_pipeline.add_act( + act_name=_("新增repl帐户{}".format(exec_act_kwargs.exec_ip)), + act_component_code=ExecuteDBActuatorScriptComponent.code, + kwargs=asdict(exec_act_kwargs), + write_payload_var="show_master_status_info", + ) + exec_act_kwargs.exec_ip = rollback_storage.machine.ip + exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.tendb_remotedb_change_master.__name__ + rollback_pipeline.add_act( + act_name=_("建立原主从关系{}".format(rollback_storage.ip_port)), + act_component_code=ExecuteDBActuatorScriptComponent.code, + kwargs=asdict(exec_act_kwargs), + ) + elif backup_type == MySQLBackupTypeEnum.LOGICAL.value: + rollback_pipeline.add_act( + act_name=_("从库start slave {}").format(rollback_storage.ip_port), + act_component_code=MySQLExecuteRdsComponent.code, + kwargs=asdict( + ExecuteRdsKwargs( + bk_cloud_id=cluster_class.bk_cloud_id, + instance_ip=rollback_storage.machine.ip, + instance_port=rollback_storage.port, + sqls=["start slave"], + ) + ), + ) + rollback_pipeline.add_act( + act_name=_("解除监控屏蔽 {}").format(rollback_storage.ip_port), + act_component_code=MysqlCrondMonitorControlComponent.code, + kwargs=asdict( + CrondMonitorKwargs( + bk_cloud_id=cluster_class.bk_cloud_id, + exec_ips=[rollback_storage.machine.ip], + port=rollback_storage.port, + enable=True, ) + ), + ) + rollback_pipeline_list.append( + rollback_pipeline.build_sub_process( + sub_name=_("定点回档到{}:{}".format(rollback_storage.machine.ip, rollback_storage.port)) ) - + ) sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=rollback_pipeline_list) - if len(repl_pipeline_list) > 0: - sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=repl_pipeline_list) sub_pipeline_list.append( sub_pipeline.build_sub_process(sub_name=_("定点回档到{}".format(rollback_class.immute_domain))) ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_rollback_data_sub_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_rollback_data_sub_flow.py index edc2d1c75d..ed4cb2a343 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_rollback_data_sub_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_rollback_data_sub_flow.py @@ -15,18 +15,16 @@ from django.utils.translation import ugettext as _ from backend.configuration.constants import MYSQL_DATA_RESTORE_TIME -from backend.db_services.mysql.fixpoint_rollback.handlers import FixPointRollbackHandler +from backend.db_meta.models import Cluster from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mysql.common.get_binlog_backup import get_backup_binlog -from backend.flow.engine.bamboo.scene.spider.common.exceptions import ( - TendbGetBackupInfoFailedException, - TendbGetBinlogFailedException, -) +from backend.flow.engine.bamboo.scene.spider.common.exceptions import TendbGetBinlogFailedException from backend.flow.plugins.components.collections.mysql.exec_actuator_script import ExecuteDBActuatorScriptComponent from backend.flow.plugins.components.collections.mysql.mysql_download_backupfile import ( MySQLDownloadBackupfileComponent, ) from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent +from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent as MySQLTransFileComponent from backend.flow.utils.mysql.mysql_act_dataclass import DownloadBackupFileKwargs, ExecActuatorKwargs, P2PFileKwargs from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload from backend.utils.time import str2datetime @@ -43,14 +41,7 @@ def rollback_remote_and_time(root_id: str, ticket_data: dict, cluster_info: dict """ cluster_info["recover_binlog"] = True sub_pipeline = SubBuilder(root_id=root_id, data=copy.deepcopy(ticket_data)) - rollback_time = cluster_info["rollback_time"] - rollback_handler = FixPointRollbackHandler(cluster_info["cluster_id"]) - # 查询接口 - backupinfo = rollback_handler.query_latest_backup_log(str2datetime(rollback_time)) - if backupinfo is None: - logger.error("cluster {} backup info not exists".format(cluster_info["cluster_id"])) - raise TendbGetBackupInfoFailedException(message=_("获取集群 {} 的备份信息失败".format(cluster_info["cluster_id"]))) - cluster_info["backupinfo"] = copy.deepcopy(backupinfo) + backupinfo = copy.deepcopy(cluster_info["backupinfo"]) cluster_info["backup_time"] = backupinfo["backup_time"] task_files = [{"file_name": i} for i in backupinfo["file_list"]] cluster_info["task_files"] = task_files @@ -217,3 +208,102 @@ def rollback_local_and_backupid(root_id: str, ticket_data: dict, cluster_info: d "{}:{}定点回滚数据 LOCAL_AND_BACKUPID ".format(cluster_info["rollback_ip"], cluster_info["rollback_port"]) ) ) + + +def rollback_local_and_time(root_id: str, ticket_data: dict, cluster_info: dict, cluster_model: Cluster): + """ + mysql 定点回档类型 本地备份+时间 + @param root_id: 流程 root_id + @param ticket_data: 单据输入的tick_data + @param cluster_info: 流程信息 + @param cluster_model: 集群元数据 + @return: + cluster: new_slave_ip,new_slave_port,master_ip,master_port,file_target_path,charset,change_master_force,backup_time + """ + rollback_time = str2datetime(cluster_info["rollback_time"], "%Y-%m-%d %H:%M:%S") + cluster_info["recover_binlog"] = True + cluster_info["change_master"] = False + backupinfo = copy.deepcopy(cluster_info["backupinfo"]) + + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + exec_act_kwargs = ExecActuatorKwargs( + bk_cloud_id=cluster_model.bk_cloud_id, + cluster_type=cluster_model.cluster_type, + cluster=cluster_info, + ) + + exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.mysql_mkdir_dir.__name__ + exec_act_kwargs.exec_ip = cluster_info["rollback_ip"] + sub_pipeline.add_act( + act_name=_("创建目录 {}".format(cluster_info["file_target_path"])), + act_component_code=ExecuteDBActuatorScriptComponent.code, + kwargs=asdict(exec_act_kwargs), + ) + + # 下载备份文件 backup_info["file_list"] 需要确认 + task_ids = ["{}/{}".format(backupinfo["backup_dir"], i["file_name"]) for i in backupinfo["file_list"]] + if backupinfo["backup_meta_file"] not in task_ids: + task_ids.append(backupinfo["backup_meta_file"]) + sub_pipeline.add_act( + act_name=_("下载备份到{}".format(cluster_info["rollback_ip"])), + act_component_code=MySQLTransFileComponent.code, + kwargs=asdict( + P2PFileKwargs( + bk_cloud_id=cluster_model.bk_cloud_id, + file_list=task_ids, + file_target_path=cluster_info["file_target_path"], + source_ip_list=[backupinfo["instance_ip"]], + exec_ip=cluster_info["rollback_ip"], + ) + ), + ) + + cluster_info["change_master"] = False + exec_act_kwargs.cluster = copy.deepcopy(cluster_info) + exec_act_kwargs.exec_ip = cluster_info["rollback_ip"] + exec_act_kwargs.job_timeout = MYSQL_DATA_RESTORE_TIME + exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.get_rollback_data_restore_payload.__name__ + sub_pipeline.add_act( + act_name=_("恢复新从节点数据 {}:{}".format(exec_act_kwargs.exec_ip, cluster_info["rollback_port"])), + act_component_code=ExecuteDBActuatorScriptComponent.code, + kwargs=asdict(exec_act_kwargs), + write_payload_var="change_master_info", + ) + + backup_time = str2datetime(backupinfo["backup_time"], "%Y-%m-%d %H:%M:%S") + cluster_info["backup_time"] = backupinfo["backup_time"] + binlog_result = get_backup_binlog( + cluster_id=cluster_model.id, + start_time=backup_time, + end_time=rollback_time, + binlog_info=backupinfo["binlog_info"], + ) + if "query_binlog_error" in binlog_result.keys(): + raise TendbGetBinlogFailedException(message=binlog_result["query_binlog_error"]) + + cluster_info.update(binlog_result) + + download_kwargs = DownloadBackupFileKwargs( + bk_cloud_id=cluster_model.bk_cloud_id, + task_ids=binlog_result["binlog_task_ids"], + dest_ip=cluster_info["rollback_ip"], + dest_dir=cluster_info["file_target_path"], + reason="rollback node rollback binlog", + cluster=cluster_info, + ) + sub_pipeline.add_act( + act_name=_("下载定点恢复的binlog到{}").format(cluster_info["rollback_ip"]), + act_component_code=MySQLDownloadBackupfileComponent.code, + kwargs=asdict(download_kwargs), + ) + exec_act_kwargs.exec_ip = cluster_info["rollback_ip"] + exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.tendb_recover_binlog_payload.__name__ + exec_act_kwargs.cluster = copy.deepcopy(cluster_info) + sub_pipeline.add_act( + act_name=_("定点恢复之前滚binlog{}".format(exec_act_kwargs.exec_ip)), + act_component_code=ExecuteDBActuatorScriptComponent.code, + kwargs=asdict(exec_act_kwargs), + ) + return sub_pipeline.build_sub_process( + sub_name=_("用本地备份恢复数据{}:{}".format(exec_act_kwargs.exec_ip, cluster_info["rollback_port"])) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py b/dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py index 2805490415..f5f7803412 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/redis/atom_jobs/redis_load_module.py @@ -137,4 +137,12 @@ def ClusterLoadModulesAtomJob(root_id, ticket_data, sub_kwargs: ActKwargs, param ) logger.info("===>redis config") + act_kwargs.get_redis_payload_func = RedisActPayload.update_cluster_module.__name__ + sub_pipeline.add_act( + act_name=_("更新cluster_module元数据"), + act_component_code=RedisConfigComponent.code, + kwargs=asdict(act_kwargs), + ) + logger.info("===>redis update_cluster_module") + return sub_pipeline.build_sub_process(sub_name=_("{}-集群加载modules").format(cluster.immute_domain)) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/common/common_sub_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/common/common_sub_flow.py index b0f716fdd9..1ded42caba 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/spider/common/common_sub_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/common/common_sub_flow.py @@ -84,6 +84,7 @@ def add_spider_slaves_sub_flow( # 获取到集群对应的spider端口，作为这次的安装 parent_global_data["spider_ports"] = [cluster.proxyinstance_set.first().port] + parent_global_data["db_module_id"] = cluster.db_module_id sub_pipeline = SubBuilder(root_id=root_id, data=parent_global_data) # 拼接执行原子任务活动节点需要的通用的私有参数结构体, 减少代码重复率，但引用时注意内部参数值传递的问题 @@ -234,6 +235,7 @@ def add_spider_masters_sub_flow( # 获取到集群对应的spider端口，作为这次的安装 parent_global_data["spider_ports"] = [cluster.proxyinstance_set.first().port] parent_global_data["ctl_port"] = cluster.proxyinstance_set.first().admin_port + parent_global_data["db_module_id"] = cluster.db_module_id sub_pipeline = SubBuilder(root_id=root_id, data=parent_global_data) # 拼接执行原子任务活动节点需要的通用的私有参数结构体, 减少代码重复率，但引用时注意内部参数值传递的问题 diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_rollback_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_rollback_flow.py index 5ae20e98a7..6f9dc80fbe 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_rollback_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_rollback_flow.py @@ -29,10 +29,11 @@ TendbGetBackupInfoFailedException, ) from backend.flow.engine.bamboo.scene.spider.spider_recover import remote_node_rollback, spider_recover_sub_flow +from backend.flow.plugins.components.collections.mysql.mysql_crond_control import MysqlCrondMonitorControlComponent from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent from backend.flow.plugins.components.collections.spider.spider_db_meta import SpiderDBMetaComponent from backend.flow.utils.mysql.common.mysql_cluster_info import get_version_and_charset -from backend.flow.utils.mysql.mysql_act_dataclass import DBMetaOPKwargs, DownloadMediaKwargs +from backend.flow.utils.mysql.mysql_act_dataclass import CrondMonitorKwargs, DBMetaOPKwargs, DownloadMediaKwargs from backend.flow.utils.mysql.mysql_context_dataclass import ClusterInfoContext from backend.flow.utils.spider.spider_db_meta import SpiderDBMeta from backend.flow.utils.spider.tendb_cluster_info import get_rollback_clusters_info @@ -149,6 +150,17 @@ def tendb_rollback_data(self): "change_master": False, } spd_sub_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(self.data)) + spd_sub_pipeline.add_act( + act_name=_("屏蔽监控 {}").format(spider_node["instance"]), + act_component_code=MysqlCrondMonitorControlComponent.code, + kwargs=asdict( + CrondMonitorKwargs( + bk_cloud_id=target_cluster.bk_cloud_id, + exec_ips=[spider_node["ip"]], + port=spider_node["port"], + ) + ), + ) cluster = {"proxy_status": InstanceStatus.RESTORING.value, "proxy_ids": [target_spider.id]} spd_sub_pipeline.add_act( act_name=_("写入初始化实例的db_meta元信息"), @@ -179,6 +191,18 @@ def tendb_rollback_data(self): ) ), ) + spd_sub_pipeline.add_act( + act_name=_("解除监控屏蔽 {}").format(spider_node["instance"]), + act_component_code=MysqlCrondMonitorControlComponent.code, + kwargs=asdict( + CrondMonitorKwargs( + bk_cloud_id=target_cluster.bk_cloud_id, + exec_ips=[spider_node["ip"]], + port=spider_node["port"], + enable=True, + ) + ), + ) ins_sub_pipeline_list.append( spd_sub_pipeline.build_sub_process(sub_name=_("{} spider节点恢复".format(spider_node["instance"]))) ) @@ -259,6 +283,18 @@ def tendb_rollback_data(self): ) ), ) + # 屏蔽监控 + ins_sub_pipeline.add_act( + act_name=_("屏蔽监控 {}").format(shard_id), + act_component_code=MysqlCrondMonitorControlComponent.code, + kwargs=asdict( + CrondMonitorKwargs( + bk_cloud_id=target_cluster.bk_cloud_id, + exec_ips=[remote_node["new_master"]["ip"], remote_node["new_slave"]["ip"]], + port=remote_node["new_master"]["port"], + ) + ), + ) ins_sub_pipeline.add_sub_pipeline( sub_flow=remote_node_rollback( @@ -280,6 +316,18 @@ def tendb_rollback_data(self): ) ), ) + ins_sub_pipeline.add_act( + act_name=_("解除监控屏蔽 {}").format(shard_id), + act_component_code=MysqlCrondMonitorControlComponent.code, + kwargs=asdict( + CrondMonitorKwargs( + bk_cloud_id=target_cluster.bk_cloud_id, + exec_ips=[remote_node["new_master"]["ip"], remote_node["new_slave"]["ip"]], + port=remote_node["new_master"]["port"], + enable=True, + ) + ), + ) ins_sub_pipeline_list.append( ins_sub_pipeline.build_sub_process(sub_name=_("{} 分片主从恢复".format(shard_id))) ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_recover.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_recover.py index 4a877ceb61..6429dc6e1f 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_recover.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_recover.py @@ -8,13 +8,14 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import copy from dataclasses import asdict from django.utils.translation import ugettext as _ -from backend.configuration.constants import MYSQL_DATA_RESTORE_TIME +from backend.configuration.constants import MYSQL_DATA_RESTORE_TIME, MYSQL_USUAL_JOB_TIME from backend.db_meta.enums import ClusterType -from backend.flow.consts import RollbackType +from backend.flow.consts import MySQLBackupTypeEnum, MysqlChangeMasterType, RollbackType from backend.flow.engine.bamboo.scene.common.builder import SubBuilder from backend.flow.engine.bamboo.scene.mysql.common.get_binlog_backup import get_backup_binlog from backend.flow.engine.bamboo.scene.spider.common.exceptions import TendbGetBinlogFailedException @@ -22,7 +23,8 @@ from backend.flow.plugins.components.collections.mysql.mysql_download_backupfile import ( MySQLDownloadBackupfileComponent, ) -from backend.flow.utils.mysql.mysql_act_dataclass import DownloadBackupFileKwargs, ExecActuatorKwargs +from backend.flow.plugins.components.collections.mysql.mysql_rds_execute import MySQLExecuteRdsComponent +from backend.flow.utils.mysql.mysql_act_dataclass import DownloadBackupFileKwargs, ExecActuatorKwargs, ExecuteRdsKwargs from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload from backend.utils.time import str2datetime @@ -126,14 +128,26 @@ def remote_node_rollback(root_id: str, ticket_data: dict, cluster: dict): sub_pipeline_all = SubBuilder(root_id=root_id, data=ticket_data) sub_pipeline_all_list = [] instance_check_list = [] + backup_info = cluster["backupinfo"] + if cluster["new_master"]["instance"] != cluster["new_slave"]["instance"]: + sub_pipeline_all.add_act( + act_name=_("从库stop slave {}").format(cluster["new_slave"]["instance"]), + act_component_code=MySQLExecuteRdsComponent.code, + kwargs=asdict( + ExecuteRdsKwargs( + bk_cloud_id=int(cluster["bk_cloud_id"]), + instance_ip=cluster["new_slave"]["ip"], + instance_port=cluster["new_slave"]["port"], + sqls=["stop slave"], + ) + ), + ) for node in [cluster["new_master"], cluster["new_slave"]]: if node["instance"] in instance_check_list: continue - instance_check_list.append(node["instance"]) cluster["rollback_ip"] = node["ip"] cluster["rollback_port"] = node["port"] - backup_info = cluster["backupinfo"] cluster["backup_time"] = backup_info["backup_time"] if cluster["rollback_type"] == RollbackType.REMOTE_AND_TIME.value: cluster["recover_binlog"] = True @@ -142,7 +156,9 @@ def remote_node_rollback(root_id: str, ticket_data: dict, cluster: dict): sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) exec_act_kwargs = ExecActuatorKwargs( - bk_cloud_id=int(cluster["bk_cloud_id"]), cluster_type=ClusterType.TenDBCluster, cluster=cluster + bk_cloud_id=int(cluster["bk_cloud_id"]), + cluster_type=ClusterType.TenDBCluster, + cluster=copy.deepcopy(cluster), ) exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.mysql_mkdir_dir.__name__ exec_act_kwargs.exec_ip = cluster["rollback_ip"] @@ -210,7 +226,56 @@ def remote_node_rollback(root_id: str, ticket_data: dict, cluster: dict): sub_pipeline_all_list.append( sub_pipeline.build_sub_process(sub_name=_("定点恢复 {}:{}".format(node["ip"], node["port"]))) ) + # 针对slave repeater角色的从库。建立复制链路。重置slave>添加复制账号和获取位点>建立主从关系 sub_pipeline_all.add_parallel_sub_pipeline(sub_pipeline_all_list) + backup_type = backup_info.get("backup_type", "") + # backup_type = MySQLBackupTypeEnum.PHYSICAL.value + if cluster["new_master"]["instance"] != cluster["new_slave"]["instance"]: + if backup_type == MySQLBackupTypeEnum.PHYSICAL.value: + repl_cluster = { + "target_ip": cluster["new_master"]["ip"], + "target_port": cluster["new_master"]["port"], + "repl_ip": cluster["new_slave"]["ip"], + "repl_port": cluster["new_slave"]["port"], + "change_master_type": MysqlChangeMasterType.MASTERSTATUS.value, + "change_master_force": True, + } + repl_exec_act_kwargs = ExecActuatorKwargs( + bk_cloud_id=cluster["bk_cloud_id"], + cluster_type=ClusterType.TenDBCluster, + cluster=copy.deepcopy(repl_cluster), + job_timeout=MYSQL_USUAL_JOB_TIME, + ) + repl_exec_act_kwargs.exec_ip = cluster["new_master"]["ip"] + repl_exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.tendb_grant_remotedb_repl_user.__name__ + sub_pipeline_all.add_act( + act_name=_("新增repl帐户{}".format(repl_exec_act_kwargs.exec_ip)), + act_component_code=ExecuteDBActuatorScriptComponent.code, + kwargs=asdict(repl_exec_act_kwargs), + write_payload_var="show_master_status_info", + ) + # 启动，或者建立组从关系 + repl_exec_act_kwargs.exec_ip = cluster["new_slave"]["ip"] + repl_exec_act_kwargs.get_mysql_payload_func = MysqlActPayload.tendb_remotedb_change_master.__name__ + sub_pipeline_all.add_act( + act_name=_("建立原主从关系{}".format(cluster["new_slave"]["instance"])), + act_component_code=ExecuteDBActuatorScriptComponent.code, + kwargs=asdict(repl_exec_act_kwargs), + ) + + elif backup_type == MySQLBackupTypeEnum.LOGICAL.value: + sub_pipeline_all.add_act( + act_name=_("从库start slave {}").format(cluster["new_slave"]["instance"]), + act_component_code=MySQLExecuteRdsComponent.code, + kwargs=asdict( + ExecuteRdsKwargs( + bk_cloud_id=cluster["bk_cloud_id"], + instance_ip=cluster["new_slave"]["ip"], + instance_port=cluster["new_slave"]["port"], + sqls=["start slave"], + ) + ), + ) return sub_pipeline_all.build_sub_process( sub_name=_( "Remote node {} 恢复: 主 {} 从 {} ".format( diff --git a/dbm-ui/backend/flow/plugins/components/collections/common/transfer_cluster_meta_to_other_biz.py b/dbm-ui/backend/flow/plugins/components/collections/common/transfer_cluster_meta_to_other_biz.py index b5219a22fb..54db348507 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/common/transfer_cluster_meta_to_other_biz.py +++ b/dbm-ui/backend/flow/plugins/components/collections/common/transfer_cluster_meta_to_other_biz.py @@ -45,6 +45,9 @@ def _execute(self, data, parent_data) -> bool: Machine.objects.filter(storageinstance__cluster__immute_domain__in=cluster_domain_list).update( bk_biz_id=target_biz_id, db_module_id=db_module_id ) + Machine.objects.filter(proxyinstance__cluster__immute_domain__in=cluster_domain_list).update( + bk_biz_id=target_biz_id, db_module_id=db_module_id + ) self.log_info("transfer cluster meta to other biz success") return True diff --git a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/insert_app_setting.py b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/insert_app_setting.py index 63b80b64f8..36760b1bc4 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/insert_app_setting.py +++ b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/insert_app_setting.py @@ -62,7 +62,7 @@ def _execute(self, data, parent_data) -> bool: # 配置数据 if insert_sqlserver_config( cluster=cluster, - storages=storages, + storages=list(storages), charset=charset, backup_config=backup_config, alarm_config=alarm_config, diff --git a/dbm-ui/backend/flow/utils/base/payload_handler.py b/dbm-ui/backend/flow/utils/base/payload_handler.py index 947e99c327..6b0cf77436 100644 --- a/dbm-ui/backend/flow/utils/base/payload_handler.py +++ b/dbm-ui/backend/flow/utils/base/payload_handler.py @@ -55,6 +55,8 @@ def __init__(self, bk_cloud_id: int, ticket_data: dict, cluster: dict, cluster_t # todo 后面可能优化这个问题 if self.ticket_data.get("module"): self.db_module_id = self.ticket_data["module"] + elif self.ticket_data.get("db_module_id"): + self.db_module_id = self.ticket_data["db_module_id"] elif self.cluster and self.cluster.get("db_module_id"): self.db_module_id = self.cluster["db_module_id"] else: diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py index 8f4ba2d789..d701a8323b 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py @@ -13,7 +13,6 @@ from backend.flow.utils.mongodb import mongodb_password from backend.ticket.constants import InstanceType - # entities # Node -> ReplicaSet -> Cluster[Rs,ShardedCluster] # MongoNodeWithLabel diff --git a/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py b/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py index f90051ea3b..0558f9bb1c 100644 --- a/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py +++ b/dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py @@ -69,6 +69,11 @@ def __get_mysql_config(self, immutable_domain, db_version) -> Any: 生成并获取mysql实例配置,集群级别配置 spider/spider-ctl/spider-mysql实例统一用这里拿去配置 """ + if db_version != "Tdbctl" and self.db_module_id == 0: + # 这里做一层判断，对传入的db_module_id值判断，非Tdbctl实例，传入的db_module_id必须是合理且存在的值，否则抛出异常 + raise Exception( + f"The db_module_id parameter is illegal, db_module_id:{self.db_module_id}, db_version:{db_version}" + ) data = DBConfigApi.get_or_generate_instance_config( { "bk_biz_id": str(self.ticket_data["bk_biz_id"]), @@ -1249,7 +1254,7 @@ def __flashback_payload(self, databases: List, tables: List, **kwargs) -> dict: }, } # 如果指定了行级别的回档,需添加4个参数,使用GoFlashBackBinlog工具回档 - rows_filter = self.cluster.get(["rows_filter"], "").strip() + rows_filter = self.cluster.get("rows_filter", "").strip() direct_write_back = self.cluster.get("direct_write_back", True) if rows_filter != "": payload["action"] = DBActuatorActionEnum.GoFlashBackBinlog.value diff --git a/dbm-ui/backend/flow/utils/redis/redis_act_playload.py b/dbm-ui/backend/flow/utils/redis/redis_act_playload.py index 6461b4f0d8..fec847ca85 100644 --- a/dbm-ui/backend/flow/utils/redis/redis_act_playload.py +++ b/dbm-ui/backend/flow/utils/redis/redis_act_playload.py @@ -35,6 +35,7 @@ get_dbmon_maxmemory_config_by_cluster_ids, ) from backend.db_services.redis.redis_dts.models.tb_tendis_dts_switch_backup import TbTendisDtsSwitchBackup +from backend.db_services.redis.redis_modules.models.redis_module_support import ClusterRedisModuleAssociate from backend.db_services.redis.redis_modules.util import get_cluster_redis_modules_detial, get_redis_moudles_detail from backend.db_services.redis.util import ( is_predixy_proxy_type, @@ -2498,3 +2499,23 @@ def redis_replicas_force_resync(self, **kwargs) -> dict: "slave_ports": params["slave_ports"], }, } + + def update_cluster_module(self, cluster_map: dict) -> bool: + """ + 更新nodes cluster_module记录 + """ + cluster_id = cluster_map["cluster_id"] + new_module_names = set(cluster_map.get("load_modules", [])) + + # 获取现有记录 + obj, created = ClusterRedisModuleAssociate.objects.get_or_create( + cluster_id=cluster_id, defaults={"module_names": list(new_module_names)} + ) + if not created: + # 如果记录已存在，合并现有的模块列表和新的模块列表 + current_module_names = set(obj.module_names) + if current_module_names != new_module_names: + updated_module_names = list(current_module_names | new_module_names) + obj.module_names = updated_module_names + obj.save() + return True diff --git a/dbm-ui/backend/flow/utils/redis/redis_db_meta.py b/dbm-ui/backend/flow/utils/redis/redis_db_meta.py index 92599e08e1..92b846efe2 100644 --- a/dbm-ui/backend/flow/utils/redis/redis_db_meta.py +++ b/dbm-ui/backend/flow/utils/redis/redis_db_meta.py @@ -836,7 +836,7 @@ def redis_rollback_host_transfer(self) -> bool: def redis_role_swap_4_scene(self) -> bool: """ - 主从互切 + 主从互切 [仅RedisCluster 类型] act_kwargs.cluster["role_swap_host"].append({"new_ejector":new_host_master,"new_receiver":old_master}) """ # 获取cluster @@ -859,25 +859,16 @@ def redis_role_swap_4_scene(self) -> bool: bk_biz_id=self.cluster["bk_biz_id"], ) - # 修改 proxy backend - temp_proxy_set = list(ins1.proxyinstance_set.all()) - ins1.proxyinstance_set.clear() - ins1.proxyinstance_set.add(*ins2.proxyinstance_set.all()) - ins2.proxyinstance_set.clear() - ins2.proxyinstance_set.add(*temp_proxy_set) # 变更同步关系 StorageInstanceTuple.objects.get(ejector=ins1, receiver=ins2).delete(keep_parents=True) StorageInstanceTuple.objects.create(ejector=ins2, receiver=ins1) # 变更角色 - temp_instance_role = ins1.instance_role - tmep_instance_inner_role = ins1.instance_inner_role + ins1.instance_role = InstanceRole.REDIS_SLAVE.value + ins1.instance_inner_role = InstanceInnerRole.SLAVE.value - ins1.instance_role = ins2.instance_role - ins1.instance_inner_role = ins2.instance_inner_role - - ins2.instance_role = temp_instance_role - ins2.instance_inner_role = tmep_instance_inner_role + ins2.instance_role = InstanceRole.REDIS_MASTER.value + ins2.instance_inner_role = InstanceInnerRole.MASTER.value ins1.save(update_fields=["instance_role", "instance_inner_role"]) ins2.save(update_fields=["instance_role", "instance_inner_role"]) diff --git a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py index 5ab46f9624..a12e1720df 100644 --- a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py +++ b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py @@ -8,6 +8,7 @@ specific language governing permissions and limitations under the License. """ import copy +import logging.config import re import secrets from collections import defaultdict @@ -29,6 +30,19 @@ from backend.flow.utils.mysql.db_table_filter import DbTableFilter from backend.flow.utils.mysql.get_mysql_sys_user import generate_mysql_tmp_user +logger = logging.getLogger("flow") + + +def base_sqlserver_drs(instances: list, bk_cloud_id: int, sqls: list): + return DRSApi.sqlserver_rpc( + { + "bk_cloud_id": bk_cloud_id, + "addresses": instances, + "cmds": sqls, + "force": False, + } + ) + def sqlserver_match_dbs( dbs: List[str], @@ -596,7 +610,7 @@ def get_backup_path_files(cluster_id: int, backup_id: str, db_name: str = None): def insert_sqlserver_config( cluster: Cluster, - storages: QuerySet, + storages: List[StorageInstance], backup_config: dict, charset: str, alarm_config: dict, @@ -758,3 +772,108 @@ def insert_sqlserver_config( raise Exception(f"[{storage.ip_port}] insert app_setting failed: {ret[0]['error_msg']}") return True + + +def get_app_setting_data(instance: StorageInstance, bk_cloud_id: int): + """ + 根据实例查询app_setting表的结果 + """ + sql = f"select * from [{SQLSERVER_CUSTOM_SYS_DB}].[dbo].[APP_SETTING]" + ret = base_sqlserver_drs(bk_cloud_id=bk_cloud_id, instances=[instance.ip_port], sqls=[sql]) + if ret[0]["error_msg"]: + return None, f"select app_setting failed: {ret[0]['error_msg']}" + + if len(ret[0]["cmd_results"][0]["table_data"]) == 0: + # 如果返回没有数据，则需要重新导入 + return {}, "app_setting_table is null, check" + + return ret[0]["cmd_results"][0]["table_data"][0], "" + + +def fix_app_setting_data(cluster: Cluster, instance: StorageInstance, sync_mode: str, master: StorageInstance): + """ + 以dbm数据为准，更新实例的app_setting数据 + """ + sql = f"""use [{SQLSERVER_CUSTOM_SYS_DB}] update [dbo].[APP_SETTING] + SET [APP] = {cluster.bk_biz_id}, +[CLUSTER_ID] = {cluster.id}, +[CLUSTER_DOMAIN] = '{cluster.immute_domain}', +[IP] = '{instance.machine.ip}', +[PORT] = {instance.port}, +[BK_BIZ_ID]= {cluster.bk_biz_id}, +[BK_CLOUD_ID] = {cluster.bk_cloud_id}, +[ROLE] = '{instance.instance_inner_role}', +[SYNCHRONOUS_MODE] = '{sync_mode}', +[MASTER_IP] = '{master.machine.ip}', +[MASTER_PORT]= {master.port} +""" + ret = base_sqlserver_drs(bk_cloud_id=cluster.bk_cloud_id, instances=[instance.ip_port], sqls=[sql]) + if ret[0]["error_msg"]: + return False, f"fix app_setting failed: {ret[0]['error_msg']}" + + return True, "fix successfully" + + +def check_sys_job_status(cluster: Cluster, instance: StorageInstance): + """ + 获取实例的系统JOB状态信息 + """ + sql = "select name, enabled from msdb.dbo.sysjobs where name like 'TC_%'" + ret = base_sqlserver_drs(bk_cloud_id=cluster.bk_cloud_id, instances=[instance.ip_port], sqls=[sql]) + msg = "" + if ret[0]["error_msg"]: + msg = f"[{instance.ip_port}] select sys job failed: {ret[0]['error_msg']}" + logger.error(f"[{instance.ip_port}] select sys job failed: {ret[0]['error_msg']}") + return False, msg + + if len(ret[0]["cmd_results"][0]["table_data"]) == 0: + # 如果返回没有数据，证明没有系统JOB + return False, f"[{instance.ip_port}] sys-job is null,check " + + for info in ret[0]["cmd_results"][0]["table_data"]: + if int(info["enabled"]) == 0: + msg += f"[{info['name']}] is disable;\n" + if msg: + return False, msg + return True, msg + + +def check_ha_config( + master_instance: StorageInstance, slave_instance: StorageInstance, bk_cloud_id: int, check_tag: str +): + """ + 检查传入的源和目的的实例的业务账号是否一致 + """ + # 获取系统账号 + sys_users = SQLSERVER_CUSTOM_SYS_USER + sys_users_str = ",".join([f"'{i}'" for i in sys_users]) + + check_sql_map = { + "user": f"""select a.sid as sid + from master.sys.sql_logins a left join master.sys.syslogins b + on a.name=b.name where principal_id>4 and a.name not in({sys_users_str}) and a.name not like '#%' + and a.name not like 'J_%' + """, + "job": "select name from msdb.dbo.sysjobs where name not like 'TC_%' and enabled = 1", + "link_server": "SELECT name FROM master.sys.servers WHERE is_linked = 1", + } + + # 查询所有的业务账号名称，这里会隐藏过滤掉job的临时账号 + + ret = base_sqlserver_drs( + bk_cloud_id=bk_cloud_id, + instances=[master_instance.ip_port, slave_instance.ip_port], + sqls=[check_sql_map[check_tag]], + ) + + if ret[0]["error_msg"] or ret[1]["error_msg"]: + return False, f"get user failed: {ret[0]['error_msg']}/{ret[1]['error_msg']}" + + list_0 = [str(value) for d in ret[0]["cmd_results"][0]["table_data"] for value in d.values()] + list_1 = [str(value) for d in ret[1]["cmd_results"][0]["table_data"] for value in d.values()] + if sorted(list_0) != sorted(list_1): + return ( + False, + f"[{slave_instance.ip_port}]-{check_tag} configuration is not equal to master[{master_instance.ip_port}]", + ) + return True, "" diff --git a/dbm-ui/backend/iam_app/dataclass/actions.py b/dbm-ui/backend/iam_app/dataclass/actions.py index e09a71bd58..a635818962 100644 --- a/dbm-ui/backend/iam_app/dataclass/actions.py +++ b/dbm-ui/backend/iam_app/dataclass/actions.py @@ -398,7 +398,7 @@ class ActionEnum: MYSQL_ADD_ACCOUNT_RULE = ActionMeta( id="mysql_add_account_rule", - name=_("MySQL账号规则变更"), + name=_("MySQL授权规则变更"), name_en="mysql_add_account_rule", type="create", related_actions=[DB_MANAGE.id], @@ -705,7 +705,7 @@ class ActionEnum: TENDBCLUSTER_ADD_ACCOUNT_RULE = ActionMeta( id="tendbcluster_add_account_rule", - name=_("TenDB Cluster 账号规则变更"), + name=_("TenDB Cluster 授权规则变更"), name_en="tendbcluster_add_account_rule", type="create", related_actions=[DB_MANAGE.id], diff --git a/dbm-ui/backend/ticket/builders/mongodb/mongo_replicaset_apply.py b/dbm-ui/backend/ticket/builders/mongodb/mongo_replicaset_apply.py index be2a658296..8be1cfb979 100644 --- a/dbm-ui/backend/ticket/builders/mongodb/mongo_replicaset_apply.py +++ b/dbm-ui/backend/ticket/builders/mongodb/mongo_replicaset_apply.py @@ -117,6 +117,7 @@ def get_replicaset_resource_spec(cls, ticket_data): "resource_spec": { "mongo_machine_set": { **spec.get_spec_info(), + "spec_name": spec.spec_name, "affinity": ticket_data["disaster_tolerance_level"], "location_spec": {"city": ticket_data["city_code"], "sub_zone_ids": []}, # 副本集的亲和性要求至少跨两个机房 diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_clone_rules.py b/dbm-ui/backend/ticket/builders/mysql/mysql_clone_rules.py index 145e06a749..c4a6bc725d 100644 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_clone_rules.py +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_clone_rules.py @@ -14,7 +14,6 @@ from rest_framework import serializers from backend import env -from backend.db_meta.enums import ClusterType from backend.db_services.mysql.permission.constants import CloneClusterType, CloneType from backend.db_services.mysql.permission.exceptions import CloneDataHasExpiredException, DBPermissionBaseException from backend.flow.engine.controller.mysql import MySQLController @@ -42,7 +41,7 @@ class MySQLCloneRulesSerializer(SkipToRepresentationMixin, serializers.Serialize clone_uid = serializers.CharField(help_text=_("权限克隆数据缓存uid"), required=False) clone_type = serializers.ChoiceField(help_text=_("权限克隆类型"), choices=CloneType.get_choices()) clone_cluster_type = serializers.ChoiceField( - help_text=_("克隆集群类型"), choices=ClusterType.get_choices(), required=False, default=CloneClusterType.MYSQL + help_text=_("克隆集群类型"), choices=CloneClusterType.get_choices(), required=False, default=CloneClusterType.MYSQL ) diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_dump_data.py b/dbm-ui/backend/ticket/builders/mysql/mysql_dump_data.py index f4b437b5ec..9b6f6efdc5 100644 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_dump_data.py +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_dump_data.py @@ -22,7 +22,7 @@ DBTableField, MySQLBaseOperateDetailSerializer, ) -from backend.ticket.constants import TicketType +from backend.ticket.constants import TicketFlowStatus, TicketType class MySQLDumpDataDetailSerializer(MySQLBaseOperateDetailSerializer): @@ -59,6 +59,9 @@ def format_ticket_data(self): def post_callback(self): flow = self.ticket.current_flow() + # 如果流程树运行不为成功，则忽略 + if flow.status != TicketFlowStatus.SUCCEEDED: + return # 往flow的detail中写入制品库的下载链接 dump_file_name = f"{flow.details['ticket_data']['dump_file_name']}.zip" flow.details["ticket_data"].update( diff --git a/dbm-ui/backend/ticket/builders/redis/redis_destroy.py b/dbm-ui/backend/ticket/builders/redis/redis_destroy.py index 09407f4327..756c557c48 100644 --- a/dbm-ui/backend/ticket/builders/redis/redis_destroy.py +++ b/dbm-ui/backend/ticket/builders/redis/redis_destroy.py @@ -56,7 +56,7 @@ class RedisInstanceDestroyDetailSerializer(SkipToRepresentationMixin, serializer class RedisInstanceDestroyFlowParamBuilder(builders.FlowParamBuilder): - controller = RedisController.fake_scene + controller = RedisController.redis_ins_shutdown @builders.BuilderFactory.register(TicketType.REDIS_INSTANCE_DESTROY, phase=ClusterPhase.DESTROY) diff --git a/dbm-ui/backend/ticket/builders/redis/redis_toolbox_load_module.py b/dbm-ui/backend/ticket/builders/redis/redis_toolbox_load_module.py new file mode 100644 index 0000000000..c1c172e71b --- /dev/null +++ b/dbm-ui/backend/ticket/builders/redis/redis_toolbox_load_module.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from backend.flow.engine.controller.redis import RedisController +from backend.ticket import builders +from backend.ticket.builders.common.base import DisplayInfoSerializer, SkipToRepresentationMixin +from backend.ticket.builders.redis.base import BaseRedisTicketFlowBuilder, ClusterValidateMixin +from backend.ticket.constants import LoadConfirmType, TicketType + + +class RedisLoadModuleSerializer(SkipToRepresentationMixin, serializers.Serializer): + """redis集群加载module""" + + class InfoSerializer(DisplayInfoSerializer, ClusterValidateMixin, serializers.Serializer): + + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + db_version = serializers.CharField(help_text=_("版本号")) + load_modules = serializers.ListField( + help_text=_("module类型列表"), + child=serializers.ChoiceField( + help_text=_("module类型"), choices=LoadConfirmType.get_choices(), default=LoadConfirmType.REDIS_BLOOM + ), + required=False, + ) + + bk_cloud_id = serializers.IntegerField(help_text=_("云区域ID")) + infos = serializers.ListField(help_text=_("批量操作参数列表"), child=InfoSerializer()) + + +class RedisLoadModuleParamBuilder(builders.FlowParamBuilder): + controller = RedisController.redis_cluster_load_modules + + def format_ticket_data(self): + super().format_ticket_data() + + +@builders.BuilderFactory.register(TicketType.REDIS_CLUSTER_LOAD_MODULES, is_apply=True) +class RedisLoadModuleFlowBuilder(BaseRedisTicketFlowBuilder): + serializer = RedisLoadModuleSerializer + inner_flow_builder = RedisLoadModuleParamBuilder + inner_flow_name = _("Redis 存量集群安装module") diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_clone_rules.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_clone_rules.py index b39ae16377..039eb71197 100644 --- a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_clone_rules.py +++ b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_clone_rules.py @@ -13,7 +13,6 @@ from rest_framework import serializers from backend.configuration.constants import DBType -from backend.db_meta.enums import ClusterType from backend.db_services.mysql.permission.constants import CloneClusterType from backend.ticket import builders from backend.ticket.builders.mysql.mysql_clone_rules import ( @@ -26,7 +25,10 @@ class TendbClusterCloneRulesSerializer(MySQLCloneRulesSerializer): clone_cluster_type = serializers.ChoiceField( - help_text=_("克隆集群类型"), choices=ClusterType.get_choices(), required=False, default=CloneClusterType.TendbCluster + help_text=_("集群类型"), + choices=CloneClusterType.get_choices(), + required=False, + default=CloneClusterType.TendbCluster, ) diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index e72dad6775..a91836ae38 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -261,7 +261,7 @@ def get_cluster_type_by_ticket(cls, ticket_type): TENDBSINGLE_STANDARDIZE = TicketEnumField("TENDBSINGLE_STANDARDIZE", _("TenDB Single 集群标准化"), register_iam=False) # noqa TENDBCLUSTER_DATA_MIGRATE = TicketEnumField("TENDBCLUSTER_DATA_MIGRATE", _("TenDB Cluster DB克隆"), _("数据处理")) TENDBCLUSTER_DUMP_DATA = TicketEnumField("TENDBCLUSTER_DUMP_DATA", _("TenDB Cluster 数据导出"), _("数据处理")) - TENDBCLUSTER_ACCOUNT_RULE_CHANGE = TicketEnumField("TENDBCLUSTER_ACCOUNT_RULE_CHANGE", _("TenDB Cluster 账号规则变更"), register_iam=False) # noqa + TENDBCLUSTER_ACCOUNT_RULE_CHANGE = TicketEnumField("TENDBCLUSTER_ACCOUNT_RULE_CHANGE", _("TenDB Cluster 授权规则变更"), register_iam=False) # noqa # Tbinlogdumper TBINLOGDUMPER_INSTALL = TicketEnumField("TBINLOGDUMPER_INSTALL", _("TBINLOGDUMPER 上架"), register_iam=False) @@ -343,7 +343,7 @@ def get_cluster_type_by_ticket(cls, ticket_type): REDIS_CLUSTER_STORAGES_CLI_CONNS_KILL = TicketEnumField("REDIS_CLUSTER_STORAGES_CLI_CONNS_KILL", _("Redis 集群存储层cli连接kill"), register_iam=False) # noqa REDIS_CLUSTER_RENAME_DOMAIN = TicketEnumField("REDIS_CLUSTER_RENAME_DOMAIN", _("Redis集群域名重命名"), _("集群维护")) REDIS_CLUSTER_MAXMEMORY_SET = TicketEnumField("REDIS_CLUSTER_MAXMEMORY_SET", _("Redis 集群设置maxmemory")) # noqa - REDIS_CLUSTER_LOAD_MODULES = TicketEnumField("REDIS_CLUSTER_LOAD_MODULES", _("Redis 集群加载modules")) # noqa + REDIS_CLUSTER_LOAD_MODULES = TicketEnumField("REDIS_CLUSTER_LOAD_MODULES", _("Redis 集群安装modules")) # noqa REDIS_TENDISPLUS_LIGHTNING_DATA = TicketEnumField("REDIS_TENDISPLUS_LIGHTNING_DATA", _("Tendisplus闪电导入数据"), _("集群维护")) # noqa REDIS_CLUSTER_INS_MIGRATE = TicketEnumField("REDIS_CLUSTER_INS_MIGRATE", _("Redis 集群指定实例迁移"), _("集群管理")) REDIS_SINGLE_INS_MIGRATE = TicketEnumField("REDIS_SINGLE_INS_MIGRATE", _("Redis 主从指定实例迁移"), _("集群管理")) @@ -568,6 +568,20 @@ class SwitchConfirmType(str, StructuredEnum): NO_CONFIRM = EnumField("no_confirm", _("无需确认")) +class LoadConfirmType(str, StructuredEnum): + """ + 加载Module类型 + """ + + REDIS_BLOOM = EnumField("redisbloom", _("redisbloom")) + REDIS_CELL = EnumField("rediscell", _("rediscell")) + FO4_LOCK = EnumField("fo4_lock", _("fo4_lock")) + FO4_MATCHMAKER = EnumField("fo4_matchmaker", _("fo4_matchmaker")) + FO4_UTIL = EnumField("fo4_util", _("fo4_util")) + JLSY_B2 = EnumField("jlsy-b2", _("jlsy-b2")) + REDIS_JSON = EnumField("redisjson", _("redisjson")) + + class SyncDisconnectSettingType(str, StructuredEnum): """ 同步断开设置 diff --git a/dbm-ui/backend/ticket/todos/__init__.py b/dbm-ui/backend/ticket/todos/__init__.py index bafffa53a9..701ce122b6 100644 --- a/dbm-ui/backend/ticket/todos/__init__.py +++ b/dbm-ui/backend/ticket/todos/__init__.py @@ -98,7 +98,6 @@ class ActionType(str, StructuredEnum): APPROVE = EnumField("APPROVE", _("确认执行")) TERMINATE = EnumField("TERMINATE", _("终止单据")) - RESOURCE_REAPPLY = EnumField("RESOURCE_REAPPLY", _("资源重新申请")) @dataclass diff --git a/dbm-ui/backend/ticket/todos/pipeline_todo.py b/dbm-ui/backend/ticket/todos/pipeline_todo.py index 6139717998..d965ef9812 100644 --- a/dbm-ui/backend/ticket/todos/pipeline_todo.py +++ b/dbm-ui/backend/ticket/todos/pipeline_todo.py @@ -16,7 +16,7 @@ from backend.constants import DEFAULT_SYSTEM_USER from backend.flow.engine.bamboo.engine import BambooEngine from backend.ticket import todos -from backend.ticket.constants import TodoType +from backend.ticket.constants import TodoStatus, TodoType from backend.ticket.exceptions import TodoWrongOperatorException from backend.ticket.models import TodoHistory from backend.ticket.todos import ActionType, BaseTodoContext @@ -44,7 +44,7 @@ def process(self, username, action, params): engine = BambooEngine(root_id=root_id) if action == ActionType.TERMINATE: - self.todo.set_terminated(username, action) + self.todo.set_status(username, TodoStatus.DONE_FAILED) # 终止时，直接将流程设置为失败 engine.force_fail_pipeline(node_id) return diff --git a/dbm-ui/frontend/src/common/const/clusterTypes.ts b/dbm-ui/frontend/src/common/const/clusterTypes.ts index dab6801cea..95d71c603d 100644 --- a/dbm-ui/frontend/src/common/const/clusterTypes.ts +++ b/dbm-ui/frontend/src/common/const/clusterTypes.ts @@ -13,7 +13,7 @@ export enum ClusterTypes { // redis export enum ClusterTypes { REDIS = 'redis', // Redis - PREDIXY_REDIS_CLUSTER = 'PredixyRedisCluster', // Redis集群 + PREDIXY_REDIS_CLUSTER = 'PredixyRedisCluster', // RedisCluster集群 PREDIXY_TENDISPLUS_CLUSTER = 'PredixyTendisplusCluster', // Tendisplus存储版集群 TWEMPROXY_REDIS_INSTANCE = 'TwemproxyRedisInstance', // TendisCache集群 TWEMPROXY_TENDIS_SSD_INSTANCE = 'TwemproxyTendisSSDInstance', // TendisSSD集群 diff --git a/dbm-ui/frontend/src/common/const/ticketTypes.ts b/dbm-ui/frontend/src/common/const/ticketTypes.ts index bb9c81c0e5..3485a6a2e3 100644 --- a/dbm-ui/frontend/src/common/const/ticketTypes.ts +++ b/dbm-ui/frontend/src/common/const/ticketTypes.ts @@ -79,6 +79,7 @@ export enum TicketTypes { REDIS_INSTANCE_OPEN = 'REDIS_INSTANCE_OPEN', // redis 主从集群启用 REDIS_INSTANCE_CLOSE = 'REDIS_INSTANCE_CLOSE', // redis 主从集群禁用 REDIS_INSTANCE_DESTROY = 'REDIS_INSTANCE_DESTROY', // redis 主从集群删除 + REDIS_CLUSTER_LOAD_MODULES = 'REDIS_CLUSTER_LOAD_MODULES', // redis 安装Module } export enum TicketTypes { TENDBCLUSTER_APPLY = 'TENDBCLUSTER_APPLY', diff --git a/dbm-ui/frontend/src/components/render-table/columns/select/index.vue b/dbm-ui/frontend/src/components/render-table/columns/select/index.vue index aa26025734..a65d5326ee 100644 --- a/dbm-ui/frontend/src/components/render-table/columns/select/index.vue +++ b/dbm-ui/frontend/src/components/render-table/columns/select/index.vue @@ -46,6 +46,7 @@ @@ -64,6 +65,7 @@ export interface IListItem { value: IKey; label: string; + disabled?: boolean; [x: string]: any; } diff --git a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/index.ts b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/index.ts index f17e1bc98e..b20ebf88f9 100644 --- a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/index.ts +++ b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/index.ts @@ -39,6 +39,7 @@ import { generateRedisDataCopyCheckRepairCloneData, generateRedisDataCopyCloneData, generateRedisDataStructureCloneData, + generateRedisInstallModule, generateRedisMasterSlaveSwitchCloneData, generateRedisOperationCloneData, generateRedisProxyScaleDownCloneData, @@ -91,6 +92,7 @@ export const generateCloneDataHandlerMap = { [TicketTypes.REDIS_KEYS_DELETE]: generateRedisOperationCloneData, // Redis 删除Key [TicketTypes.REDIS_BACKUP]: generateRedisOperationCloneData, // Redis 集群备份 [TicketTypes.REDIS_PURGE]: generateRedisOperationCloneData, + [TicketTypes.REDIS_CLUSTER_LOAD_MODULES]: generateRedisInstallModule, // Redis 安装Module [TicketTypes.MYSQL_AUTHORIZE_RULES]: generateMysqlAuthorizeRuleCloneData, // Mysql 集群授权 [TicketTypes.MYSQL_IMPORT_SQLFILE]: generateMysqlImportSqlFileCloneData, // Mysql SQL变更执行 [TicketTypes.MYSQL_CHECKSUM]: generateMysqlChecksumCloneData, // Mysql 数据校验 diff --git a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/mysql/versionLocalUpgrade.ts b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/mysql/versionLocalUpgrade.ts index db8e30aa45..bf1003d137 100644 --- a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/mysql/versionLocalUpgrade.ts +++ b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/mysql/versionLocalUpgrade.ts @@ -55,6 +55,7 @@ export async function generateMysqlVersionLocalUpgradeCloneData(ticketData: Tick currentVersion: clusters[clusterId].major_version, packageVersion: clusterListMap[clusterId].masters[0].version, moduleName: item.display_info.current_module_name, + moduleId: clusters[clusterId].db_module_id, cloudId: clusters[clusterId].bk_cloud_id, }, targetPackage: item.pkg_id, diff --git a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/mysql/versionMigrateUpgrade.ts b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/mysql/versionMigrateUpgrade.ts index 6af49b5c0d..d0ea40691a 100644 --- a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/mysql/versionMigrateUpgrade.ts +++ b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/mysql/versionMigrateUpgrade.ts @@ -46,6 +46,7 @@ export async function generateMysqlVersionMigrateUpgradeCloneData(ticketData: Ti currentVersion: clusters[clusterId].major_version, packageVersion: clusterListMap[clusterId].masters[0].version, moduleName: item.display_info.current_module_name, + moduleId: clusters[clusterId].db_module_id, cloudId: clusters[clusterId].bk_cloud_id, masterSlaveList: [ ...clusterListMap[clusterId].masters, diff --git a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/redis/index.ts b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/redis/index.ts index 08d2110cba..06fe5a9d17 100644 --- a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/redis/index.ts +++ b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/redis/index.ts @@ -6,6 +6,7 @@ export * from './clusterTypeUpdate'; export * from './dataCopy'; export * from './dataCopyCheckRepair'; export * from './dataStructure'; +export * from './installModule'; export * from './masterSlaveSwitch'; export * from './operation'; export * from './proxyScaleDown'; diff --git a/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/redis/installModule.ts b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/redis/installModule.ts new file mode 100644 index 0000000000..ac029404f9 --- /dev/null +++ b/dbm-ui/frontend/src/hooks/useTicketCloneInfo/generateCloneData/redis/installModule.ts @@ -0,0 +1,38 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for + * the specific language governing permissions and limitations under the License. + */ +import type { RedisInstallModuleDetails } from '@services/model/ticket/details/redis'; +import TicketModel from '@services/model/ticket/ticket'; + +import { random } from '@utils'; + +// Redis 安装Module +export function generateRedisInstallModule(ticketData: TicketModel) { + const { clusters, infos } = ticketData.details; + return Promise.resolve({ + tableDataList: infos.map((info) => { + const cluster = clusters[info.cluster_id]; + return { + rowKey: random(), + isLoading: false, + srcCluster: cluster.immute_domain, + clusterId: info.cluster_id, + bkCloudId: cluster.bk_cloud_id, + clusterType: cluster.cluster_type, + clusterTypeName: cluster.cluster_type_name, + dbVersion: info.db_version, + loadModules: info.load_modules, + }; + }), + remark: ticketData.remark, + }); +} diff --git a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/SqlServer.vue b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/SqlServer.vue index 6482b4ff65..fd17622b2f 100644 --- a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/SqlServer.vue +++ b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/SqlServer.vue @@ -14,7 +14,9 @@ {{ t('集群视图') }} - + @@ -22,7 +24,9 @@ - + @@ -34,6 +38,7 @@