From b149e819474f9b6a8a1722f2ee90b9558a42c73f Mon Sep 17 00:00:00 2001 From: yksitu <1297650644@qq.com> Date: Tue, 26 Nov 2024 18:41:15 +0800 Subject: [PATCH] =?UTF-8?q?fix(mysql):=20=E4=BC=98=E5=8C=96proxy=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E6=B5=81=E7=A8=8B=20#8165?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../subcmd/mysqlcmd/clone_client_grant.go | 4 - .../scene/mysql/mysql_proxy_cluster_switch.py | 173 ++++++++---------- .../mysql/clone_proxy_client_in_backend.py | 102 +++++++++++ .../mysql/clone_proxy_user_in_cluster.py | 86 +++++++++ .../mysql/drop_proxy_client_in_backend.py | 75 ++++++++ .../collections/mysql/set_backend_in_porxy.py | 54 ++++++ .../flow/utils/mysql/mysql_act_dataclass.py | 44 +++++ .../flow/utils/mysql/mysql_commom_query.py | 87 +++++++++ 8 files changed, 521 insertions(+), 104 deletions(-) create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mysql/clone_proxy_client_in_backend.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mysql/clone_proxy_user_in_cluster.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mysql/drop_proxy_client_in_backend.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mysql/set_backend_in_porxy.py diff --git a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/clone_client_grant.go b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/clone_client_grant.go index 8650471e96..73251d8474 100644 --- a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/clone_client_grant.go +++ b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/clone_client_grant.go @@ -65,10 +65,6 @@ func (g *CloneClineGrantAct) Run() (err error) { FunName: "克隆client权限", Func: g.Service.CloneTargetClientPriv, }, - { - FunName: "回收旧client权限", - Func: g.Service.DropOriginClientPriv, - }, } if err := steps.Run(); err != nil { diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_proxy_cluster_switch.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_proxy_cluster_switch.py index 021bae366a..a14f215b07 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_proxy_cluster_switch.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_proxy_cluster_switch.py @@ -16,7 +16,8 @@ from django.utils.translation import ugettext as _ from backend.configuration.constants import DBType -from backend.db_meta.enums import ClusterEntryType, ClusterType, InstanceInnerRole, InstanceStatus +from backend.constants import IP_PORT_DIVIDER +from backend.db_meta.enums import ClusterEntryType, ClusterType, InstanceInnerRole from backend.db_meta.models import Cluster, ProxyInstance, StorageInstance from backend.flow.engine.bamboo.scene.common.builder import Builder, SubBuilder from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList @@ -24,17 +25,31 @@ from backend.flow.plugins.components.collections.common.delete_cc_service_instance import DelCCServiceInstComponent from backend.flow.plugins.components.collections.common.pause import PauseComponent from backend.flow.plugins.components.collections.mysql.clear_machine import MySQLClearMachineComponent +from backend.flow.plugins.components.collections.mysql.clone_proxy_client_in_backend import ( + CloneProxyUsersInBackendComponent, +) +from backend.flow.plugins.components.collections.mysql.clone_proxy_user_in_cluster import ( + CloneProxyUsersInClusterComponent, +) from backend.flow.plugins.components.collections.mysql.dns_manage import MySQLDnsManageComponent +from backend.flow.plugins.components.collections.mysql.drop_proxy_client_in_backend import ( + DropProxyUsersInBackendComponent, +) from backend.flow.plugins.components.collections.mysql.exec_actuator_script import ExecuteDBActuatorScriptComponent from backend.flow.plugins.components.collections.mysql.mysql_db_meta import MySQLDBMetaComponent +from backend.flow.plugins.components.collections.mysql.set_backend_in_porxy import SetBackendInProxyComponent from backend.flow.plugins.components.collections.mysql.trans_flies import TransFileComponent from backend.flow.utils.mysql.mysql_act_dataclass import ( + CloneProxyClientInBackendKwargs, + CloneProxyUsersKwargs, CreateDnsKwargs, DBMetaOPKwargs, DelServiceInstKwargs, DownloadMediaKwargs, + DropProxyUsersInBackendKwargs, ExecActuatorKwargs, RecycleDnsRecordKwargs, + SetBackendInProxyKwargs, ) from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload from backend.flow.utils.mysql.mysql_db_meta import MySQLDBMeta @@ -67,33 +82,24 @@ def __get_switch_cluster_info(cluster_id: int, origin_proxy_ip: str, target_prox """ cluster = Cluster.objects.get(id=cluster_id) - # 选择集群标记running状态的proxy实例，作为流程中克隆权限的依据, 排除待替换的ip - template_proxy = ( - ProxyInstance.objects.filter(cluster=cluster, status=InstanceStatus.RUNNING.value) - .exclude(machine__ip=origin_proxy_ip) - .all()[0] - ) - mysql_ip_list = StorageInstance.objects.filter(cluster=cluster).all() + origin_proxy = ProxyInstance.objects.filter(cluster=cluster, machine__ip=origin_proxy_ip) master = StorageInstance.objects.get(cluster=cluster, instance_inner_role=InstanceInnerRole.MASTER) - dns_list = template_proxy.bind_entry.filter(cluster_entry_type=ClusterEntryType.DNS.value).all() + dns_list = origin_proxy.bind_entry.filter(cluster_entry_type=ClusterEntryType.DNS.value).all() return { "id": cluster_id, "bk_cloud_id": cluster.bk_cloud_id, "name": cluster.name, "cluster_type": cluster.cluster_type, - "template_proxy_ip": template_proxy.machine.ip, # 集群所有的backend实例的端口是一致的，获取第一个对象的端口信息即可 - "mysql_ip_list": [m.machine.ip for m in mysql_ip_list], "mysql_port": master.port, # 每套集群的proxy端口必须是相同的，取第一个proxy的端口信息即可 - "proxy_port": template_proxy.port, + "proxy_port": origin_proxy.port, "origin_proxy_ip": origin_proxy_ip, "target_proxy_ip": target_proxy_ip, # 新的proxy配置后端ip "set_backend_ip": master.machine.ip, "add_domain_list": [i.entry for i in dns_list], - "is_drop": True, } @staticmethod @@ -113,15 +119,8 @@ def __get_proxy_install_ports(cluster_ids: list) -> list: def switch_mysql_cluster_proxy_flow(self): """ 定义mysql集群proxy替换实例流程 - 增加单据临时ADMIN账号的添加和删除逻辑 """ - cluster_ids = [] - for i in self.data["infos"]: - cluster_ids.extend(i["cluster_ids"]) - - mysql_proxy_cluster_add_pipeline = Builder( - root_id=self.root_id, data=self.data, need_random_pass_cluster_ids=list(set(cluster_ids)) - ) + mysql_proxy_cluster_add_pipeline = Builder(root_id=self.root_id, data=self.data) sub_pipelines = [] # 多集群操作时循环加入集群proxy替换子流程 @@ -133,16 +132,6 @@ def switch_mysql_cluster_proxy_flow(self): sub_flow_context["proxy_ports"] = self.__get_proxy_install_ports(cluster_ids=info["cluster_ids"]) sub_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(sub_flow_context)) - # 初始化同机替换的proxy集群信息 - clusters = [ - self.__get_switch_cluster_info( - cluster_id=cluster_id, - origin_proxy_ip=info["origin_proxy_ip"]["ip"], - target_proxy_ip=info["target_proxy_ip"]["ip"], - ) - for cluster_id in info["cluster_ids"] - ] - # 拼接执行原子任务活动节点需要的通用的私有参数结构体, 减少代码重复率，但引用时注意内部参数值传递的问题 exec_act_kwargs = ExecActuatorKwargs( cluster_type=ClusterType.TenDBHA, @@ -190,89 +179,62 @@ def switch_mysql_cluster_proxy_flow(self): act_component_code=ExecuteDBActuatorScriptComponent.code, kwargs=asdict(exec_act_kwargs), ) + # 后续流程需要在这里加一个暂停节点，让用户在合适的时间执行切换 + sub_pipeline.add_act(act_name=_("人工确认"), act_component_code=PauseComponent.code, kwargs={}) # 阶段2 根据需要替换的proxy的集群，依次添加 - add_proxy_sub_list = [] - for cluster in clusters: + switch_proxy_sub_list = [] + for cluster_id in info["cluster_ids"]: # 拼接子流程需要全局参数 sub_sub_flow_context = copy.deepcopy(self.data) sub_sub_flow_context.pop("infos") + # 获取集群的实例信息 + cluster = self.__get_switch_cluster_info( + cluster_id=cluster_id, + target_proxy_ip=info["target_proxy_ip"]["ip"], + origin_proxy_ip=info["origin_proxy_ip"]["ip"], + ) + # 针对集群维度声明替换子流程 switch_proxy_sub_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(sub_sub_flow_context)) - # 拼接替换proxy节点需要的通用的私有参数结构体, 减少代码重复率，但引用时注意内部参数值传递的问题 - switch_proxy_sub_act_kwargs = ExecActuatorKwargs( - bk_cloud_id=cluster["bk_cloud_id"], - cluster=cluster, - ) - switch_proxy_sub_pipeline.add_act( - act_name=_("下发db-actuator介质"), - act_component_code=TransFileComponent.code, + act_name=_("新的proxy配置后端实例[{}]".format(info["target_proxy_ip"]["ip"])), + act_component_code=SetBackendInProxyComponent.code, kwargs=asdict( - DownloadMediaKwargs( + SetBackendInProxyKwargs( + proxys=[f"{info['target_proxy_ip']['ip']}{IP_PORT_DIVIDER}{cluster['proxy_port']}"], bk_cloud_id=cluster["bk_cloud_id"], - exec_ip=[cluster["template_proxy_ip"]] + cluster["mysql_ip_list"], - file_list=GetFileList(db_type=DBType.MySQL).get_db_actuator_package(), - ), + backend_host=cluster["set_backend_ip"], + backend_port=cluster["mysql_port"], + ) ), ) - switch_proxy_sub_act_kwargs.exec_ip = cluster["target_proxy_ip"] - switch_proxy_sub_act_kwargs.get_mysql_payload_func = MysqlActPayload.get_set_proxy_backends.__name__ - switch_proxy_sub_pipeline.add_act( - act_name=_("新的proxy配置后端实例"), - act_component_code=ExecuteDBActuatorScriptComponent.code, - kwargs=asdict(switch_proxy_sub_act_kwargs), - ) - - switch_proxy_sub_act_kwargs.exec_ip = cluster["template_proxy_ip"] - switch_proxy_sub_act_kwargs.get_mysql_payload_func = ( - MysqlActPayload.get_clone_proxy_user_payload.__name__ - ) switch_proxy_sub_pipeline.add_act( act_name=_("克隆proxy用户白名单"), - act_component_code=ExecuteDBActuatorScriptComponent.code, - kwargs=asdict(switch_proxy_sub_act_kwargs), - ) - - acts_list = [] - for cluster_mysql_ip in cluster["mysql_ip_list"]: - switch_proxy_sub_act_kwargs.exec_ip = cluster_mysql_ip - switch_proxy_sub_act_kwargs.get_mysql_payload_func = ( - MysqlActPayload.get_clone_client_grant_payload.__name__ - ) - acts_list.append( - { - "act_name": _("集群对新的proxy添加权限"), - "act_component_code": ExecuteDBActuatorScriptComponent.code, - "kwargs": asdict(switch_proxy_sub_act_kwargs), - } - ) - switch_proxy_sub_pipeline.add_parallel_acts(acts_list=acts_list) - - add_proxy_sub_list.append( - switch_proxy_sub_pipeline.build_sub_process(sub_name=_("{}集群添加proxy实例").format(cluster["name"])) + act_component_code=CloneProxyUsersInClusterComponent.code, + kwargs=asdict( + CloneProxyUsersKwargs( + cluster_id=cluster["id"], + target_proxy_host=info["target_proxy_ip"]["ip"], + proxy_port=cluster["proxy_port"], + ) + ), ) - sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=add_proxy_sub_list) - - # 后续流程需要在这里加一个暂停节点，让用户在合适的时间执行切换 - sub_pipeline.add_act(act_name=_("人工确认"), act_component_code=PauseComponent.code, kwargs={}) - - # 阶段3 根据集群维度切换域名 - switch_dns_sub_list = [] - for cluster in clusters: - - # 拼接子流程需要全局参数 - sub_sub_flow_context = copy.deepcopy(self.data) - sub_sub_flow_context.pop("infos") - - # 针对集群维度声明替换子流程 - switch_cluster_dns_pipeline = SubBuilder( - root_id=self.root_id, data=copy.deepcopy(sub_sub_flow_context) + switch_proxy_sub_pipeline.add_act( + act_name=_("集群对新的proxy添加权限"), + act_component_code=CloneProxyUsersInBackendComponent.code, + kwargs=asdict( + CloneProxyClientInBackendKwargs( + cluster_id=cluster["id"], + target_proxy_host=info["target_proxy_ip"]["ip"], + origin_proxy_host=info["origin_proxy_ip"]["ip"], + ) + ), ) acts_list = [] @@ -292,9 +254,9 @@ def switch_mysql_cluster_proxy_flow(self): ), } ) - switch_cluster_dns_pipeline.add_parallel_acts(acts_list=acts_list) + switch_proxy_sub_pipeline.add_parallel_acts(acts_list=acts_list) - switch_cluster_dns_pipeline.add_act( + switch_proxy_sub_pipeline.add_act( act_name=_("回收旧proxy集群映射"), act_component_code=MySQLDnsManageComponent.code, kwargs=asdict( @@ -306,11 +268,22 @@ def switch_mysql_cluster_proxy_flow(self): ), ) - switch_dns_sub_list.append( - switch_cluster_dns_pipeline.build_sub_process(sub_name=_("{}集群切换proxy域名").format(cluster["name"])) + switch_proxy_sub_pipeline.add_act( + act_name=_("回收旧proxy在backend权限"), + act_component_code=DropProxyUsersInBackendComponent.code, + kwargs=asdict( + DropProxyUsersInBackendKwargs( + cluster_id=cluster["id"], + origin_proxy_host=info["origin_proxy_ip"]["ip"], + ), + ), + ) + + switch_proxy_sub_list.append( + switch_proxy_sub_pipeline.build_sub_process(sub_name=_("{}集群替换proxy实例").format(cluster["name"])) ) - sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=switch_dns_sub_list) + sub_pipeline.add_parallel_sub_pipeline(sub_flow_list=switch_proxy_sub_list) # 先把新的节点数据写入 sub_pipeline.add_act( @@ -373,7 +346,7 @@ def switch_mysql_cluster_proxy_flow(self): sub_pipelines.append(sub_pipeline.build_sub_process(sub_name=_("替换proxy子流程"))) mysql_proxy_cluster_add_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) - mysql_proxy_cluster_add_pipeline.run_pipeline(is_drop_random_user=True) + mysql_proxy_cluster_add_pipeline.run_pipeline() def proxy_reduce_sub_flow(self, cluster_id: int, bk_cloud_id: int, origin_proxy_ip: str, origin_proxy_port: int): """ diff --git a/dbm-ui/backend/flow/plugins/components/collections/mysql/clone_proxy_client_in_backend.py b/dbm-ui/backend/flow/plugins/components/collections/mysql/clone_proxy_client_in_backend.py new file mode 100644 index 0000000000..e5029134e5 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mysql/clone_proxy_client_in_backend.py @@ -0,0 +1,102 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from django.utils.translation import ugettext as _ +from pipeline.component_framework.component import Component + +from backend.components import DRSApi +from backend.db_meta.exceptions import ClusterNotExistException +from backend.db_meta.models import Cluster, StorageInstance +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.plugins.components.collections.mysql.drop_proxy_client_in_backend import ( + DropProxyUsersInBackendService, +) +from backend.flow.utils.mysql.mysql_commom_query import show_privilege_for_user + +logger = logging.getLogger("flow") + + +class CloneProxyUsersInBackendService(BaseService): + """ + 在集群内,根据旧proxy权限，克隆一份对新proxy的权限。proxy替换和添加单据调用 + 操作步骤： + 1: 先处理新proxy在集群所有backend节点的残留权限，避免冲突。因为理论上新proxy的授权出现在集群上 + 2：根据旧proxy的授权模式，给新proxy授权一份 + """ + + @staticmethod + def clone_proxy_client(origin_proxy_host: str, target_proxy_host: str, backend: StorageInstance, cluster: Cluster): + """ + 克隆proxy权限 + """ + result, grant_sqls = show_privilege_for_user( + host=origin_proxy_host, instance=backend, db_version=cluster.major_version + ) + if not result: + return f"[{backend.ip_port}] show proxy client[{origin_proxy_host}] failed" + + # 执行授权 + res = DRSApi.rpc( + { + "addresses": [backend.ip_port], + "cmds": [i.replace(origin_proxy_host, target_proxy_host, -1) for i in grant_sqls], + "force": False, + "bk_cloud_id": backend.machine.bk_cloud_id, + } + ) + if res["error_msg"]: + return f"[{backend.ip_port}] clone proxy client[{target_proxy_host}] failed: [{res['error_msg']}]" + + return "" + + def _execute(self, data, parent_data, callback=None) -> bool: + kwargs = data.get_one_of_inputs("kwargs") + global_data = data.get_one_of_inputs("global_data") + try: + cluster = Cluster.objects.get(id=kwargs["cluster_id"]) + except Cluster.DoesNotExist: + raise ClusterNotExistException( + cluster_id=kwargs["cluster_id"], bk_biz_id=int(global_data["bk_biz_id"]), message=_("集群不存在") + ) + err_no = False + for s in cluster.storageinstance_set.all(): + # 1： 先处理新proxy在集群所有backend节点的残留权限 + status, err = DropProxyUsersInBackendService.drop_proxy_client(kwargs["target_proxy_host"], s) + if not status: + self.log_error(err) + err_no = True + continue + + # 2: 根据旧proxy的授权模式，给新proxy授权一份 + log = self.clone_proxy_client( + origin_proxy_host=kwargs["origin_proxy_host"], + target_proxy_host=kwargs["target_proxy_host"], + backend=s, + cluster=cluster, + ) + if log: + self.log_error(log) + err_no = True + continue + + self.log_info(f"[{s.ip_port}]clone proxy client [{kwargs['target_proxy_host']}] successfully") + + if err_no: + return False + + return True + + +class CloneProxyUsersInBackendComponent(Component): + name = __name__ + code = "clone_proxy_client_in_backend" + bound_service = CloneProxyUsersInBackendService diff --git a/dbm-ui/backend/flow/plugins/components/collections/mysql/clone_proxy_user_in_cluster.py b/dbm-ui/backend/flow/plugins/components/collections/mysql/clone_proxy_user_in_cluster.py new file mode 100644 index 0000000000..5b5583fc2f --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mysql/clone_proxy_user_in_cluster.py @@ -0,0 +1,86 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from django.utils.translation import ugettext as _ +from pipeline.component_framework.component import Component + +from backend.components import DRSApi +from backend.db_meta.enums import InstanceStatus, MachineType +from backend.db_meta.exceptions import ClusterNotExistException +from backend.db_meta.models import Cluster +from backend.flow.consts import AUTH_ADDRESS_DIVIDER +from backend.flow.plugins.components.collections.mysql.clone_user import CloneUserService + +logger = logging.getLogger("flow") + + +class CloneProxyUsersInClusterService(CloneUserService): + """ + 场景化处理：集群内克隆proxy的用户白名单，同时给后端mysql对白名单授权，提供proxy替换和添加使用 + 理论上某个状态点，集群的所有proxy的授权名单都是同等的。 + 所以这里会即时计算running状态的proxy实例作为权限克隆源，保证克隆时集群的权限的最新可用的。 + """ + + def _calc_running_status_in_cluster(self, cluster: Cluster): + """ + 计算集群可用的proxy实例，作为权限克隆源 + """ + proxys = cluster.proxyinstance_set.filter(status=InstanceStatus.RUNNING) + if not proxys: + # 如果在dbm系统找不到running状态的proxy实例，则报异常 + self.log_error(f"no running-status-proxys in cluster[{cluster.immute_domain}]") + return None + for proxy in proxys: + res = DRSApi.proxyrpc( + { + "addresses": [proxy.ip_port], + "cmds": ["select 1"], + "force": False, + "bk_cloud_id": cluster.bk_cloud_id, + } + ) + if not res["error_msg"]: + self.log_info(f"get running proxy [{proxy.ip_port}]") + return proxy + + self.log_error(f"no running proxy in cluster [{cluster.immute_domain}] with drs-check") + return None + + def _execute(self, data, parent_data, callback=None) -> bool: + kwargs = data.get_one_of_inputs("kwargs") + global_data = data.get_one_of_inputs("global_data") + try: + cluster = Cluster.objects.get(id=kwargs["cluster_id"]) + except Cluster.DoesNotExist: + raise ClusterNotExistException( + cluster_id=kwargs["cluster_id"], bk_biz_id=int(global_data["bk_biz_id"]), message=_("集群不存在") + ) + temp_proxy = self._calc_running_status_in_cluster(cluster) + if not temp_proxy: + return False + + # 执行clone-user接口 + data.get_one_of_inputs("kwargs")["clone_data"] = [ + { + "source": temp_proxy.ip_port, + "target": f"{kwargs['target_proxy_host']}{AUTH_ADDRESS_DIVIDER}{kwargs['proxy_port']}", + "machine_type": MachineType.PROXY.value, + "bk_cloud_id": cluster.bk_cloud_id, + } + ] + return super()._execute(data, parent_data) + + +class CloneProxyUsersInClusterComponent(Component): + name = __name__ + code = "clone_proxy_users_in_cluster" + bound_service = CloneProxyUsersInClusterService diff --git a/dbm-ui/backend/flow/plugins/components/collections/mysql/drop_proxy_client_in_backend.py b/dbm-ui/backend/flow/plugins/components/collections/mysql/drop_proxy_client_in_backend.py new file mode 100644 index 0000000000..bfc4a93ad8 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mysql/drop_proxy_client_in_backend.py @@ -0,0 +1,75 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from django.utils.translation import ugettext as _ +from pipeline.component_framework.component import Component + +from backend.components import DRSApi +from backend.db_meta.exceptions import ClusterNotExistException +from backend.db_meta.models import Cluster, StorageInstance +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils.mysql.mysql_commom_query import show_user_host_for_host + +logger = logging.getLogger("flow") + + +class DropProxyUsersInBackendService(BaseService): + """ + 在集群内清理旧proxy的后端权限 + """ + + @staticmethod + def drop_proxy_client(origin_proxy_host: str, backend: StorageInstance): + """ + 计算集群可用的proxy实例，作为权限克隆源 + """ + result, user_hosts = show_user_host_for_host(host=origin_proxy_host, instance=backend) + if not result: + return False, f"[{backend.ip_port}] get user_host[{origin_proxy_host}] failed" + + # 执行删除旧proxy client + res = DRSApi.rpc( + { + "addresses": [backend.ip_port], + "cmds": [f"drop user {i};" for i in user_hosts], + "force": False, + "bk_cloud_id": backend.machine.bk_cloud_id, + } + ) + if res["error_msg"]: + return ( + False, + f"[{backend.ip_port}] drop old proxy client[{origin_proxy_host}] failed: [{res['error_msg']}]", + ) + + def _execute(self, data, parent_data, callback=None) -> bool: + kwargs = data.get_one_of_inputs("kwargs") + global_data = data.get_one_of_inputs("global_data") + try: + cluster = Cluster.objects.get(id=kwargs["cluster_id"]) + except Cluster.DoesNotExist: + raise ClusterNotExistException( + cluster_id=kwargs["cluster_id"], bk_biz_id=int(global_data["bk_biz_id"]), message=_("集群不存在") + ) + for s in cluster.storageinstance_set.all(): + status, err = self.drop_proxy_client(kwargs["origin_proxy_host"], s) + if not status: + self.log_error(err) + return False + self.log_info(f"[{s.ip_port}]drop old proxy client [{kwargs['origin_proxy_host']}] successfully") + return True + + +class DropProxyUsersInBackendComponent(Component): + name = __name__ + code = "drop_proxy_users_in_backend" + bound_service = DropProxyUsersInBackendService diff --git a/dbm-ui/backend/flow/plugins/components/collections/mysql/set_backend_in_porxy.py b/dbm-ui/backend/flow/plugins/components/collections/mysql/set_backend_in_porxy.py new file mode 100644 index 0000000000..4206733c4a --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mysql/set_backend_in_porxy.py @@ -0,0 +1,54 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from pipeline.component_framework.component import Component + +from backend.components import DRSApi +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils.mysql.mysql_commom_query import check_backend_in_proxy + +logger = logging.getLogger("flow") + + +class SetBackendInProxyService(BaseService): + """ + 在新proxy设置backend后端信息，设置之前需要保证proxy的backend是1.1.1.1:3306 + 如果不是则证明不是最新的，则作为异常退出 + """ + + def _execute(self, data, parent_data, callback=None) -> bool: + kwargs = data.get_one_of_inputs("kwargs") + if not check_backend_in_proxy(proxys=kwargs["proxys"], bk_cloud_id=int(kwargs["bk_cloud_id"])): + # 检测不通过，异常 + return False + + # 刷新backend + res = DRSApi.proxyrpc( + { + "addresses": kwargs["proxys"], + "cmds": [f"refresh_backends('{kwargs['backend_host']}:{kwargs['backend_port']}',1)"], + "force": False, + "bk_cloud_id": int(kwargs["bk_cloud_id"]), + } + ) + if not res["error_msg"]: + self.log_error(f"the proxy [{kwargs['proxys']}] set backend failed:{res['error_msg']}") + return False + + self.log_info(f"the proxy [{kwargs['proxys']}] set backend successfully") + return True + + +class SetBackendInProxyComponent(Component): + name = __name__ + code = "set_backend_in_proxy" + bound_service = SetBackendInProxyService diff --git a/dbm-ui/backend/flow/utils/mysql/mysql_act_dataclass.py b/dbm-ui/backend/flow/utils/mysql/mysql_act_dataclass.py index cefe498d76..7e6757b2ca 100644 --- a/dbm-ui/backend/flow/utils/mysql/mysql_act_dataclass.py +++ b/dbm-ui/backend/flow/utils/mysql/mysql_act_dataclass.py @@ -510,3 +510,47 @@ class MysqlSyncMasterKwargs: is_gtid: bool = False is_add_any: bool = False is_master_add_priv: bool = True + + +@dataclass +class SetBackendInProxyKwargs: + """ + 定义set_backend_in_proxy活动节点的私有变量结构体 + """ + + proxys: List[str] + bk_cloud_id: int + backend_host: str + backend_port: int + + +@dataclass +class CloneProxyUsersKwargs: + """ + 定义clone_proxy_users_in_cluster活动节点的私有变量结构体 + """ + + cluster_id: int + target_proxy_host: str + proxy_port: int + + +@dataclass +class CloneProxyClientInBackendKwargs: + """ + 定义clone_proxy_client_in_backend活动节点的私有变量结构体 + """ + + cluster_id: int + target_proxy_host: str + origin_proxy_host: str + + +@dataclass +class DropProxyUsersInBackendKwargs: + """ + 定义drop_proxy_users_in_backend活动节点的私有变量结构体 + """ + + cluster_id: int + origin_proxy_host: str diff --git a/dbm-ui/backend/flow/utils/mysql/mysql_commom_query.py b/dbm-ui/backend/flow/utils/mysql/mysql_commom_query.py index 355935d48a..f73a781ab2 100644 --- a/dbm-ui/backend/flow/utils/mysql/mysql_commom_query.py +++ b/dbm-ui/backend/flow/utils/mysql/mysql_commom_query.py @@ -9,11 +9,14 @@ specific language governing permissions and limitations under the License. """ import logging.config +from typing import List from django.utils.translation import gettext as _ from backend.components.db_remote_service.client import DRSApi from backend.constants import IP_PORT_DIVIDER +from backend.db_meta.models import StorageInstance +from backend.flow.utils.mysql.mysql_version_parse import mysql_version_parse logger = logging.getLogger("flow") @@ -44,3 +47,87 @@ def query_mysql_variables(host: str, port: int, bk_cloud_id: int): val = var_item["Value"] var_map[var_name] = val return var_map + + +def show_user_host_for_host(host: str, instance: StorageInstance): + """ + 根据host查询账号信息 + """ + res = DRSApi.rpc( + { + "addresses": [instance.ip_port], + "cmds": [f"select concat('`',user,'`@`',host,'`') as user_host from mysql.user where host = '{host}'"], + "force": False, + "bk_cloud_id": instance.machine.bk_cloud_id, + } + ) + if res["error_msg"]: + logger.error(f"[{instance.ip_port}] get user info [{host}] failed: [{res['error_msg']}]") + return False, [] + + return True, list(res[0]["cmd_results"][0]["table_data"].values()) + + +def show_privilege_for_user(db_version: str, host: str, instance: StorageInstance): + """ + 根据user_host 在实例查询授权情况，并拼接成对应的版本的授权语句 + """ + result, user_hosts = show_user_host_for_host(host=host, instance=instance) + if not result: + return result, [] + + grants_sql = [] + if mysql_version_parse(db_version) >= mysql_version_parse("5.7"): + res = DRSApi.rpc( + { + "addresses": [instance.ip_port], + "cmds": [f"show create user {u} " for u in user_hosts], + "force": False, + "bk_cloud_id": instance.machine.bk_cloud_id, + } + ) + if res["error_msg"]: + logger.error(f"[{instance.ip_port}] show create user failed: [{res['error_msg']}]") + return False, [] + grants_sql.extend(list(res[0]["cmd_results"][0]["table_data"].values())) + + res = DRSApi.rpc( + { + "addresses": [instance.ip_port], + "cmds": [f"show grants for {u} " for u in user_hosts], + "force": False, + "bk_cloud_id": instance.machine.bk_cloud_id, + } + ) + if res["error_msg"]: + logger.error(f"[{instance.ip_port}] show grants failed: [{res['error_msg']}]") + return False, [] + + grants_sql.extend(list(res[0]["cmd_results"][0]["table_data"].values())) + return True, grants_sql + + +def check_backend_in_proxy(proxys: List[str], bk_cloud_id: int): + """ + 检测传入的proxy是否1.1.1.1:3306 + """ + res = DRSApi.rpc( + { + "addresses": proxys, + "cmds": ["SELECT * FROM backends;"], + "force": False, + "bk_cloud_id": bk_cloud_id, + } + ) + if res["error_msg"]: + logger.error(f"get proxy backends failed: [{res['error_msg']}]") + return False + + is_pass = True + for i in res[0]["cmd_results"]: + backend_address = str(i["table_data"][0]["address"]).strip() + if backend_address != "1.1.1.1:3306": + logger.error(f"[{res[0]['address']}] the backends is not empty [{backend_address}] ") + is_pass = False + + return is_pass