Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(dbm-services): 动态加载mysql pod 启动参数 #8218 #8258

Merged
merged 1 commit into from
Nov 29, 2024
Merged

feat(dbm-services): 动态加载mysql pod 启动参数 #8218 #8258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 98 additions & 30 deletions dbm-services/mysql/db-simulation/app/service/kubernets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"net/url"
"os"
"path"
"regexp"
"strings"
"time"

Expand Down Expand Up @@ -76,6 +77,8 @@ type ClusterPodSets struct {
DbPodSets
}

var startArgsSplitRe *regexp.Regexp

func init() {
logger.Info("start init bcs client ")
Kcs.RestConfig = &rest.Config{
Expand All @@ -94,6 +97,7 @@ func init() {
}
Kcs.Cli = clientSet
Kcs.Namespace = config.GAppConfig.Bcs.NameSpace
startArgsSplitRe = regexp.MustCompile(` |,`)
}

// NewDbPodSets new db pod sets
Expand Down Expand Up @@ -122,18 +126,6 @@ func (k *DbPodSets) getCreateClusterSqls() []string {
// getClusterPodContanierSpec create cluster pod container spec
// nolint
func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Container {
mysqldStartArgs := []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--log_bin_trust_function_creators",
"--port=20000",
"--max_allowed_packet=1073741824",
"--sql-mode=",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"}
if cmutil.MySQLVersionParse(mysqlVersion) >= cmutil.MySQLVersionParse("8.0.0") {
mysqldStartArgs = append(mysqldStartArgs, "--default-authentication-plugin=mysql_native_password")
}
return []v1.Container{
{
Name: "backend",
Expand All @@ -144,7 +136,7 @@ func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Contain
Resources: k.getResourceLimit(),
ImagePullPolicy: v1.PullIfNotPresent,
Image: k.DbImage,
Args: mysqldStartArgs,
Args: k.getbackendStartArgs(mysqlVersion),
ReadinessProbe: &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Expand All @@ -163,14 +155,7 @@ func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Contain
Resources: k.getResourceLimit(),
ImagePullPolicy: v1.PullIfNotPresent,
Image: k.SpiderImage,
Args: []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--log_bin_trust_function_creators",
"--port=25000",
"--max_allowed_packet=1073741824",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"},
Args: k.getSpiderStartArgs(),
ReadinessProbe: &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Expand All @@ -190,11 +175,7 @@ func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Contain
Resources: k.gettdbctlResourceLimit(),
ImagePullPolicy: v1.PullIfNotPresent,
Image: k.TdbCtlImage,
Args: []string{"mysqld", "--defaults-file=/etc/my.cnf", "--port=26000", "--tc-admin=1",
"--dbm-allow-standalone-primary",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"},
Args: k.getTdbctlStartArgs(),
ReadinessProbe: &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Expand All @@ -208,6 +189,77 @@ func (k *DbPodSets) getClusterPodContanierSpec(mysqlVersion string) []v1.Contain
}
}

func (k *DbPodSets) getTdbctlStartArgs() (args []string) {
args = []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--port=26000",
"--tc-admin=1",
"--dbm-allow-standalone-primary",
"--max_allowed_packet=1073741824",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"}
dbArgs, err := model.GetStartArsg("tdbctl", LatestVersion)
if err != nil {
logger.Warn("get tdbctl start args failed %s", err.Error())
return
}
for _, arg := range startArgsSplitRe.Split(dbArgs, -1) {
if lo.IsNotEmpty(arg) {
args = append(args, strings.TrimSpace(arg))
}
}
return
}

func (k *DbPodSets) getSpiderStartArgs() (args []string) {
args = []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--log_bin_trust_function_creators",
"--port=25000",
"--max_allowed_packet=1073741824",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"}
dbArgs, err := model.GetStartArsg("spider", LatestVersion)
if err != nil {
logger.Warn("get spider start args failed %s", err.Error())
return
}
for _, arg := range startArgsSplitRe.Split(dbArgs, -1) {
if lo.IsNotEmpty(arg) {
args = append(args, strings.TrimSpace(arg))
}
}
return
}

func (k *DbPodSets) getbackendStartArgs(mysqlVersion string) (args []string) {
args = []string{"mysqld",
"--defaults-file=/etc/my.cnf",
"--log_bin_trust_function_creators",
"--port=20000",
"--max_allowed_packet=1073741824",
"--sql-mode=",
fmt.Sprintf("--character-set-server=%s",
k.BaseInfo.Charset),
"--user=mysql"}
if cmutil.MySQLVersionParse(mysqlVersion) >= cmutil.MySQLVersionParse("8.0.0") {
args = append(args, "--default-authentication-plugin=mysql_native_password")
}
dbArgs, err := model.GetStartArsg("mysql", mysqlVersion)
if err != nil {
logger.Warn("get mysql start args failed %s", err.Error())
return
}
for _, arg := range startArgsSplitRe.Split(dbArgs, -1) {
if lo.IsNotEmpty(arg) {
args = append(args, strings.TrimSpace(arg))
}
}
return
}

// CreateClusterPod create tendbcluster simulation pod
func (k *DbPodSets) CreateClusterPod(mySQLVersion string) (err error) {
c := &v1.Pod{
Expand Down Expand Up @@ -356,16 +408,32 @@ func (k *DbPodSets) gettdbctlResourceLimit() v1.ResourceRequirements {
return v1.ResourceRequirements{}
}

// CreateMySQLPod create mysql pod
func (k *DbPodSets) CreateMySQLPod(mysqlVersion string) (err error) {
startArgs := []string{
func (k *DbPodSets) getTendbhaPodStartArgs(mysqlVersion string) (args []string) {
args = []string{
"mysqld",
"--defaults-file=/etc/my.cnf",
"--skip-log-bin",
"--max_allowed_packet=1073741824",
fmt.Sprintf("--character-set-server=%s", k.BaseInfo.Charset)}
if cmutil.MySQLVersionParse(mysqlVersion) >= cmutil.MySQLVersionParse("8.0.0") {
startArgs = append(startArgs, "--default-authentication-plugin=mysql_native_password")
args = append(args, "--default-authentication-plugin=mysql_native_password")
}
dbArgs, err := model.GetStartArsg("mysql", mysqlVersion)
if err != nil {
logger.Warn("get mysql start args failed %s", err.Error())
return
}
for _, arg := range startArgsSplitRe.Split(dbArgs, -1) {
if lo.IsNotEmpty(arg) {
args = append(args, strings.TrimSpace(arg))
}
}
return
}

// CreateMySQLPod create mysql pod
func (k *DbPodSets) CreateMySQLPod(mysqlVersion string) (err error) {
startArgs := k.getTendbhaPodStartArgs(mysqlVersion)
startArgs = append(startArgs, k.BaseInfo.Args...)
startArgs = append(startArgs, "--user=mysql")
logger.Info("start pod args %v", startArgs)
Expand Down
Empty file added dbm-services/mysql/db-simulation/assets/migrations/000004_add_pod_start_args_table.down.sql
View file
Open in desktop
Empty file.
10 changes: 10 additions & 0 deletions dbm-services/mysql/db-simulation/assets/migrations/000004_add_pod_start_args_table.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS `tb_mysql_pod_start_cfgs` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`component_type` varchar(64) NOT NULL,
`version` varchar(64) NOT NULL,
`start_args` varchar(1024) NOT NULL,
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `uk_cv` (`component_type`, `version`)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
31 changes: 31 additions & 0 deletions dbm-services/mysql/db-simulation/model/tb_mysql_pod_start_cfg.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
* Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
* Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://opensource.org/licenses/MIT
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

package model

import "time"

// TbMysqlPodStartCfg 模拟执行 pod 启动参数
type TbMysqlPodStartCfg struct {
ID int `gorm:"primaryKey;column:id;type:int(11);not null" json:"-"`
ComponentType string `gorm:"unique;column:uk_cv;type:varchar(64);not null" json:"component_type"`
Version string `gorm:"unique;column:uk_cv;type:varchar(64);not null" json:"version"`
StartArgs string `gorm:"column:start_args;type:varchar(1024);not null" json:"start_args"`
UpdateTime time.Time `gorm:"column:update_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"update_time"`
CreateTime time.Time `gorm:"column:create_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"create_time"`
}

// GetStartArsg pod 启动参数
func GetStartArsg(componentType, version string) (start_args string, err error) {
err = DB.Model(&TbMysqlPodStartCfg{}).Select("start_args").Where("component_type = ? AND version = ?", componentType,
version).
First(&start_args).Error
return
}
138 changes: 0 additions & 138 deletions dbm-services/mysql/db-simulation/model/tb_syntax_rule.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,144 +56,6 @@ func (obj *TbSyntaxRule) GetTableName() string {
return "tb_syntax_rules"
}

// func init() {
// if err := InitRule(); err != nil {
// logger.Fatal("init syntax rule failed %s", err.Error())
// return
// }
// }

// InitRule init rules
// func InitRule() (err error) {
// initRules := []TbSyntaxRule{}
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CommandRule",
// RuleName: "HighRiskCommandRule",
// Expr: "Val in Item",
// ItemType: ArryItem,
// Item: []byte(
// `["drop_table", "drop_index", "lock_tables", "drop_db", "analyze","rename_table",
// "drop_procedure", "drop_view","drop_trigger","drop_function", "drop_server",
// "drop_event", "drop_compression_dictionary","optimize", "alter_tablespace"]`),
// Desc: "高危命令",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CommandRule",
// RuleName: "BanCommandRule",
// Expr: "Val in Item",
// ItemType: ArryItem,
// Item: []byte(`["truncate", "revoke", "kill", "reset", "drop_user", "grant",
// "create_user", "revoke_all", "shutdown", "lock_tables_for_backup",
// "reset", "purge", "lock_binlog_for_backup","lock_tables_for_backup",
// "install_plugin", "uninstall_plugin","alter_user"]`),
// Desc: "高危变更类型",
// WarnLevel: 1,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CreateTableRule",
// RuleName: "SuggestBlobColumCount",
// Expr: "Val >= Item ",
// ItemType: IntItem,
// Item: []byte(`10`),
// Desc: "建议单表Blob字段不要过多",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CreateTableRule",
// RuleName: "SuggestEngine",
// Expr: "not (Val contains Item) and ( len(Val) != 0 )",
// ItemType: StringItem,
// Item: []byte(`"innodb"`),
// Desc: "建议使用Innodb表",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CreateTableRule",
// RuleName: "NeedPrimaryKey",
// Expr: "Val == Item",
// ItemType: IntItem,
// Item: []byte(`1`),
// Desc: "建议包含主键",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "CreateTableRule",
// RuleName: "DefinerRule",
// Expr: "Val not in Item ",
// ItemType: ArryItem,
// Item: []byte(`["ADMIN@localhost"]`),
// Desc: "必须指定definer",
// WarnLevel: 0,
// Status: true,
// })

// initRules = append(initRules, TbSyntaxRule{
// GroupName: "AlterTableRule",
// RuleName: "HighRiskType",
// Expr: "Val in Item",
// ItemType: ArryItem,
// Item: []byte(`["drop_column"]`),
// Desc: "高危变更类型",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "AlterTableRule",
// RuleName: "HighRiskPkAlterType",
// Expr: "Val in Item",
// ItemType: ArryItem,
// Item: []byte(`["add_column", "add_key", "change_column"]`),
// Desc: "主键高危变更类型",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "AlterTableRule",
// RuleName: "AlterUseAfter",
// Expr: "Val != Item",
// ItemType: StringItem,
// Item: []byte(`""`),
// Desc: "变更表时使用了after",
// WarnLevel: 0,
// Status: true,
// })
// initRules = append(initRules, TbSyntaxRule{
// GroupName: "AlterTableRule",
// RuleName: "AddColumnMixed",
// Expr: "( Item in Val ) && ( len(Val) > 1 )",
// ItemType: StringItem,
// Item: []byte(`"add_column"`),
// Desc: "加字段和其它alter table 类型混用，可能导致非在线加字段",
// WarnLevel: 0,
// Status: true,
// })

// initRules = append(initRules, TbSyntaxRule{
// GroupName: "DmlRule",
// RuleName: "DmlNotHasWhere",
// Expr: " Val != Item ",
// ItemType: BoolItem,
// Item: []byte(`true`),
// Desc: "没有使用WHERE或者LIMIT,可能会导致全表数据更改",
// WarnLevel: 0,
// Status: true,
// })

// for i := range initRules {
// if err = CreateRule(&initRules[i]); err != nil {
// logger.Error("初始化规则失败%s", err.Error())
// return err
// }
// }
// return err
// }

// CreateRule create rule
func CreateRule(m *TbSyntaxRule) (err error) {
return DB.Clauses(clause.OnConflict{
Expand Down