From 5a96b2a8e6f74bc51fe222e6c346213b3d048b9c Mon Sep 17 00:00:00 2001 From: ymakedaq <996156275@qq.com> Date: Wed, 18 Oct 2023 10:20:35 +0800 Subject: [PATCH] =?UTF-8?q?feat(dbm-services):=20=E8=A1=A5=E5=85=85tendbcl?= =?UTF-8?q?uster=20=E8=AF=AD=E4=B9=89=E6=A3=80=E9=AA=8C=E8=A7=84=E5=88=99?= =?UTF-8?q?=20close=20#1437?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/service/simulation_task.go | 4 +- .../mysql/db-simulation/app/syntax/rule.go | 12 +- .../app/syntax/spider_create_table_rule.go | 131 +++++++++++------- .../db-simulation/app/syntax/spider_rule.go | 1 + .../mysql/db-simulation/spider_rule.yaml | 10 +- .../bk-dbm/charts/db-simulation/Chart.yaml | 2 +- 6 files changed, 100 insertions(+), 60 deletions(-) diff --git a/dbm-services/mysql/db-simulation/app/service/simulation_task.go b/dbm-services/mysql/db-simulation/app/service/simulation_task.go index 2b3be034c2..314d09776a 100644 --- a/dbm-services/mysql/db-simulation/app/service/simulation_task.go +++ b/dbm-services/mysql/db-simulation/app/service/simulation_task.go @@ -296,8 +296,8 @@ func (t *SimulationTask) SimulationRun(containerName string, xlogger *logger.Log xlogger.Error("when execute %s at %s, failed %s\n", e.SQLFile, realexcutedbs[idx-1], err.Error()) xlogger.Error("stderr:\n %s", stderr.String()) xlogger.Error("stdout:\n %s", stdout.String()) - return sstdout, sstderr, fmt.Errorf("exec %s in %s failed:%s", e.SQLFile, realexcutedbs[idx-1], - err.Error()) + return sstdout, sstderr, fmt.Errorf("\nexec %s in %s failed:%s\n %s", e.SQLFile, realexcutedbs[idx-1], + err.Error(), stderr.String()) } xlogger.Info("%s \n %s", stdout.String(), stderr.String()) } diff --git a/dbm-services/mysql/db-simulation/app/syntax/rule.go b/dbm-services/mysql/db-simulation/app/syntax/rule.go index 7541a08993..b7c7f78bfe 100644 --- a/dbm-services/mysql/db-simulation/app/syntax/rule.go +++ b/dbm-services/mysql/db-simulation/app/syntax/rule.go @@ -106,9 +106,9 @@ func (c *CheckerResult) Trigger(rule *BoolRuleItem, additionalMsg string) { return } if rule.Ban { - c.BanWarns = append(c.BanWarns, fmt.Sprintf("%s:%s", rule.Desc, additionalMsg)) + c.BanWarns = append(c.BanWarns, fmt.Sprintf("%s:%s\n%s", rule.Desc, additionalMsg, rule.Suggestion)) } else { - c.RiskWarns = append(c.RiskWarns, fmt.Sprintf("%s:%s", rule.Desc, additionalMsg)) + c.RiskWarns = append(c.RiskWarns, fmt.Sprintf("%s:%s\n%s", rule.Desc, additionalMsg, rule.Suggestion)) } } @@ -128,13 +128,15 @@ type RuleItem struct { Expr string `yaml:"expr"` Desc string `yaml:"desc"` Ban bool `yaml:"ban"` + Suggestion string `yaml:"suggestion"` } // BoolRuleItem 开关型规则，只需配置开启或者关闭即可 type BoolRuleItem struct { - Desc string `yaml:"desc"` - Ban bool `yaml:"ban"` - TurnOn bool `yaml:"turnOn"` + Desc string `yaml:"desc"` + Ban bool `yaml:"ban"` + TurnOn bool `yaml:"turnOn"` + Suggestion string `yaml:"suggestion"` } // Rules TODO diff --git a/dbm-services/mysql/db-simulation/app/syntax/spider_create_table_rule.go b/dbm-services/mysql/db-simulation/app/syntax/spider_create_table_rule.go index d76ef2b875..28fbfc388a 100644 --- a/dbm-services/mysql/db-simulation/app/syntax/spider_create_table_rule.go +++ b/dbm-services/mysql/db-simulation/app/syntax/spider_create_table_rule.go @@ -46,83 +46,102 @@ func (c CreateTableResult) SpiderChecker(spiderVersion string) (r *CheckerResult } func (c CreateTableResult) shardKeyChecker(r *CheckerResult) { + var has_shard_key bool + var shardKeyCol string + var err error logger.Info("start shardKeyChecker...") // 如果沒有任何索引,则直接返回错误 if len(c.CreateDefinitions.KeyDefs) <= 0 { r.Trigger(SR.SpiderCreateTableRule.NoIndexExists, "") return } - uniqueKeys, commonKeys := c.findTablesIndex() + pk, uks, keys := c.findTablesIndex() tableComment := c.GetComment() - logger.Info("tableComment is %s", tableComment) - // 如果table comment 为空,表示没有指定shard key - // 或者table comnent 没有指定shardkey - // 由中控自主选择 - if cmutil.IsEmpty(tableComment) || !strings.Contains(tableComment, "shard_key") { - // 如果没有唯一索引，如果包含多个普通索引，则必须指定shard_key,否则需要报错 - if len(commonKeys) > 1 { - r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, "") - return - } - } // 如果存在多个唯一健（含主键),多个唯一键都没有包含相同的字段也是不允许的 var pubCols []string - logger.Info("uniqueKeys is %v,len is %d", uniqueKeys, len(uniqueKeys)) - if len(uniqueKeys) > 1 { - pubCols = findCommonColByKeys(uniqueKeys) + logger.Info("uniqueKeys is %v,len is %d", uks, len(uks)) + if len(uks) > 1 { + pubCols = findCommonColByKeys(uks) if len(pubCols) < 1 { r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, "") return } } - // table comment 不为空的时候 先校验comment 格式是否合法 - legal, msg := c.validateSpiderComment(tableComment) - if !legal { - r.Trigger(SR.SpiderCreateTableRule.IllegalComment, msg) - return - } - shardKeyCol, err := util.ParseGetShardKeyForSpider(tableComment) - if err != nil { - // Todo 错误处理 - logger.Error("parse %s comment %s shard key failed %s", c.TableName, tableComment, err.Error()) - return - } - // 如果存在索引,但是shard key不属于任何索引 - if !c.shardKeyIsIndex(shardKeyCol) { - r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotIndex, "") - return - } - switch { - case len(uniqueKeys) == 1: - if !c.shardKeyExistInKeys(shardKeyCol, uniqueKeys) { - r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol) + logger.Info("tableComment is %s", tableComment) + + if cmutil.IsNotEmpty(tableComment) { + // table comment 不为空的时候 先校验comment 格式是否合法 + legal, msg := c.validateSpiderComment(tableComment) + if !legal { + r.Trigger(SR.SpiderCreateTableRule.IllegalComment, msg) + return + } + shardKeyCol, err = util.ParseGetShardKeyForSpider(tableComment) + if err != nil { + // Todo 错误处理 + logger.Error("parse %s comment %s shard key failed %s", c.TableName, tableComment, err.Error()) return } - // 如果存在 一个或者多个唯一索引(包含主键) - case len(uniqueKeys) > 1: - // shard_key只能是其中的共同部分；否则无法建表； - if !slices.Contains(pubCols, shardKeyCol) { - r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol) + has_shard_key = true + } + // 如果table comment 为空,表示没有指定shard key,或table comnent 没有指定shardkey 由中控自主选择 + if !has_shard_key { + switch { + case pk != nil: + return + case len(uks) > 1: + return + case len(keys) > 1: + // 如果没有唯一索引，如果包含多个普通索引，则必须指定shard_key,否则需要报错 + r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, "") return } - // 如果只存在多个普通索引，shard_key只能是其中任意一个的一部分 - case len(uniqueKeys) < 1 && len(commonKeys) > 1: - if !c.shardKeyExistInKeys(shardKeyCol, commonKeys) { - r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, shardKeyCol) + } else { + // 如果存在索引,但是shard key不属于任何索引 + if !c.shardKeyIsIndex(shardKeyCol) { + r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotIndex, "") return } + switch { + case len(uks) == 1: + if !c.shardKeyExistInKeys(shardKeyCol, uks) { + r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol) + return + } + // 如果存在 一个或者多个唯一索引(包含主键) + case len(uks) > 1: + // shard_key只能是其中的共同部分；否则无法建表； + if !slices.Contains(pubCols, shardKeyCol) { + r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol) + return + } + // 如果只存在多个普通索引，shard_key只能是其中任意一个的一部分 + case len(uks) < 1 && len(uks) > 1: + if !c.shardKeyExistInKeys(shardKeyCol, uks) { + r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, shardKeyCol) + return + } + } + shardKeyColDef := c.getColDef(shardKeyCol) + // 如果shard key 列允许为null + if shardKeyColDef.Nullable { + r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotNull, shardKeyColDef.ColName) + } } } -func (c CreateTableResult) findTablesIndex() (uniqueKeys []KeyDef, commonKeys []KeyDef) { +func (c CreateTableResult) findTablesIndex() (pk *KeyDef, uks []KeyDef, keys []KeyDef) { for _, key := range c.CreateDefinitions.KeyDefs { - if key.PrimaryKey || key.UniqueKey { - uniqueKeys = append(uniqueKeys, key) - } else { - commonKeys = append(commonKeys, key) + switch { + case key.PrimaryKey: + pk = &key + case key.UniqueKey: + uks = append(uks, key) + default: + keys = append(keys, key) } } - return uniqueKeys, commonKeys + return pk, uks, keys } // findCommonColByKeys 寻找多个唯一键中的公共列 @@ -177,3 +196,13 @@ func (c CreateTableResult) shardKeyExistInKeys(shardKeyCol string, keys []KeyDef } return false } + +func (c CreateTableResult) getColDef(colName string) (colDef ColDef) { + for _, col := range c.CreateDefinitions.ColDefs { + if strings.Compare(col.ColName, colName) == 0 { + colDef = col + break + } + } + return colDef +} diff --git a/dbm-services/mysql/db-simulation/app/syntax/spider_rule.go b/dbm-services/mysql/db-simulation/app/syntax/spider_rule.go index 116fe6ec59..921d43b960 100644 --- a/dbm-services/mysql/db-simulation/app/syntax/spider_rule.go +++ b/dbm-services/mysql/db-simulation/app/syntax/spider_rule.go @@ -35,6 +35,7 @@ type SpiderCreateTableRule struct { NoIndexExists *BoolRuleItem `yaml:"NoIndexExists"` NoPubColAtMultUniqueIndex *BoolRuleItem `yaml:"NoPubColAtMultUniqueIndex"` MustSpecialShardKeyOnlyHaveCommonIndex *BoolRuleItem `yaml:"MustSpecialShardKeyOnlyHaveCommonIndex"` + ShardKeyNotNull *BoolRuleItem `yaml:"ShardKeyNotNull"` } func init() { diff --git a/dbm-services/mysql/db-simulation/spider_rule.yaml b/dbm-services/mysql/db-simulation/spider_rule.yaml index d0e9260647..2cedd1a754 100644 --- a/dbm-services/mysql/db-simulation/spider_rule.yaml +++ b/dbm-services/mysql/db-simulation/spider_rule.yaml @@ -51,6 +51,7 @@ SpiderCreateTableRule: turnOn: true ban: true desc: "create table 语句中列字符集定义与表字符集不一致" + suggestion: "去掉列字符集" CreateWithSelect: turnOn: true ban: true @@ -75,11 +76,18 @@ SpiderCreateTableRule: turnOn: true ban: true desc: "没有任何索引存在" + suggestion: "请添加索引、唯一键或主键" NoPubColAtMultUniqueIndex: turnOn: true ban: true - desc: "存在多个唯一健（含主键),shard_key只能是其中的共同部分" + desc: "用户comment指定了shard_key,存在多个唯一健（含主键),shard_key只能是其中的共同部分" MustSpecialShardKeyOnlyHaveCommonIndex: turnOn: true ban: true desc: "不存在唯一索引,只包含包含多个普通索引,则必须指定shard_key" + suggestion: "请指定shard_key" + ShardKeyNotNull: + turnOn: true + ban: true + desc: "如果是shard key 不是主键,默认值就一定要指定成 not null" + suggestion: "请指定shard_key列 is not null" diff --git a/helm-charts/bk-dbm/charts/db-simulation/Chart.yaml b/helm-charts/bk-dbm/charts/db-simulation/Chart.yaml index a1b4e144c9..2261b656d2 100644 --- a/helm-charts/bk-dbm/charts/db-simulation/Chart.yaml +++ b/helm-charts/bk-dbm/charts/db-simulation/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 0.0.1-alpha.38 +appVersion: 0.0.1-alpha.39 description: A Helm chart for Kubernetes name: db-simulation type: application