Skip to content

Commit

Permalink
feat(dbm-services): 补充tendbcluster 语义检验规则 close #1437
Browse files Browse the repository at this point in the history
  • Loading branch information
@ymakedaq
ymakedaq committed Oct 18, 2023
1 parent 59e52d1 commit 5a96b2a
Show file tree
Hide file tree
Showing 6 changed files with 100 additions and 60 deletions.
4 changes: 2 additions & 2 deletions dbm-services/mysql/db-simulation/app/service/simulation_task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,8 +296,8 @@ func (t *SimulationTask) SimulationRun(containerName string, xlogger *logger.Log
xlogger.Error("when execute %s at %s, failed %s\n", e.SQLFile, realexcutedbs[idx-1], err.Error())
xlogger.Error("stderr:\n %s", stderr.String())
xlogger.Error("stdout:\n %s", stdout.String())
return sstdout, sstderr, fmt.Errorf("exec %s in %s failed:%s", e.SQLFile, realexcutedbs[idx-1],
err.Error())
return sstdout, sstderr, fmt.Errorf("\nexec %s in %s failed:%s\n %s", e.SQLFile, realexcutedbs[idx-1],
err.Error(), stderr.String())
}
xlogger.Info("%s \n %s", stdout.String(), stderr.String())
}
Expand Down
12 changes: 7 additions & 5 deletions dbm-services/mysql/db-simulation/app/syntax/rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,9 @@ func (c *CheckerResult) Trigger(rule *BoolRuleItem, additionalMsg string) {
return
}
if rule.Ban {
c.BanWarns = append(c.BanWarns, fmt.Sprintf("%s:%s", rule.Desc, additionalMsg))
c.BanWarns = append(c.BanWarns, fmt.Sprintf("%s:%s\n%s", rule.Desc, additionalMsg, rule.Suggestion))
} else {
c.RiskWarns = append(c.RiskWarns, fmt.Sprintf("%s:%s", rule.Desc, additionalMsg))
c.RiskWarns = append(c.RiskWarns, fmt.Sprintf("%s:%s\n%s", rule.Desc, additionalMsg, rule.Suggestion))
}
}

Expand All @@ -128,13 +128,15 @@ type RuleItem struct {
Expr string `yaml:"expr"`
Desc string `yaml:"desc"`
Ban bool `yaml:"ban"`
Suggestion string `yaml:"suggestion"`
}

// BoolRuleItem 开关型规则，只需配置开启或者关闭即可
type BoolRuleItem struct {
Desc string `yaml:"desc"`
Ban bool `yaml:"ban"`
TurnOn bool `yaml:"turnOn"`
Desc string `yaml:"desc"`
Ban bool `yaml:"ban"`
TurnOn bool `yaml:"turnOn"`
Suggestion string `yaml:"suggestion"`
}

// Rules TODO
Expand Down
131 changes: 80 additions & 51 deletions dbm-services/mysql/db-simulation/app/syntax/spider_create_table_rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,83 +46,102 @@ func (c CreateTableResult) SpiderChecker(spiderVersion string) (r *CheckerResult
}

func (c CreateTableResult) shardKeyChecker(r *CheckerResult) {
var has_shard_key bool
var shardKeyCol string
var err error
logger.Info("start shardKeyChecker...")
// 如果沒有任何索引,则直接返回错误
if len(c.CreateDefinitions.KeyDefs) <= 0 {
r.Trigger(SR.SpiderCreateTableRule.NoIndexExists, "")
return
}
uniqueKeys, commonKeys := c.findTablesIndex()
pk, uks, keys := c.findTablesIndex()
tableComment := c.GetComment()
logger.Info("tableComment is %s", tableComment)
// 如果table comment 为空,表示没有指定shard key
// 或者table comnent 没有指定shardkey
// 由中控自主选择
if cmutil.IsEmpty(tableComment) || !strings.Contains(tableComment, "shard_key") {
// 如果没有唯一索引，如果包含多个普通索引，则必须指定shard_key,否则需要报错
if len(commonKeys) > 1 {
r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, "")
return
}
}
// 如果存在多个唯一健（含主键),多个唯一键都没有包含相同的字段也是不允许的
var pubCols []string
logger.Info("uniqueKeys is %v,len is %d", uniqueKeys, len(uniqueKeys))
if len(uniqueKeys) > 1 {
pubCols = findCommonColByKeys(uniqueKeys)
logger.Info("uniqueKeys is %v,len is %d", uks, len(uks))
if len(uks) > 1 {
pubCols = findCommonColByKeys(uks)
if len(pubCols) < 1 {
r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, "")
return
}
}
// table comment 不为空的时候 先校验comment 格式是否合法
legal, msg := c.validateSpiderComment(tableComment)
if !legal {
r.Trigger(SR.SpiderCreateTableRule.IllegalComment, msg)
return
}
shardKeyCol, err := util.ParseGetShardKeyForSpider(tableComment)
if err != nil {
// Todo 错误处理
logger.Error("parse %s comment %s shard key failed %s", c.TableName, tableComment, err.Error())
return
}
// 如果存在索引,但是shard key不属于任何索引
if !c.shardKeyIsIndex(shardKeyCol) {
r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotIndex, "")
return
}
switch {
case len(uniqueKeys) == 1:
if !c.shardKeyExistInKeys(shardKeyCol, uniqueKeys) {
r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol)
logger.Info("tableComment is %s", tableComment)

if cmutil.IsNotEmpty(tableComment) {
// table comment 不为空的时候 先校验comment 格式是否合法
legal, msg := c.validateSpiderComment(tableComment)
if !legal {
r.Trigger(SR.SpiderCreateTableRule.IllegalComment, msg)
return
}
shardKeyCol, err = util.ParseGetShardKeyForSpider(tableComment)
if err != nil {
// Todo 错误处理
logger.Error("parse %s comment %s shard key failed %s", c.TableName, tableComment, err.Error())
return
}
// 如果存在 一个或者多个唯一索引(包含主键)
case len(uniqueKeys) > 1:
// shard_key只能是其中的共同部分；否则无法建表；
if !slices.Contains(pubCols, shardKeyCol) {
r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol)
has_shard_key = true
}
// 如果table comment 为空,表示没有指定shard key,或table comnent 没有指定shardkey 由中控自主选择
if !has_shard_key {
switch {
case pk != nil:
return
case len(uks) > 1:
return
case len(keys) > 1:
// 如果没有唯一索引，如果包含多个普通索引，则必须指定shard_key,否则需要报错
r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, "")
return
}
// 如果只存在多个普通索引，shard_key只能是其中任意一个的一部分
case len(uniqueKeys) < 1 && len(commonKeys) > 1:
if !c.shardKeyExistInKeys(shardKeyCol, commonKeys) {
r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, shardKeyCol)
} else {
// 如果存在索引,但是shard key不属于任何索引
if !c.shardKeyIsIndex(shardKeyCol) {
r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotIndex, "")
return
}
switch {
case len(uks) == 1:
if !c.shardKeyExistInKeys(shardKeyCol, uks) {
r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol)
return
}
// 如果存在 一个或者多个唯一索引(包含主键)
case len(uks) > 1:
// shard_key只能是其中的共同部分；否则无法建表；
if !slices.Contains(pubCols, shardKeyCol) {
r.Trigger(SR.SpiderCreateTableRule.NoPubColAtMultUniqueIndex, shardKeyCol)
return
}
// 如果只存在多个普通索引，shard_key只能是其中任意一个的一部分
case len(uks) < 1 && len(uks) > 1:
if !c.shardKeyExistInKeys(shardKeyCol, uks) {
r.Trigger(SR.SpiderCreateTableRule.MustSpecialShardKeyOnlyHaveCommonIndex, shardKeyCol)
return
}
}
shardKeyColDef := c.getColDef(shardKeyCol)
// 如果shard key 列允许为null
if shardKeyColDef.Nullable {
r.Trigger(SR.SpiderCreateTableRule.ShardKeyNotNull, shardKeyColDef.ColName)
}
}
}

func (c CreateTableResult) findTablesIndex() (uniqueKeys []KeyDef, commonKeys []KeyDef) {
func (c CreateTableResult) findTablesIndex() (pk *KeyDef, uks []KeyDef, keys []KeyDef) {
for _, key := range c.CreateDefinitions.KeyDefs {
if key.PrimaryKey || key.UniqueKey {
uniqueKeys = append(uniqueKeys, key)
} else {
commonKeys = append(commonKeys, key)
switch {
case key.PrimaryKey:
pk = &key
case key.UniqueKey:
uks = append(uks, key)
default:
keys = append(keys, key)
}
}
return uniqueKeys, commonKeys
return pk, uks, keys
}

// findCommonColByKeys 寻找多个唯一键中的公共列
Expand Down Expand Up @@ -177,3 +196,13 @@ func (c CreateTableResult) shardKeyExistInKeys(shardKeyCol string, keys []KeyDef
}
return false
}

func (c CreateTableResult) getColDef(colName string) (colDef ColDef) {
for _, col := range c.CreateDefinitions.ColDefs {
if strings.Compare(col.ColName, colName) == 0 {
colDef = col
break
}
}
return colDef
}
1 change: 1 addition & 0 deletions dbm-services/mysql/db-simulation/app/syntax/spider_rule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ type SpiderCreateTableRule struct {
NoIndexExists *BoolRuleItem `yaml:"NoIndexExists"`
NoPubColAtMultUniqueIndex *BoolRuleItem `yaml:"NoPubColAtMultUniqueIndex"`
MustSpecialShardKeyOnlyHaveCommonIndex *BoolRuleItem `yaml:"MustSpecialShardKeyOnlyHaveCommonIndex"`
ShardKeyNotNull *BoolRuleItem `yaml:"ShardKeyNotNull"`
}

func init() {
Expand Down
10 changes: 9 additions & 1 deletion dbm-services/mysql/db-simulation/spider_rule.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ SpiderCreateTableRule:
turnOn: true
ban: true
desc: "create table 语句中列字符集定义与表字符集不一致"
suggestion: "去掉列字符集"
CreateWithSelect:
turnOn: true
ban: true
Expand All @@ -75,11 +76,18 @@ SpiderCreateTableRule:
turnOn: true
ban: true
desc: "没有任何索引存在"
suggestion: "请添加索引、唯一键或主键"
NoPubColAtMultUniqueIndex:
turnOn: true
ban: true
desc: "存在多个唯一健（含主键),shard_key只能是其中的共同部分"
desc: "用户comment指定了shard_key,存在多个唯一健（含主键),shard_key只能是其中的共同部分"
MustSpecialShardKeyOnlyHaveCommonIndex:
turnOn: true
ban: true
desc: "不存在唯一索引,只包含包含多个普通索引,则必须指定shard_key"
suggestion: "请指定shard_key"
ShardKeyNotNull:
turnOn: true
ban: true
desc: "如果是shard key 不是主键,默认值就一定要指定成 not null"
suggestion: "请指定shard_key列 is not null"
2 changes: 1 addition & 1 deletion helm-charts/bk-dbm/charts/db-simulation/Chart.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
apiVersion: v2
appVersion: 0.0.1-alpha.38
appVersion: 0.0.1-alpha.39
description: A Helm chart for Kubernetes
name: db-simulation
type: application
Expand Down

0 comments on commit 5a96b2a

Please sign in to comment.