From 3fea3a14310a346a1969726a3697674a4c71e1a8 Mon Sep 17 00:00:00 2001 From: yuanruji Date: Fri, 25 Oct 2024 14:26:07 +0800 Subject: [PATCH] =?UTF-8?q?feat(dbm-services):=20=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B1=A0=E6=A0=87=E7=AD=BE=E5=92=8C=E8=B5=84=E6=BA=90=E6=B5=81?= =?UTF-8?q?=E8=BD=AC=E5=8A=9F=E8=83=BD=E6=94=AF=E6=8C=81=20#7586?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../000003_rename_lable_to_lables.down.sql | 0 .../000003_rename_lable_to_lables.up.sql | 2 + .../internal/controller/apply/apply.go | 53 +-- .../internal/controller/controller.go | 21 +- .../internal/controller/manage/rs_import.go | 148 +++++++-- .../internal/controller/manage/rs_lable.go | 12 +- .../internal/controller/manage/rs_list.go | 25 +- .../internal/controller/manage/rs_manage.go | 101 ++++-- .../controller/manage/rs_operation_info.go | 17 +- .../internal/controller/manage/rs_spec.go | 11 +- .../controller/statistic/statistic.go | 31 +- ...daily_snapshot.go => TbRpDailySnapShot.go} | 0 .../db-resource/internal/model/TbRpDetail.go | 89 +++-- .../internal/model/TbRpDetailArchive.go | 49 +-- .../internal/model/TbRpOperatorInfo.go | 7 + .../db-resource/internal/svr/apply/apply.go | 309 +++++++++++------- .../svr/apply/{api.go => apply_base.go} | 65 +++- .../svr/apply/{core.go => apply_core.go} | 61 +++- .../{match_resource.go => apply_match.go} | 1 - ...ority_queue.go => apply_priority_queue.go} | 0 .../internal/svr/apply/instance.go | 55 ---- .../db-resource/internal/svr/bk/cc_test.go | 2 +- .../internal/svr/dbmapi/dbm_spec.go | 3 + .../internal/svr/task/inspection_task.go | 6 +- 24 files changed, 668 insertions(+), 400 deletions(-) create mode 100644 dbm-services/common/db-resource/assets/migrations/000003_rename_lable_to_lables.down.sql create mode 100644 dbm-services/common/db-resource/assets/migrations/000003_rename_lable_to_lables.up.sql rename dbm-services/common/db-resource/internal/model/{tb_rp_daily_snapshot.go => TbRpDailySnapShot.go} (100%) rename dbm-services/common/db-resource/internal/svr/apply/{api.go => apply_base.go} (82%) rename dbm-services/common/db-resource/internal/svr/apply/{core.go => apply_core.go} (84%) rename dbm-services/common/db-resource/internal/svr/apply/{match_resource.go => apply_match.go} (99%) rename dbm-services/common/db-resource/internal/svr/apply/{priority_queue.go => apply_priority_queue.go} (100%) delete mode 100644 dbm-services/common/db-resource/internal/svr/apply/instance.go diff --git a/dbm-services/common/db-resource/assets/migrations/000003_rename_lable_to_lables.down.sql b/dbm-services/common/db-resource/assets/migrations/000003_rename_lable_to_lables.down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-services/common/db-resource/assets/migrations/000003_rename_lable_to_lables.up.sql b/dbm-services/common/db-resource/assets/migrations/000003_rename_lable_to_lables.up.sql new file mode 100644 index 0000000000..a08ad05a57 --- /dev/null +++ b/dbm-services/common/db-resource/assets/migrations/000003_rename_lable_to_lables.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE tb_rp_detail change column label labels json; +ALTER TABLE tb_rp_detail_archive change column label labels json; \ No newline at end of file diff --git a/dbm-services/common/db-resource/internal/controller/apply/apply.go b/dbm-services/common/db-resource/internal/controller/apply/apply.go index 0e8bb426eb..f865614a50 100644 --- a/dbm-services/common/db-resource/internal/controller/apply/apply.go +++ b/dbm-services/common/db-resource/internal/controller/apply/apply.go @@ -26,6 +26,7 @@ import ( "dbm-services/common/go-pubpkg/logger" "github.com/gin-gonic/gin" + "github.com/samber/lo" ) func init() { @@ -65,33 +66,27 @@ func (c *ApplyHandler) ConfirmApply(r *gin.Context) { if c.Prepare(r, ¶m) != nil { return } - requestId := r.GetString("request_id") - hostIds := cmutil.RemoveDuplicate(param.HostIds) + hostIds := lo.Uniq(param.HostIds) var cnt int64 err := model.DB.Self.Table(model.TbRpApplyDetailLogName()).Where("request_id = ?", param.RequestId).Count(&cnt).Error if err != nil { logger.Error("use request id %s,query apply resouece failed %s", param.RequestId, err.Error()) - c.SendResponse(r, fmt.Errorf("%w", err), "use request id search applyed resource failed", requestId) return } if len(hostIds) != int(cnt) { c.SendResponse(r, fmt.Errorf("need return resource count is %d,but use request id only found total count %d", - len(hostIds), cnt), requestId, "") + len(hostIds), cnt), "") return } var rs []model.TbRpDetail err = model.DB.Self.Table(model.TbRpDetailName()).Where(" bk_host_id in (?) and status != ? ", hostIds, model.Prepoccupied).Find(&rs).Error if err != nil { - c.SendResponse(r, err, err.Error(), requestId) + c.SendResponse(r, err, err.Error()) return } if len(rs) > 0 { - var errMsg string - for _, v := range rs { - errMsg += fmt.Sprintf("%s:%s\n", v.IP, v.Status) - } - c.SendResponse(r, fmt.Errorf("the following example:%s,abnormal state", errMsg), "", requestId) + c.SendResponse(r, fmt.Errorf("the following example:%s,abnormal state", buildErrMsg(rs)), "") return } // update to used status @@ -103,7 +98,7 @@ func (c *ApplyHandler) ConfirmApply(r *gin.Context) { }, ) if err != nil { - c.SendResponse(r, err, err.Error(), requestId) + c.SendResponse(r, err, err.Error()) return } uerr := model.DB.Self.Table(model.TbRpOperationInfoTableName()).Where("request_id = ?", @@ -112,7 +107,15 @@ func (c *ApplyHandler) ConfirmApply(r *gin.Context) { logger.Warn("update tb_rp_operation_info failed %s ", uerr.Error()) } archive(hostIds) - c.SendResponse(r, nil, "successful", requestId) + c.SendResponse(r, nil, "successful") +} + +func buildErrMsg(abnormalRsList []model.TbRpDetail) string { + var errMsg string + for _, v := range abnormalRsList { + errMsg += fmt.Sprintf("%s:%s\n", v.IP, v.Status) + } + return errMsg } func archive(bkHostIds []int) { @@ -151,20 +154,18 @@ func (c *ApplyHandler) ApplyBase(r *gin.Context, mode string) { var param apply.RequestInputParam var pickers []*apply.PickerObject var err error - var requestId string if c.Prepare(r, ¶m) != nil { return } - requestId = r.GetString("request_id") if err = param.ParamCheck(); err != nil { - c.SendResponse(r, errno.ErrApplyResourceParamCheck.AddErr(err), err.Error(), requestId) + c.SendResponse(r, errno.ErrApplyResourceParamCheck.AddErr(err), err.Error()) return } // get the resource lock if it is dry run you do not need to acquire it if !param.DryRun { - lock := newLocker(param.LockKey(), requestId) + lock := newLocker(param.LockKey(), c.RequestId) if err = lock.Lock(); err != nil { - c.SendResponse(r, errno.ErrResourceLock.AddErr(err), err.Error(), requestId) + c.SendResponse(r, errno.ErrResourceLock.AddErr(err), err.Error()) return } defer func() { @@ -175,27 +176,29 @@ func (c *ApplyHandler) ApplyBase(r *gin.Context, mode string) { }() } defer func() { - apply.RollBackAllInstanceUnused(pickers) + if err != nil { + apply.RollBackAllInstanceUnused(pickers) + } }() pickers, err = apply.CycleApply(param) if err != nil { - c.SendResponse(r, err, "", requestId) + c.SendResponse(r, errno.ErrResourceinsufficient.Add(param.BuildMessage()+"\n"+err.Error()), "") return } if param.DryRun { - c.SendResponse(r, nil, map[string]interface{}{"check_success": true}, requestId) + c.SendResponse(r, nil, map[string]interface{}{"check_success": true}) return } data, err := apply.LockReturnPickers(pickers, mode) if err != nil { - c.SendResponse(r, errno.ErresourceLockReturn.AddErr(err), nil, requestId) + c.SendResponse(r, errno.ErresourceLockReturn.AddErr(err), nil) return } - logger.Info(fmt.Sprintf("The %s, will return %d machines", requestId, len(data))) + logger.Info(fmt.Sprintf("The %s, will return %d machines", c.RequestId, len(data))) task.ApplyResponeLogChan <- task.ApplyResponeLogItem{ - RequestId: requestId, + RequestId: c.RequestId, Data: data, } - task.RecordRsOperatorInfoChan <- param.GetOperationInfo(requestId, mode, data) - c.SendResponse(r, nil, data, requestId) + task.RecordRsOperatorInfoChan <- param.GetOperationInfo(c.RequestId, mode, data) + c.SendResponse(r, nil, data) } diff --git a/dbm-services/common/db-resource/internal/controller/controller.go b/dbm-services/common/db-resource/internal/controller/controller.go index 8ee84ac1b3..578bd0539e 100644 --- a/dbm-services/common/db-resource/internal/controller/controller.go +++ b/dbm-services/common/db-resource/internal/controller/controller.go @@ -23,8 +23,10 @@ import ( "github.com/gin-gonic/gin" ) -// BaseHandler TODO -type BaseHandler struct{} +// BaseHandler base handler +type BaseHandler struct { + RequestId string +} // Response http respone type Response struct { @@ -39,12 +41,13 @@ func (c *BaseHandler) Prepare(r *gin.Context, schema interface{}) error { requestId := r.GetString("request_id") if cmutil.IsEmpty(requestId) { err := fmt.Errorf("get request id error ~") - c.SendResponse(r, err, nil, requestId) + c.SendResponse(r, err, nil) return err } + c.RequestId = requestId if err := r.ShouldBind(&schema); err != nil { logger.Error("ShouldBind Failed %s", err.Error()) - c.SendResponse(r, err, nil, requestId) + c.SendResponse(r, err, nil) return err } logger.Info("param is %v", schema) @@ -52,13 +55,13 @@ func (c *BaseHandler) Prepare(r *gin.Context, schema interface{}) error { } // SendResponse retrnurns a response -func (c *BaseHandler) SendResponse(r *gin.Context, err error, data interface{}, requestId string) { +func (c *BaseHandler) SendResponse(r *gin.Context, err error, data interface{}) { code, message := errno.DecodeErr(err) r.JSON(http.StatusOK, Response{ Code: code, Message: message, Data: data, - RequestId: requestId, + RequestId: c.RequestId, }) } @@ -76,13 +79,13 @@ func (c *BackStageHandler) RegisterRouter(engine *gin.Engine) { } } -// RunModuleCheck 运行模块检查 +// RunModuleCheck run module check func (c BackStageHandler) RunModuleCheck(r *gin.Context) { err := task.InspectCheckResource() if err != nil { logger.Error("inspectCheckResource failed %v", err) } - c.SendResponse(r, nil, "Check Success", "") + c.SendResponse(r, nil, "Check Success") } // RunAsyncCmdb async from cmdb @@ -91,5 +94,5 @@ func (c BackStageHandler) RunAsyncCmdb(r *gin.Context) { if err != nil { logger.Error("asyncResourceHardInfo failed %v", err) } - c.SendResponse(r, nil, "async success", "") + c.SendResponse(r, nil, "async success") } diff --git a/dbm-services/common/db-resource/internal/controller/manage/rs_import.go b/dbm-services/common/db-resource/internal/controller/manage/rs_import.go index 3d0309d15a..236d285a91 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/rs_import.go +++ b/dbm-services/common/db-resource/internal/controller/manage/rs_import.go @@ -24,7 +24,6 @@ import ( "dbm-services/common/db-resource/internal/svr/task" "dbm-services/common/db-resource/internal/svr/yunti" "dbm-services/common/go-pubpkg/cc.v3" - "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/errno" "dbm-services/common/go-pubpkg/logger" @@ -35,11 +34,13 @@ import ( // ImportMachParam 资源导入请求参数 type ImportMachParam struct { // ForBizs 业务标签,表示这个资源将来给ForBizs这个业务使用 - ForBiz int `json:"for_biz"` - RsType string `json:"resource_type"` - BkBizId int `json:"bk_biz_id" binding:"number"` - Hosts []HostBase `json:"hosts" binding:"gt=0,dive,required"` - Labels map[string]string `json:"labels"` + ForBiz int `json:"for_biz"` + RsType string `json:"resource_type"` + BkBizId int `json:"bk_biz_id" binding:"number"` + Hosts []HostBase `json:"hosts" binding:"gt=0,dive,required"` + Labels []string `json:"labels"` + // return_resource 如果为ture，则资源导入的时候 检查主机是否存在，存在的话 就把预占用标记改成空闲。 如果主机不存在 则正常导入 + ReturnResource bool `json:"return_resource"` apply.ActionInfo } @@ -60,7 +61,7 @@ func (p ImportMachParam) getIps() []string { func (p ImportMachParam) getIpsByCloudId() (ipMap map[int][]string) { ipMap = make(map[int][]string) for _, v := range p.Hosts { - if !cmutil.IsEmpty(v.Ip) { + if lo.IsNotEmpty(v.Ip) { ipMap[v.BkCloudId] = append(ipMap[v.BkCloudId], v.Ip) } } @@ -82,12 +83,27 @@ func (p *ImportMachParam) existCheck() (err error) { for _, r := range alreadyExistRs { errMsg += fmt.Sprintf(" bk_cloud_id:%d,ip:%s \n", r.BkCloudID, r.IP) } - return fmt.Errorf(errMsg) + return fmt.Errorf("%s", errMsg) } } return nil } +func (p *ImportMachParam) queryInDb(status string) (rs []model.TbRpDetail, err error) { + ipmap := p.getIpsByCloudId() + for cloudId, ips := range ipmap { + var tmpList []model.TbRpDetail + err = model.DB.Self.Table(model.TbRpDetailName()).Where("bk_cloud_id = ? and ip in (?) and status = ? ", cloudId, + ips, status). + Scan(&tmpList).Error + if err != nil { + return nil, err + } + rs = append(rs, tmpList...) + } + return +} + // Import 导入主机资源 func (c *MachineResourceHandler) Import(r *rf.Context) { var input ImportMachParam @@ -95,22 +111,23 @@ func (c *MachineResourceHandler) Import(r *rf.Context) { logger.Error(fmt.Sprintf("Preare Error %s", err.Error())) return } - requestId := r.GetString("request_id") - if err := input.existCheck(); err != nil { - c.SendResponse(r, errno.RepeatedIpExistSystem.Add(err.Error()), requestId, err.Error()) - return + if !input.ReturnResource { + if err := input.existCheck(); err != nil { + c.SendResponse(r, errno.RepeatedIpExistSystem.Add(err.Error()), err.Error()) + return + } } - resp, err := Doimport(input) + resp, err := Doimport(input, c.RequestId) if err != nil { logger.Error(fmt.Sprintf("ImportByIps failed %s", err.Error())) - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } if len(resp.NotFoundInCCHosts) == len(input.Hosts) { - c.SendResponse(r, fmt.Errorf("all machines failed to query cmdb information"), resp, requestId) + c.SendResponse(r, fmt.Errorf("all machines failed to query cmdb information"), resp) return } - c.SendResponse(r, err, resp, requestId) + c.SendResponse(r, err, resp) } // ImportHostResp 导入主机参数 @@ -121,8 +138,7 @@ type ImportHostResp struct { } func (p ImportMachParam) transParamToBytes() (lableJson json.RawMessage, err error) { - // lableJson = []byte("{}") - lableJson, err = json.Marshal(cmutil.CleanStrMap(p.Labels)) + lableJson, err = json.Marshal(p.Labels) if err != nil { logger.Error(fmt.Sprintf("ConverLableToJsonStr Failed,Error:%s", err.Error())) return @@ -139,15 +155,97 @@ func (p ImportMachParam) getJobIpList() (ipList []bk.IPList) { }) } -// Doimport 导入主机获取主机信息 -func Doimport(param ImportMachParam) (resp *ImportHostResp, err error) { +// resetPrepoccupiedResource 重置选中确认的资源改成unused +func resetPrepoccupiedResource(param ImportMachParam, requestId string) (resetIpList []string, err error) { + var allprepoccupiedRsList []model.TbRpDetail + allprepoccupiedRsList, err = param.queryInDb(model.Prepoccupied) + if len(allprepoccupiedRsList) == 0 { + logger.Info("resources without preoccupied status need to be processed") + return + } + defer func() { + var desc string + status := model.StatusSuccess + if err != nil { + status = model.StatusFailed + desc = fmt.Sprintf("failed to reset prepoccupied status, error:%s", err.Error()) + } + ipListBytes, errx := json.Marshal(resetIpList) + if errx != nil { + desc += "failed to serialize ipList" + logger.Error("json marshal failed %s", errx.Error()) + } + // insert operation log + model.DB.Self.Table(model.TbRpOperationInfoTableName()).Create(&model.TbRpOperationInfo{ + RequestID: requestId, + TotalCount: len(allprepoccupiedRsList), + IpList: ipListBytes, + OperationType: "RESET_PREPOCUPED_TO_UNUSED", + Operator: param.ActionInfo.Operator, + BillId: param.ActionInfo.BillId, + Description: desc, + Status: status, + CreateTime: time.Now(), + UpdateTime: time.Now(), + }) + }() + // get all primary key + ids := lo.Map(allprepoccupiedRsList, func(item model.TbRpDetail, _ int) int { + return item.ID + }) + resetIpList = lo.Map(allprepoccupiedRsList, func(item model.TbRpDetail, _ int) string { + return item.IP + }) + db := model.DB.Self.Table(model.TbRpDetailName()).Where("id in (?) and status = ? ", ids, model.Prepoccupied). + Update("status", model.Unused) + if err = db.Error; err != nil { + logger.Error("reset prepoccupied status failed %s", err.Error()) + return + } + if db.RowsAffected != int64(len(allprepoccupiedRsList)) { + logger.Error("reset prepoccupied status failed %d rows affected", db.RowsAffected) + return + } + return +} + +func filerUnusedRs(param ImportMachParam) (ret []string, err error) { + rsList, err := param.queryInDb(model.Unused) + if err != nil { + return nil, err + } + okRsIplist := lo.Map(rsList, func(item model.TbRpDetail, _ int) string { + return item.IP + }) + ips := param.getIps() + ret, _ = lo.Difference(ips, okRsIplist) + return +} + +// Doimport do import +func Doimport(param ImportMachParam, requestId string) (resp *ImportHostResp, err error) { var ccHostsInfo []*cc.Host var derr error var diskResp bk.GetDiskResp var notFoundHosts, gseAgentIds []string var elems []model.TbRpDetail resp = &ImportHostResp{} - targetHosts := cmutil.RemoveDuplicate(param.getIps()) + targetHosts := lo.Uniq(param.getIps()) + if param.ReturnResource { + targetHosts, err = filerUnusedRs(param) + if err != nil { + return resp, err + } + resetIpList, errx := resetPrepoccupiedResource(param, requestId) + if errx != nil { + return resp, errx + } + subtargetHosts, _ := lo.Difference(targetHosts, resetIpList) + targetHosts = subtargetHosts + } + if len(targetHosts) == 0 { + return resp, nil + } ccHostsInfo, notFoundHosts, derr = bk.BatchQueryHostsInfo(param.BkBizId, targetHosts) if derr != nil { logger.Error("query cc info from cmdb failed %s", derr.Error()) @@ -175,8 +273,6 @@ func Doimport(param ImportMachParam) (resp *ImportHostResp, err error) { for _, emptyhost := range notFoundHosts { delete(hostsMap, emptyhost) } - logger.Info("yunti config %v", config.AppConfig.Yunti) - // if yunti config is not empty var cvmInfoMap map[string]yunti.InstanceDetail if config.AppConfig.Yunti.IsNotEmpty() { logger.Info("try to get machine info from yunti") @@ -192,7 +288,7 @@ func Doimport(param ImportMachParam) (resp *ImportHostResp, err error) { el.SetMore(h.InnerIP, diskResp.IpLogContentMap) // gse agent 1.0的 agent 是用 cloudid:ip gseAgentId := h.BkAgentId - if cmutil.IsEmpty(gseAgentId) { + if lo.IsEmpty(gseAgentId) { gseAgentId = fmt.Sprintf("%d:%s", h.BkCloudId, h.InnerIP) } gseAgentIds = append(gseAgentIds, gseAgentId) @@ -251,7 +347,7 @@ func getCvmMachList(hosts []*cc.Host) []string { // transHostInfoToDbModule 获取的到的主机信息赋值给db model func (p ImportMachParam) transHostInfoToDbModule(h *cc.Host, bkCloudId int, label []byte) model.TbRpDetail { osType := h.BkOsType - if cmutil.IsEmpty(osType) { + if lo.IsEmpty(osType) { osType = bk.OsLinux } return model.TbRpDetail{ @@ -262,7 +358,7 @@ func (p ImportMachParam) transHostInfoToDbModule(h *cc.Host, bkCloudId int, labe AssetID: h.AssetID, BkHostID: h.BKHostId, IP: h.InnerIP, - Label: label, + Labels: label, DeviceClass: h.DeviceClass, DramCap: h.BkMem, CPUNum: h.BkCpu, diff --git a/dbm-services/common/db-resource/internal/controller/manage/rs_lable.go b/dbm-services/common/db-resource/internal/controller/manage/rs_lable.go index 0bfc3b53f0..9c9f45697a 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/rs_lable.go +++ b/dbm-services/common/db-resource/internal/controller/manage/rs_lable.go @@ -29,24 +29,24 @@ func (c *LableHandler) Edit(r *rf.Context) { logger.Error(fmt.Sprintf("Preare Error %s", err.Error())) return } - requestId := r.GetString("request_id") + // requestId := r.GetString("request_id") lableJson, err := cmutil.ConverMapToJsonStr(cmutil.CleanStrMap(input.Labels)) if err != nil { logger.Error(fmt.Sprintf("ConverLableToJsonStr Failed,Error:%s", err.Error())) - c.SendResponse(r, err, nil, requestId) + c.SendResponse(r, err, nil) return } if len(input.BkHostIds) == 0 { - c.SendResponse(r, nil, nil, requestId) + c.SendResponse(r, nil, nil) return } - err = model.DB.Self.Table(model.TbRpDetailName()).Where("bk_host_id in ? ", input.BkHostIds).Update("label", + err = model.DB.Self.Table(model.TbRpDetailName()).Where("bk_host_id in ? ", input.BkHostIds).Update("labels", lableJson). Error if err != nil { logger.Error(fmt.Sprintf("Update Lable Failed %s", err.Error())) - c.SendResponse(r, err, nil, requestId) + c.SendResponse(r, err, nil) return } - c.SendResponse(r, nil, nil, requestId) + c.SendResponse(r, nil, nil) } diff --git a/dbm-services/common/db-resource/internal/controller/manage/rs_list.go b/dbm-services/common/db-resource/internal/controller/manage/rs_list.go index 8b59d354c1..415ef80aff 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/rs_list.go +++ b/dbm-services/common/db-resource/internal/controller/manage/rs_list.go @@ -34,7 +34,7 @@ type MachineResourceGetterInputParam struct { City []string `json:"city"` SubZoneIds []string `json:"subzone_ids"` DeviceClass []string `json:"device_class"` - Labels map[string]string `json:"labels"` + Labels []string `json:"labels"` Hosts []string `json:"hosts"` BkCloudIds []int `json:"bk_cloud_ids"` RsType string `json:"resource_type"` @@ -62,18 +62,17 @@ func (c *MachineResourceHandler) List(r *rf.Context) { if c.Prepare(r, &input) != nil { return } - requestId := r.GetString("request_id") if err := input.paramCheck(); err != nil { - c.SendResponse(r, errno.ErrErrInvalidParam.AddErr(err), nil, requestId) + c.SendResponse(r, errno.ErrErrInvalidParam.AddErr(err), nil) return } db := model.DB.Self.Table(model.TbRpDetailName()) if err := input.queryBs(db); err != nil { - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } if err := db.Count(&count).Error; err != nil { - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } if input.Limit > 0 { @@ -81,10 +80,10 @@ func (c *MachineResourceHandler) List(r *rf.Context) { } var data []model.TbRpDetail if err := db.Find(&data).Error; err != nil { - c.SendResponse(r, errno.ErrDBQuery.AddErr(err), requestId, err.Error()) + c.SendResponse(r, errno.ErrDBQuery.AddErr(err), err.Error()) return } - c.SendResponse(r, nil, map[string]interface{}{"details": data, "count": count}, requestId) + c.SendResponse(r, nil, map[string]interface{}{"details": data, "count": count}) } func (c *MachineResourceGetterInputParam) paramCheck() (err error) { @@ -193,7 +192,9 @@ func (c *MachineResourceGetterInputParam) queryBs(db *gorm.DB) (err error) { if len(c.SubZoneIds) > 0 { db.Where(" sub_zone_id in (?) ", c.SubZoneIds) } - + if len(c.Labels) > 0 { + db.Where(model.JSONQuery("labels").JointOrContains(c.Labels)) + } if cmutil.IsNotEmpty(c.OsType) { db.Where("os_type = ?", c.OsType) } @@ -203,19 +204,19 @@ func (c *MachineResourceGetterInputParam) queryBs(db *gorm.DB) (err error) { // ListAll TODO func (c *MachineResourceHandler) ListAll(r *rf.Context) { - requestId := r.GetString("request_id") + // requestId := r.GetString("request_id") var data []model.TbRpDetail db := model.DB.Self.Table(model.TbRpDetailName()).Where("status in (?)", []string{model.Unused, model.Prepoccupied, model.Preselected}) err := db.Scan(&data).Error if err != nil { - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } var count int64 if err := db.Count(&count).Error; err != nil { - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } - c.SendResponse(r, nil, map[string]interface{}{"details": data, "count": count}, requestId) + c.SendResponse(r, nil, map[string]interface{}{"details": data, "count": count}) } diff --git a/dbm-services/common/db-resource/internal/controller/manage/rs_manage.go b/dbm-services/common/db-resource/internal/controller/manage/rs_manage.go index 88803cec6b..c320b04df1 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/rs_manage.go +++ b/dbm-services/common/db-resource/internal/controller/manage/rs_manage.go @@ -55,6 +55,7 @@ func (c *MachineResourceHandler) RegisterRouter(engine *rf.Engine) { r.POST("/operation/list", c.OperationInfoList) r.POST("/operation/create", c.RecordImportResource) r.POST("/spec/sum", c.SpecSum) + r.POST("/groupby/label/count", c.GroupByLabelCount) } } @@ -70,24 +71,24 @@ func (c *MachineResourceHandler) Delete(r *rf.Context) { logger.Error("Preare Error %s", err.Error()) return } - requestId := r.GetString("request_id") affect_row, err := model.DeleteTbRpDetail(input.BkHostIds) if err != nil { logger.Error("failed to delete data:%s", err.Error()) - c.SendResponse(r, err, nil, requestId) + c.SendResponse(r, err, nil) return } if affect_row == 0 { - c.SendResponse(r, fmt.Errorf("no data was deleted"), nil, requestId) + c.SendResponse(r, fmt.Errorf("no data was deleted"), nil) return } - c.SendResponse(r, nil, requestId, "Delete Success") + c.SendResponse(r, nil, "Delete Success") } // BatchUpdateMachineInput 批量编辑主机信息请求参数 type BatchUpdateMachineInput struct { BkHostIds []int `json:"bk_host_ids" binding:"required,dive,gt=0" ` ForBiz int `json:"for_biz"` + Labels []string `json:"labels"` RsType string `json:"resource_type"` RackId string `json:"rack_id"` SetBizEmpty bool `json:"set_empty_biz"` @@ -103,10 +104,12 @@ const ( // BatchUpdate 批量编辑主机信息 func (c *MachineResourceHandler) BatchUpdate(r *rf.Context) { var input BatchUpdateMachineInput - requestId := r.GetString("request_id") + var err error + var lableJson, storageJson []byte + updateMap := make(map[string]interface{}) - if err := c.Prepare(r, &input); err != nil { + if err = c.Prepare(r, &input); err != nil { logger.Error("Preare Error %s", err.Error()) return } @@ -120,13 +123,20 @@ func (c *MachineResourceHandler) BatchUpdate(r *rf.Context) { if lo.IsNotEmpty(input.RsType) { updateMap["rs_type"] = input.RsType } + lableJson, err = json.Marshal(input.Labels) + if err != nil { + logger.Error(fmt.Sprintf("ConverLableToJsonStr Failed,Error:%s", err.Error())) + c.SendResponse(r, err, err.Error()) + return + } + updateMap["labels"] = lableJson // update disk if len(input.StorageDevice) > 0 { - storageJson, err := json.Marshal(input.StorageDevice) + storageJson, err = json.Marshal(input.StorageDevice) if err != nil { logger.Error(fmt.Sprintf("conver resource types Failed,Error:%s", err.Error())) - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } updateMap["storage_device"] = storageJson @@ -138,15 +148,16 @@ func (c *MachineResourceHandler) BatchUpdate(r *rf.Context) { } // do update - err := model.DB.Self.Table(model.TbRpDetailName()).Select("dedicated_biz", "rs_type", "storage_device", "rack_id"). + err = model.DB.Self.Table(model.TbRpDetailName()).Select("dedicated_biz", "rs_type", "storage_device", "rack_id", + "labels"). Where("bk_host_id in (?)", input.BkHostIds).Updates(updateMap).Error if err != nil { - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } // return respone - c.SendResponse(r, nil, "ok", requestId) + c.SendResponse(r, nil, "ok") } // MachineResourceInputParam 多个不同的主句的编辑的不同的参数 @@ -157,7 +168,7 @@ type MachineResourceInputParam struct { // MachineResource 主机参数 type MachineResource struct { BkHostID int `json:"bk_host_id" binding:"required"` - Labels map[string]string `json:"labels"` + Labels []string `json:"labels"` ForBiz int `json:"for_biz"` RsType string `json:"resource_type"` StorageDevice map[string]bk.DiskDetail `json:"storage_device"` @@ -166,7 +177,6 @@ type MachineResource struct { // Update 编辑主机信息 func (c *MachineResourceHandler) Update(r *rf.Context) { var input MachineResourceInputParam - requestId := r.GetString("request_id") if err := c.Prepare(r, &input); err != nil { logger.Error("Preare Error %s", err.Error()) return @@ -176,11 +186,13 @@ func (c *MachineResourceHandler) Update(r *rf.Context) { for _, v := range input.Data { updateMap := make(map[string]interface{}) if len(v.Labels) > 0 { - l, err := cmutil.ConverMapToJsonStr(v.Labels) + lableJson, err := json.Marshal(v.Labels) if err != nil { - logger.Error(fmt.Sprintf("ConverMapToJsonStr Failed %s", err.Error())) + logger.Error(fmt.Sprintf("ConverLableToJsonStr Failed,Error:%s", err.Error())) + c.SendResponse(r, err, err.Error()) + return } - updateMap["lable"] = l + updateMap["labels"] = lableJson } if v.ForBiz > 0 { updateMap["dedicated_biz"] = v.ForBiz @@ -192,25 +204,25 @@ func (c *MachineResourceHandler) Update(r *rf.Context) { storageJson, err := json.Marshal(v.StorageDevice) if err != nil { logger.Error(fmt.Sprintf("conver resource types Failed,Error:%s", err.Error())) - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } updateMap["storage_device"] = storageJson } err := tx.Model(&model.TbRpDetail{}).Table(model.TbRpDetailName()).Select("dedicated_biz", "rs_type", - "label").Where("bk_host_id=?", v.BkHostID).Updates(updateMap).Error + "labels").Where("bk_host_id=?", v.BkHostID).Updates(updateMap).Error if err != nil { tx.Rollback() logger.Error(fmt.Sprintf("conver resource types Failed,Error:%s", err.Error())) - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } } if err := tx.Commit().Error; err != nil { - c.SendResponse(r, err, requestId, err.Error()) + c.SendResponse(r, err, err.Error()) return } - c.SendResponse(r, nil, requestId, "Save Success") + c.SendResponse(r, nil, "Save Success") } // GetMountPoints get disk mount points @@ -219,7 +231,7 @@ func (c MachineResourceHandler) GetMountPoints(r *rf.Context) { var rs []json.RawMessage if err := db.Select("json_keys(storage_device)").Where("JSON_LENGTH(storage_device) > 0").Find(&rs).Error; err != nil { logger.Error("get mountpoints failed %s", err.Error()) - c.SendResponse(r, err, err.Error(), "") + c.SendResponse(r, err, err.Error()) return } var mountpoints []string @@ -227,14 +239,14 @@ func (c MachineResourceHandler) GetMountPoints(r *rf.Context) { var mp []string if err := json.Unmarshal(v, &mp); err != nil { logger.Error("unmarshal failed %s", err.Error()) - c.SendResponse(r, err, err.Error(), "") + c.SendResponse(r, err, err.Error()) return } if len(mp) > 0 { mountpoints = append(mountpoints, mp...) } } - c.SendResponse(r, nil, cmutil.RemoveDuplicate(mountpoints), r.GetString("request_id")) + c.SendResponse(r, nil, cmutil.RemoveDuplicate(mountpoints)) } // GetDiskTypes get disk types @@ -245,7 +257,7 @@ func (c MachineResourceHandler) GetDiskTypes(r *rf.Context) { Find(&rs).Error if err != nil { logger.Error("get DiskType failed %s", err.Error()) - c.SendResponse(r, err, err.Error(), "") + c.SendResponse(r, err, err.Error()) return } var diskTypes []string @@ -253,14 +265,14 @@ func (c MachineResourceHandler) GetDiskTypes(r *rf.Context) { var mp []string if err := json.Unmarshal(v, &mp); err != nil { logger.Error("unmarshal failed %s", err.Error()) - c.SendResponse(r, err, err.Error(), "") + c.SendResponse(r, err, err.Error()) return } if len(mp) > 0 { diskTypes = append(diskTypes, mp...) } } - c.SendResponse(r, nil, cmutil.RemoveDuplicate(diskTypes), r.GetString("request_id")) + c.SendResponse(r, nil, cmutil.RemoveDuplicate(diskTypes)) } // GetSubZoneParam get subzones param @@ -278,10 +290,10 @@ func (c MachineResourceHandler) GetSubZones(r *rf.Context) { db := model.DB.Self.Table(model.TbRpDetailName()) err := db.Distinct("sub_zone").Where("city in ? ", input.LogicCitys).Find(&subZones).Error if err != nil { - c.SendResponse(r, err, "", err.Error()) + c.SendResponse(r, err, err.Error()) return } - c.SendResponse(r, nil, subZones, r.GetString("request_id")) + c.SendResponse(r, nil, subZones) } // GetDeviceClass 获取机型 @@ -290,8 +302,35 @@ func (c MachineResourceHandler) GetDeviceClass(r *rf.Context) { db := model.DB.Self.Table(model.TbRpDetailName()) err := db.Distinct("device_class").Where("device_class !=''").Find(&class).Error if err != nil { - c.SendResponse(r, err, "", err.Error()) + c.SendResponse(r, err, err.Error()) + return + } + c.SendResponse(r, nil, class) +} + +// GroupByLabelCount group by label count +func (c *MachineResourceHandler) GroupByLabelCount(r *rf.Context) { + var rs []model.TbRpDetail + err := model.DB.Self.Table(model.TbRpDetailName()).Find(&rs, "status = ?", model.Unused).Error + if err != nil { + c.SendResponse(r, err, err.Error()) return } - c.SendResponse(r, nil, class, r.GetString("request_id")) + logger.Info("rs len %d", len(rs)) + ret := make(map[string]int) + for _, v := range rs { + var lables []string + logger.Info("labels %s", string(v.Labels)) + if err := json.Unmarshal(v.Labels, &lables); err != nil { + logger.Error("unmarshal failed %s", err.Error()) + c.SendResponse(r, err, err.Error()) + return + } + if len(lables) > 0 { + for _, l := range lables { + ret[l]++ + } + } + } + c.SendResponse(r, nil, ret) } diff --git a/dbm-services/common/db-resource/internal/controller/manage/rs_operation_info.go b/dbm-services/common/db-resource/internal/controller/manage/rs_operation_info.go index fe83ca94e4..3f00f3f736 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/rs_operation_info.go +++ b/dbm-services/common/db-resource/internal/controller/manage/rs_operation_info.go @@ -42,10 +42,9 @@ type GetOperationInfoParam struct { Offset int `json:"offset"` } -// OperationInfoList TODO +// OperationInfoList returns the list of operation info func (o MachineResourceHandler) OperationInfoList(r *gin.Context) { var input GetOperationInfoParam - requestId := r.GetString("request_id") if err := o.Prepare(r, &input); err != nil { logger.Error(fmt.Sprintf("Preare Error %s", err.Error())) return @@ -54,7 +53,7 @@ func (o MachineResourceHandler) OperationInfoList(r *gin.Context) { input.query(db) var count int64 if err := db.Count(&count).Error; err != nil { - o.SendResponse(r, errno.ErrDBQuery.AddErr(err), requestId, err.Error()) + o.SendResponse(r, errno.ErrDBQuery.AddErr(err), err.Error()) return } var data []model.TbRpOperationInfo @@ -62,11 +61,11 @@ func (o MachineResourceHandler) OperationInfoList(r *gin.Context) { db = db.Offset(input.Offset).Limit(input.Limit) } if err := db.Scan(&data).Error; err != nil { - o.SendResponse(r, errno.ErrDBQuery.AddErr(err), err.Error(), requestId) + o.SendResponse(r, errno.ErrDBQuery.AddErr(err), err.Error()) return } - o.SendResponse(r, nil, map[string]interface{}{"details": data, "count": count}, requestId) + o.SendResponse(r, nil, map[string]interface{}{"details": data, "count": count}) } func (p GetOperationInfoParam) query(db *gorm.DB) { @@ -134,12 +133,12 @@ func (o MachineResourceHandler) RecordImportResource(r *gin.Context) { } ipJsStr, err := json.Marshal(ipList) if err != nil { - o.SendResponse(r, errno.ErrJSONMarshal.Add("input ips"), "failed", "") + o.SendResponse(r, errno.ErrJSONMarshal.Add("input ips"), "failed") return } bkHostIdJsStr, err := json.Marshal(hostIdList) if err != nil { - o.SendResponse(r, errno.ErrJSONMarshal.Add("input hostids"), "failed", "") + o.SendResponse(r, errno.ErrJSONMarshal.Add("input hostids"), "failed") } m := model.TbRpOperationInfo{ TotalCount: len(ipList), @@ -152,7 +151,7 @@ func (o MachineResourceHandler) RecordImportResource(r *gin.Context) { CreateTime: time.Now(), } if err := model.DB.Self.Table(model.TbRpOperationInfoTableName()).Create(&m).Error; err != nil { - o.SendResponse(r, fmt.Errorf("记录导入任务日志失败,%w", err), err.Error(), "") + o.SendResponse(r, fmt.Errorf("记录导入任务日志失败,%w", err), err.Error()) } - o.SendResponse(r, nil, "记录日志成功", "") + o.SendResponse(r, nil, "记录日志成功") } diff --git a/dbm-services/common/db-resource/internal/controller/manage/rs_spec.go b/dbm-services/common/db-resource/internal/controller/manage/rs_spec.go index 903da29b04..d0b08f576e 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/rs_spec.go +++ b/dbm-services/common/db-resource/internal/controller/manage/rs_spec.go @@ -38,15 +38,15 @@ type SpecInfo struct { GroupMark string `json:"group_mark" binding:"required" ` DeviceClass []string `json:"device_class"` Spec meta.Spec `json:"spec"` + Labels []string `json:"labels"` StorageSpecs []meta.DiskSpec `json:"storage_spec"` } // SpecSum TODO func (m MachineResourceHandler) SpecSum(r *gin.Context) { var input SpecCheckInput - requestId := r.GetString("request_id") if err := m.Prepare(r, &input); err != nil { - m.SendResponse(r, err, err.Error(), requestId) + m.SendResponse(r, err, err.Error()) return } rpdata := make(map[string]int64) @@ -58,7 +58,7 @@ func (m MachineResourceHandler) SpecSum(r *gin.Context) { idcCitys, err = dbmapi.GetIdcCityByLogicCity(input.LocationSpec.City) if err != nil { logger.Error("request real citys by logic city %s from bkdbm api failed:%v", input.LocationSpec.City, err) - m.SendResponse(r, err, err.Error(), requestId) + m.SendResponse(r, err, err.Error()) return } } @@ -93,12 +93,13 @@ func (m MachineResourceHandler) SpecSum(r *gin.Context) { s.MatchStorage(db) s.MatchSpec(db) s.MatchLocationSpec(db) + s.MatchLabels(db) if err := db.Scan(&count).Error; err != nil { logger.Error("query pre check count failed %s", err.Error()) - m.SendResponse(r, errno.ErrDBQuery.AddErr(err), err.Error(), requestId) + m.SendResponse(r, errno.ErrDBQuery.AddErr(err), err.Error()) return } rpdata[item.GroupMark] = count } - m.SendResponse(r, nil, rpdata, requestId) + m.SendResponse(r, nil, rpdata) } diff --git a/dbm-services/common/db-resource/internal/controller/statistic/statistic.go b/dbm-services/common/db-resource/internal/controller/statistic/statistic.go index 3fde36d190..6672050f85 100644 --- a/dbm-services/common/db-resource/internal/controller/statistic/statistic.go +++ b/dbm-services/common/db-resource/internal/controller/statistic/statistic.go @@ -51,8 +51,6 @@ func (s *Handler) RegisterRouter(engine *gin.Engine) { } } -// CountGroupbyResourceType 按照资源类型统计 - // CountGroupbyResourceTypeResult 按照资源类型统计结果数据 type CountGroupbyResourceTypeResult struct { RsType string `json:"rs_type"` @@ -67,9 +65,9 @@ func (s *Handler) CountGroupbyResourceType(c *gin.Context) { Find(&data, "status = ? ", model.Unused).Error if err != nil { logger.Error("query failed %s", err.Error) - s.SendResponse(c, err, err.Error(), "") + s.SendResponse(c, err, err.Error()) } - s.SendResponse(c, nil, data, "") + s.SendResponse(c, nil, data) } // ResourDistributionParam 统计资源分布参数 @@ -117,7 +115,7 @@ func (s *Handler) ResourceDistribution(c *gin.Context) { var param ResourDistributionParam if err := s.Prepare(c, ¶m); err != nil { logger.Error("parse ResourDistributionParam failed: %v", err) - s.SendResponse(c, err, "Failed to parse parameters", "") + s.SendResponse(c, err, "Failed to parse parameters") return } @@ -125,13 +123,13 @@ func (s *Handler) ResourceDistribution(c *gin.Context) { specList, err := dbmClient.GetDbmSpec(param.SpecParam.getQueryParam()) if err != nil { logger.Error("get dbm spec failed: %v", err) - s.SendResponse(c, err, "Failed to get DBM specifications", "") + s.SendResponse(c, err, "Failed to get DBM specifications") return } allLogicCityInfos, err := dbmapi.GetAllLogicCityInfo() if err != nil { logger.Error("get all logic city info failed: %v", err) - s.SendResponse(c, err, "Failed to get logic city info", "") + s.SendResponse(c, err, "Failed to get logic city info") return } cityMap := make(map[string]string) @@ -140,14 +138,14 @@ func (s *Handler) ResourceDistribution(c *gin.Context) { } db := model.DB.Self.Table(model.TbRpDetailName()) if err := param.dbFilter(db); err != nil { - s.SendResponse(c, err, "Failed to apply database filter", "") + s.SendResponse(c, err, "Failed to apply database filter") return } var rsListBefore, rsList []model.TbRpDetail if err := db.Find(&rsListBefore).Error; err != nil { logger.Error("query failed: %v", err) - s.SendResponse(c, err, "Failed to query resource list", "") + s.SendResponse(c, err, "Failed to query resource list") return } for _, rs := range rsListBefore { @@ -167,16 +165,16 @@ func (s *Handler) ResourceDistribution(c *gin.Context) { default: err := errors.New("unknown aggregation type") msg := fmt.Sprintf("Unknown aggregation type: %s", param.GroupBy) - s.SendResponse(c, err, msg, "") + s.SendResponse(c, err, msg) return } if processErr != nil { - s.SendResponse(c, processErr, "Failed to process data", "") + s.SendResponse(c, processErr, "Failed to process data") return } - s.SendResponse(c, nil, result, "") + s.SendResponse(c, nil, result) } func (s *Handler) processDeviceClassGroup( @@ -184,6 +182,7 @@ func (s *Handler) processDeviceClassGroup( specList []dbmapi.DbmSpec, clusterType string, ) interface{} { + if lo.IsEmpty(clusterType) { return groupByDeviceClass(rsList) } @@ -195,6 +194,7 @@ func (s *Handler) processDeviceClassGroup( filteredList = append(filteredList, rs) break } + } } return groupByDeviceClass(filteredList) @@ -219,13 +219,14 @@ func (r ResourDistributionParam) dbFilter(db *gorm.DB) (err error) { if !r.SetRsTypeEmpty { db.Where("rs_type = ?", r.SpecParam.DbType) } + db.Where("os_type = ? ", model.LiunxOs) + if strings.Contains(strings.ToLower(r.SpecParam.DbType), "sqlserver") { + db.Where("os_type = ? ", model.WindowsOs) + } return nil } func dealCity(city string) string { - // if lo.IsEmpty(city) { - // city = "无区域信息" - // } return city } diff --git a/dbm-services/common/db-resource/internal/model/tb_rp_daily_snapshot.go b/dbm-services/common/db-resource/internal/model/TbRpDailySnapShot.go similarity index 100% rename from dbm-services/common/db-resource/internal/model/tb_rp_daily_snapshot.go rename to dbm-services/common/db-resource/internal/model/TbRpDailySnapShot.go diff --git a/dbm-services/common/db-resource/internal/model/TbRpDetail.go b/dbm-services/common/db-resource/internal/model/TbRpDetail.go index 77c233fbb9..c446006d36 100644 --- a/dbm-services/common/db-resource/internal/model/TbRpDetail.go +++ b/dbm-services/common/db-resource/internal/model/TbRpDetail.go @@ -64,62 +64,87 @@ type TbRpDetail struct { StorageDevice json.RawMessage `gorm:"column:storage_device;type:json;comment:'磁盘设备'" json:"storage_device"` TotalStorageCap int `gorm:"column:total_storage_cap;type:int(11);comment:'磁盘总容量'" json:"total_storage_cap"` Storages map[string]bk.DiskDetail `gorm:"-" json:"-"` + // 操作系统类型 Liunx,Windows /*Linux(1) Windows(2) AIX(3) Unix(4) Solaris(5) FreeBSD(7)*/ - OsType string `gorm:"column:os_type;type:varchar(32);not null;comment:'操作系统类型'" json:"os_type"` // 操作系统类型 Liunx,Windows - OsBit string `gorm:"column:os_bit;type:varchar(32);not null;comment:'操作系统位数'" json:"os_bit"` - OsVerion string `gorm:"column:os_version;type:varchar(64);not null;comment:'操作系统版本'" json:"os_version"` // 操作系统版本 - OsName string `gorm:"column:os_name;type:varchar(64);not null;comment:'操作系统名称'" json:"os_name"` // 操作系统名称 - Raid string `gorm:"column:raid;type:varchar(20);not null" json:"raid"` // 磁盘Raid - CityID string `gorm:"column:city_id;type:varchar(64);not null" json:"city_id"` // 实际城市ID - City string `gorm:"column:city;type:varchar(128);not null" json:"city"` // 实际城市名称 - SubZone string `gorm:"column:sub_zone;type:varchar(32);not null" json:"sub_zone"` // 园区, 例如光明 cc_device_szone - SubZoneID string `gorm:"column:sub_zone_id;type:varchar(64);not null" json:"sub_zone_id"` // 园区ID cc_device_szone_id - RackID string `gorm:"column:rack_id;type:varchar(64);not null" json:"rack_id"` // 存放机架ID,判断是否是同机架 - NetDeviceID string `gorm:"column:net_device_id;type:varchar(128)" json:"net_device_id"` // 网络设备ID, 判断是同交换机 - Label json.RawMessage `gorm:"column:label;type:json" json:"label"` // 标签 - LabelMap map[string]string `gorm:"-" json:"-"` - IsInit int `gorm:"column:is_init;type:int(11);comment:'是否初始化过'" json:"-"` // 是否初始化过 - IsIdle int `gorm:"column:is_idle;type:int(11);comment:'是否空闲检查过'" json:"-"` // 是否空闲检查过 - Status string `gorm:"column:status;type:varchar(20);not null" json:"status"` // Unused: 未使用 Used: 已经售卖被使用: Preselected:预占用 - BkAgentId string `gorm:"index:idx_bk_agent_id;column:bk_agent_id;type:varchar(64);not null" json:"bk_agent_id"` + OsType string `gorm:"column:os_type;type:varchar(32);not null;comment:'操作系统类型'" json:"os_type"` + OsBit string `gorm:"column:os_bit;type:varchar(32);not null;comment:'操作系统位数'" json:"os_bit"` + // 操作系统版本 + OsVerion string `gorm:"column:os_version;type:varchar(64);not null;comment:'操作系统版本'" json:"os_version"` + // 操作系统名称 + OsName string `gorm:"column:os_name;type:varchar(64);not null;comment:'操作系统名称'" json:"os_name"` + // 磁盘Raid + Raid string `gorm:"column:raid;type:varchar(20);not null" json:"raid"` + // 实际城市ID + CityID string `gorm:"column:city_id;type:varchar(64);not null" json:"city_id"` + // 实际城市名称 + City string `gorm:"column:city;type:varchar(128);not null" json:"city"` + // 园区, 例如光明 cc_device_szone + SubZone string `gorm:"column:sub_zone;type:varchar(32);not null" json:"sub_zone"` + // 园区ID cc_device_szone_id + SubZoneID string `gorm:"column:sub_zone_id;type:varchar(64);not null" json:"sub_zone_id"` + // 存放机架ID,判断是否是同机架 + RackID string `gorm:"column:rack_id;type:varchar(64);not null" json:"rack_id"` + // 网络设备ID, 判断是同交换机 + NetDeviceID string `gorm:"column:net_device_id;type:varchar(128)" json:"net_device_id"` + // 标签 + Labels json.RawMessage `gorm:"column:labels;type:json" json:"labels"` + // 是否初始化过 + IsInit int `gorm:"column:is_init;type:int(11);comment:'是否初始化过'" json:"-"` + // 是否空闲检查过 + IsIdle int `gorm:"column:is_idle;type:int(11);comment:'是否空闲检查过'" json:"-"` + // Unused: 未使用 Used: 已经售卖被使用: Preselected:预占用 + Status string `gorm:"column:status;type:varchar(20);not null" json:"status"` + BkAgentId string `gorm:"index:idx_bk_agent_id;column:bk_agent_id;type:varchar(64);not null" json:"bk_agent_id"` // gse Agent当前运行状态码, -1:未知 0:初始安装 1:启动中 2:运行中 3:有损状态 4:繁忙状态 5:升级中 6:停止中 7:解除安装 AgentStatusCode int `gorm:"column:gse_agent_status_code;type:int(11);not null" json:"gse_agent_status_code"` // agent status 最后一次更新时间 AgentStatusUpdateTime time.Time `gorm:"column:agent_status_update_time;type:timestamp;default:1970-01-01 08:00:01" json:"agent_status_update_time"` - ConsumeTime time.Time `gorm:"column:consume_time;type:timestamp;default:1970-01-01 08:00:01" json:"consume_time"` // 消费时间 - UpdateTime time.Time `gorm:"column:update_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"update_time"` // 最后修改时间 - CreateTime time.Time `gorm:"column:create_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"create_time"` // 创建时间 + // 消费时间 + ConsumeTime time.Time `gorm:"column:consume_time;type:timestamp;default:1970-01-01 08:00:01" json:"consume_time"` + // 最后修改时间 + UpdateTime time.Time `gorm:"column:update_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"update_time"` + // 创建时间 + CreateTime time.Time `gorm:"column:create_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"create_time"` // foreiginKey:关联表的结构字段 references:当前表的结构字段 // SubStorages []TbRpStorageItem `gorm:"foreignKey:BkHostID;references:BkHostID"` } +const ( + // LiunxOs linux + LiunxOs = "Linux" + // WindowsOs windows + WindowsOs = "Windows" + // UnixOs unix + UnixOs = "Unix" +) + // ConvertOsTypeToHuman 转换系统类型到可读字符 // // /*Linux(1) Windows(2) AIX(3) Unix(4) Solaris(5) FreeBSD(7)*/ func ConvertOsTypeToHuman(osType string) string { switch osType { case "1": - return "Linux" + return LiunxOs case "2": - return "Windows" + return WindowsOs case "4": - return "Unix" + return UnixOs default: return "Unknown Operating System" } } -// TableName TODO +// TableName table name func (TbRpDetail) TableName() string { return TbRpDetailName() } -// TbRpDetailName TODO +// TbRpDetailName tbrp detail table name func TbRpDetailName() string { return "tb_rp_detail" } -// MatchDbmSpec 资源是否匹配dbm的规格 +// MatchDbmSpec whether the resource matches dbm specifications func (t TbRpDetail) MatchDbmSpec(spec dbmapi.DbmSpec) bool { logger.Info("spec:%+v", spec) logger.Info("cpu:%d,mem:%d,city:%s,disk:%s", t.CPUNum, t.DramCap, t.City, string(t.StorageDevice)) @@ -259,7 +284,7 @@ type BatchGetTbDetailResult struct { Data []TbRpDetail `json:"data"` } -// BatchGetSatisfiedByAssetIds 批量设置资源状态 +// BatchGetSatisfiedByAssetIds batch setting resource status func BatchGetSatisfiedByAssetIds(elements []BatchGetTbDetail, mode string) (result []BatchGetTbDetailResult, err error) { db := DB.Self.Begin() @@ -272,20 +297,20 @@ func BatchGetSatisfiedByAssetIds(elements []BatchGetTbDetail, mode string) (resu for _, v := range elements { d, err = SetSatisfiedStatus(db, v.BkHostIds, mode) if err != nil { - logger.Error(fmt.Sprintf("Item:%s,failed to obtain resource details!,Error is %s", v.Item, err.Error())) + logger.Error("Item:%s,failed to obtain resource details!,Error is %s", v.Item, err.Error()) return nil, err } result = append(result, BatchGetTbDetailResult{Item: v.Item, Data: d}) } err = db.Commit().Error if err != nil { - logger.Error(fmt.Sprintf("transaction commit failed: %s", err.Error())) + logger.Error("transaction commit failed: %v", err) return nil, err } return } -// SetSatisfiedStatus 获取满足条件的的资源,并更新状态 +// SetSatisfiedStatus get resources that meet the conditions and update status func SetSatisfiedStatus(tx *gorm.DB, bkhostIds []int, status string) (result []TbRpDetail, err error) { err = tx.Exec("select * from tb_rp_detail where bk_host_id in (?) for update", bkhostIds).Error if err != nil { @@ -296,12 +321,12 @@ func SetSatisfiedStatus(tx *gorm.DB, bkhostIds []int, status string) (result []T return nil, err } if len(bkhostIds) != len(result) { - logger.Error(fmt.Sprintf("Get TbRpDetail is %v", result)) + logger.Error("Get TbRpDetail is %v", result) return nil, fmt.Errorf("requried count is %d,But Only Get %d", len(bkhostIds), len(result)) } rdb := tx.Exec("update tb_rp_detail set status=?,consume_time=now() where bk_host_id in ?", status, bkhostIds) if rdb.Error != nil { - logger.Error(fmt.Sprintf("update status Failed,Error %v", rdb.Error)) + logger.Error("update status Failed,Error %v", rdb.Error) return nil, err } if int(rdb.RowsAffected) != len(bkhostIds) { diff --git a/dbm-services/common/db-resource/internal/model/TbRpDetailArchive.go b/dbm-services/common/db-resource/internal/model/TbRpDetailArchive.go index 36768bb50a..886f5c429a 100644 --- a/dbm-services/common/db-resource/internal/model/TbRpDetailArchive.go +++ b/dbm-services/common/db-resource/internal/model/TbRpDetailArchive.go @@ -12,7 +12,6 @@ package model import ( "encoding/json" - "fmt" "time" "dbm-services/common/db-resource/internal/svr/bk" @@ -38,22 +37,30 @@ type TbRpDetailArchive struct { StorageDevice json.RawMessage `gorm:"column:storage_device;type:json;comment:'磁盘设备'" json:"storage_device"` TotalStorageCap int `gorm:"column:total_storage_cap;type:int(11);comment:'磁盘总容量'" json:"total_storage_cap"` Storages map[string]bk.DiskDetail `gorm:"-"` - /*Linux(1) Windows(2) AIX(3) Unix(4) Solaris(5) FreeBSD(7)*/ - OsType string `gorm:"column:os_type;type:varchar(32);not null;comment:'操作系统类型'" json:"os_type"` // 操作系统类型 Liunx,Windows - OsBit string `gorm:"column:os_bit;type:varchar(32);not null;comment:'操作系统位数'" json:"os_bit"` - OsVerion string `gorm:"column:os_version;type:varchar(64);not null;comment:'操作系统版本'" json:"os_version"` // 操作系统版本 - OsName string `gorm:"column:os_name;type:varchar(64);not null;comment:'操作系统名称'" json:"os_name"` // 操作系统名称 - Raid string `gorm:"column:raid;type:varchar(20);not null" json:"raid"` // 磁盘Raid - CityID string `gorm:"column:city_id;type:varchar(64);not null" json:"city_id"` // 实际城市ID - City string `gorm:"column:city;type:varchar(128);not null" json:"city"` // 实际城市名称 - SubZone string `gorm:"column:sub_zone;type:varchar(32);not null" json:"sub_zone"` // 园区, 例如光明 cc_device_szone - SubZoneID string `gorm:"column:sub_zone_id;type:varchar(64);not null" json:"sub_zone_id"` // 园区ID cc_device_szone_id - RackID string `gorm:"column:rack_id;type:varchar(64);not null" json:"rack_id"` // 存放机架ID,判断是否是同机架 - NetDeviceID string `gorm:"column:net_device_id;type:varchar(128)" json:"net_device_id"` // 网络设备ID, 判断是同交换机 - Label string `gorm:"column:label;type:json" json:"label"` // 标签 - LabelMap map[string]string `gorm:"-"` - IsInit int `gorm:"column:is_init;type:int(11);comment:'是否初始化过'" json:"-"` // 是否初始化过 - IsIdle int `gorm:"column:is_idle;type:int(11);comment:'是否空闲检查过'" json:"-"` // 是否空闲检查过 + OsType string `gorm:"column:os_type;type:varchar(32);not null;comment:'操作系统类型'" json:"os_type"` + OsBit string `gorm:"column:os_bit;type:varchar(32);not null;comment:'操作系统位数'" json:"os_bit"` + // 操作系统版本 + OsVerion string `gorm:"column:os_version;type:varchar(64);not null;comment:'操作系统版本'" json:"os_version"` + // 操作系统名称 + OsName string `gorm:"column:os_name;type:varchar(64);not null;comment:'操作系统名称'" json:"os_name"` + // 磁盘Raid + Raid string `gorm:"column:raid;type:varchar(20);not null" json:"raid"` + // 实际城市ID + CityID string `gorm:"column:city_id;type:varchar(64);not null" json:"city_id"` + // 实际城市名称 + City string `gorm:"column:city;type:varchar(128);not null" json:"city"` + // 园区, 例如光明 cc_device_szone + SubZone string `gorm:"column:sub_zone;type:varchar(32);not null" json:"sub_zone"` + // 园区ID cc_device_szone_id + SubZoneID string `gorm:"column:sub_zone_id;type:varchar(64);not null" json:"sub_zone_id"` + // 存放机架ID,判断是否是同机架 + RackID string `gorm:"column:rack_id;type:varchar(64);not null" json:"rack_id"` + // 网络设备ID, 判断是同交换机 + NetDeviceID string `gorm:"column:net_device_id;type:varchar(128)" json:"net_device_id"` + // 标签 + Labels string `gorm:"column:labels;type:json" json:"labels"` + IsInit int `gorm:"column:is_init;type:int(11);comment:'是否初始化过'" json:"-"` + IsIdle int `gorm:"column:is_idle;type:int(11);comment:'是否空闲检查过'" json:"-"` // Status: Unused: 未使用 Used: 已经售卖被使用: Preselected:预占用 Status string `gorm:"column:status;type:varchar(20);not null" json:"status"` BkAgentId string `gorm:"index:idx_bk_agent_id;column:bk_agent_id;type:varchar(64);not null" json:"bk_agent_id"` @@ -61,9 +68,9 @@ type TbRpDetailArchive struct { AgentStatusCode int `gorm:"column:gse_agent_status_code;type:int(11);not null" json:"gse_agent_status_code"` // agent status 最后一次更新时间 AgentStatusUpdateTime time.Time `gorm:"column:agent_status_update_time;type:timestamp;default:1970-01-01 08:00:01" json:"agent_status_update_time"` - ConsumeTime time.Time `gorm:"column:consume_time;type:timestamp;default:1970-01-01 08:00:01" json:"consume_time"` // 消费时间 - UpdateTime time.Time `gorm:"column:update_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"update_time"` // 最后修改时间 - CreateTime time.Time `gorm:"column:create_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"create_time"` // 创建时间 + ConsumeTime time.Time `gorm:"column:consume_time;type:timestamp;default:1970-01-01 08:00:01" json:"consume_time"` + UpdateTime time.Time `gorm:"column:update_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"update_time"` + CreateTime time.Time `gorm:"column:create_time;type:timestamp;default:CURRENT_TIMESTAMP()" json:"create_time"` } // initarchive 启动的时候归档未清理的资源 @@ -95,7 +102,7 @@ func ArchiverResouce(ids []int) (err error) { defer func() { if err != nil { if tx.Rollback().Error != nil { - logger.Error(fmt.Sprintf("archive resource exception %s,rollback failed!!", err)) + logger.Error("archive resource exception %s,rollback failed!!", err) } } }() diff --git a/dbm-services/common/db-resource/internal/model/TbRpOperatorInfo.go b/dbm-services/common/db-resource/internal/model/TbRpOperatorInfo.go index 45c09f6a06..7c8d6b966d 100644 --- a/dbm-services/common/db-resource/internal/model/TbRpOperatorInfo.go +++ b/dbm-services/common/db-resource/internal/model/TbRpOperatorInfo.go @@ -22,6 +22,13 @@ const ( Imported = "imported" ) +const ( + // StatusSuccess operator success + StatusSuccess = "success" + // StatusFailed failed + StatusFailed = "failed" +) + // TbRpOperationInfo 资源池操作记录表 // nolint type TbRpOperationInfo struct { diff --git a/dbm-services/common/db-resource/internal/svr/apply/apply.go b/dbm-services/common/db-resource/internal/svr/apply/apply.go index 58c2a45910..453508c111 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/apply.go +++ b/dbm-services/common/db-resource/internal/svr/apply/apply.go @@ -36,6 +36,7 @@ type SearchContext struct { RsType string IntetionBkBizId int IdcCitys []string + SpecialHostIds []int } // CycleApply 循环匹配 @@ -55,7 +56,9 @@ func CycleApply(param RequestInputParam) (pickers []*PickerObject, err error) { var idcCitys []string if config.AppConfig.RunMode == "dev" { idcCitys = []string{} - } else if cmutil.ElementNotInArry(v.Affinity, []string{CROSS_RACK, NONE}) || lo.IsNotEmpty(&v.LocationSpec.City) { + } else if cmutil.ElementNotInArry(v.Affinity, []string{CROSS_RACK, NONE}) || + lo.IsNotEmpty(v.LocationSpec.City) || + len(v.Hosts) > 0 { idcCitys, err = dbmapi.GetIdcCityByLogicCity(v.LocationSpec.City) if err != nil { logger.Error("request real citys by logic city %s from bkdbm api failed:%v", v.LocationSpec.City, err) @@ -68,6 +71,7 @@ func CycleApply(param RequestInputParam) (pickers []*PickerObject, err error) { RsType: param.ResourceType, ObjectDetail: &v, IdcCitys: idcCitys, + SpecialHostIds: v.Hosts.GetBkHostIds(), } if err = s.PickCheck(); err != nil { return pickers, err @@ -90,7 +94,7 @@ func CycleApply(param RequestInputParam) (pickers []*PickerObject, err error) { return pickers, nil } -// RollBackAllInstanceUnused 将 Instance Status Selling ==> Not Selled : 2 --> 0 +// RollBackAllInstanceUnused reserve all instance unused func RollBackAllInstanceUnused(ms []*PickerObject) { for _, m := range ms { if err := m.RollbackUnusedInstance(); err != nil { @@ -100,53 +104,18 @@ func RollBackAllInstanceUnused(ms []*PickerObject) { } func (o *SearchContext) pickBase(db *gorm.DB) { - db.Where("gse_agent_status_code = ? ", bk.GSE_AGENT_OK) - if o.BkCloudId <= 0 { - db.Where(" bk_cloud_id = ? and status = ? ", o.ObjectDetail.BkCloudId, model.Unused) - } else { - db.Where(" bk_cloud_id = ? and status = ? ", o.BkCloudId, model.Unused) - } - // os type - // Windows - // Liunx - osType := o.ObjectDetail.OsType - if cmutil.IsEmpty(o.ObjectDetail.OsType) { - osType = "Linux" - } - db.Where("os_type = ? ", osType) - - // match os name like Windows Server 2012 - if len(o.ObjectDetail.OsNames) > 0 { - conditions := []clause.Expression{} - for _, osname := range o.ObjectDetail.OsNames { - conditions = append(conditions, clause.Like{ - Column: "os_name", - Value: "%" + strings.TrimSpace(strings.ToLower(osname)) + "%", - }) - } - if len(conditions) == 1 { - db.Clauses(clause.AndConditions{Exprs: conditions}) - } else { - // 有多个条件，使用or，才会被用（）包括起来所有的or条件 - db.Clauses(clause.OrConditions{Exprs: conditions}) - } + // 如果指定了特殊资源，就只查询这些资源 + if len(o.SpecialHostIds) > 0 { + db.Where("bk_host_id in (?) and status = ? ", o.SpecialHostIds, model.Unused) + return } + db.Where(" bk_cloud_id = ? and status = ? and gse_agent_status_code = ? ", o.BkCloudId, model.Unused, bk.GSE_AGENT_OK) - // 如果没有指定资源类型，表示只能选择无资源类型标签的资源 - // 没有资源类型标签的资源可以被所有其他类型使用 - if lo.IsEmpty(o.RsType) { - db.Where("rs_type == 'PUBLIC' ") - } else { - db.Where("rs_type in (?)", []string{"PUBLIC", o.RsType}) - } - // 如果没有指定专属业务，就表示只能选用公共的资源 - // 不能匹配打了业务标签的资源 - if o.IntetionBkBizId <= 0 { - db.Where("dedicated_biz == 0") - } else { - db.Where("dedicated_biz in (?)", []int{0, o.IntetionBkBizId}) - } - o.MatchLables(db) + o.MatchIntetionBkBiz(db) + o.MatchRsType(db) + o.MatchOsType(db) + o.MatchOsName(db) + o.MatchLabels(db) o.MatchLocationSpec(db) o.MatchStorage(db) o.MatchSpec(db) @@ -162,8 +131,10 @@ func (o *SearchContext) pickBase(db *gorm.DB) { // PickCheck precheck func (o *SearchContext) PickCheck() (err error) { var count int64 - logger.Info("前置检查轮资源匹配") + if len(o.SpecialHostIds) > 0 { + return o.PickCheckSpecialBkhostIds() + } db := model.DB.Self.Table(model.TbRpDetailName()).Select("count(*)") o.pickBase(db) if err := db.Scan(&count).Error; err != nil { @@ -172,12 +143,61 @@ func (o *SearchContext) PickCheck() (err error) { } if int(count) < o.Count { - return errno.ErrResourceinsufficient.AddErr(fmt.Errorf("申请需求:%s\n\r资源池符合条件的资源总数:%d 小于申请的数量", o.GetMessage(), - count)) + return fmt.Errorf("申请需求:%s\n\r资源池符合条件的资源总数:%d 小于申请的数量", o.GetMessage(), count) + } + return nil +} + +// PickCheckSpecialBkhostIds 根据bkhostids取资源 +func (o *SearchContext) PickCheckSpecialBkhostIds() (err error) { + var rs []int + err = model.DB.Self.Table(model.TbRpDetailName()).Select("bk_host_id").Where( + "bk_host_id in (?) and status = ? and bk_cloud_id = ? ", + o.SpecialHostIds, model.Unused, o.BkCloudId).Scan(&rs).Error + if err != nil { + logger.Error("query pre check count failed %s", err.Error()) + return errno.ErrDBQuery.AddErr(err) + } + if len(rs) != len(o.SpecialHostIds) { + emptyIps := []string{} + hostIpMap := lo.SliceToMap(o.Hosts, func(item Host) (int, string) { return item.BkHostId, item.IP }) + for hostid, ip := range hostIpMap { + if !lo.Contains(rs, hostid) { + emptyIps = append(emptyIps, ip) + } + } + return fmt.Errorf("指定ip申请资源,部分资源不存在:%v", emptyIps) } return nil } +// filterEmptyMountPointStorage 过滤没有挂载点的磁盘匹配需求 +func (o *SearchContext) filterEmptyMountPointStorage(items []model.TbRpDetail, + diskSpecs []meta.DiskSpec) (ts []model.TbRpDetail, err error) { + for _, ins := range items { + if err = ins.UnmarshalDiskInfo(); err != nil { + logger.Error("%s umarshal disk failed %s", ins.IP, err.Error()) + return nil, err + } + logger.Info("%v", ins.Storages) + noUseStorages := make(map[string]bk.DiskDetail) + smp := meta.GetDiskSpecMountPoints(o.StorageSpecs) + for mp, v := range ins.Storages { + if cmutil.ElementNotInArry(mp, smp) { + noUseStorages[mp] = v + } + } + logger.Info("nouse: %v", noUseStorages) + if matchNoMountPointStorage(diskSpecs, noUseStorages) { + ts = append(ts, ins) + } + } + if len(ts) == 0 { + return nil, errno.ErrResourceinsufficient.Add(fmt.Sprintf("匹配磁盘%s,的资源为 0", o.GetDiskMatchInfo())) + } + return ts, nil +} + // PickInstance match resource func (o *SearchContext) PickInstance() (picker *PickerObject, err error) { picker = NewPicker(o.Count, o.GroupMark) @@ -190,35 +210,20 @@ func (o *SearchContext) PickInstance() (picker *PickerObject, err error) { } // 过滤没有挂载点的磁盘匹配需求 logger.Info("storage spec %v", o.StorageSpecs) + diskSpecs := meta.GetEmptyDiskSpec(o.StorageSpecs) - if len(diskSpecs) > 0 { - ts := []model.TbRpDetail{} - for _, ins := range items { - if err = ins.UnmarshalDiskInfo(); err != nil { - logger.Error("%s umarshal disk failed %s", ins.IP, err.Error()) - return picker, err - } - logger.Info("%v", ins.Storages) - noUseStorages := make(map[string]bk.DiskDetail) - smp := meta.GetDiskSpecMountPoints(o.StorageSpecs) - for mp, v := range ins.Storages { - if cmutil.ElementNotInArry(mp, smp) { - noUseStorages[mp] = v - } - } - logger.Info("nouse: %v", noUseStorages) - if matchNoMountPointStorage(diskSpecs, noUseStorages) { - ts = append(ts, ins) - } - } - if len(ts) == 0 { - return picker, errno.ErrResourceinsufficient.Add(fmt.Sprintf("匹配磁盘%s,的资源为 0", o.GetDiskMatchInfo())) + if len(diskSpecs) > 0 && len(o.SpecialHostIds) == 0 { + items, err = o.filterEmptyMountPointStorage(items, diskSpecs) + if err != nil { + logger.Error("filter empty mount point storage failed %s", err.Error()) + return picker, err } - items = ts } + if err = o.PickInstanceBase(picker, items); err != nil { return nil, err } + if picker.PickerDone() { return picker, nil } @@ -227,52 +232,16 @@ func (o *SearchContext) PickInstance() (picker *PickerObject, err error) { o.GetMessage())) } -// MatchLables match lables -func (o *SearchContext) MatchLables(db *gorm.DB) { - if len(o.Labels) > 0 { - for key, v := range o.Labels { - db.Where(" ( json_contains(label,json_object(?,?) )", key, v) - } - return - } - db.Where(" JSON_TYPE(label) = 'NULL' OR JSON_LENGTH(label) <= 1 ") -} - -func matchNoMountPointStorage(spec []meta.DiskSpec, sinc map[string]bk.DiskDetail) bool { - mcount := 0 - for _, s := range spec { - for mp, d := range sinc { - if diskDetailMatch(d, s) { - delete(sinc, mp) - mcount++ - break - } - } - } - return mcount == len(spec) -} - -func diskDetailMatch(d bk.DiskDetail, s meta.DiskSpec) bool { - logger.Info("spec %v", s) - logger.Info("detail %v", d) - if d.DiskType != s.DiskType && cmutil.IsNotEmpty(s.DiskType) { - logger.Info("disk type not match") - return false - } - if d.Size > s.MaxSize && s.MaxSize > 0 { - logger.Info("max size not match") - return false - } - if d.Size < s.MinSize { - logger.Info("min size not match") - return false - } - return true -} - -// PickInstanceBase TODO +// PickInstanceBase pick instance base func (o *SearchContext) PickInstanceBase(picker *PickerObject, items []model.TbRpDetail) (err error) { logger.Info("the anti-affinity is %s", o.Affinity) + if len(o.SpecialHostIds) > 0 { + for _, v := range items { + picker.SatisfiedHostIds = append(picker.SatisfiedHostIds, v.BkHostID) + } + picker.Count = len(o.SpecialHostIds) + return nil + } switch o.Affinity { case NONE: picker.PriorityElements, err = o.AnalysisResourcePriority(items, true) @@ -296,7 +265,69 @@ func (o *SearchContext) PickInstanceBase(picker *PickerObject, items []model.TbR return } -// MatchLocationSpec 匹配location参数 +// MatchIntetionBkBiz match intetion biz +func (o *SearchContext) MatchIntetionBkBiz(db *gorm.DB) { + // 如果没有指定专属业务，就表示只能选用公共的资源 + // 不能匹配打了业务标签的资源 + if o.IntetionBkBizId <= 0 { + db.Where("dedicated_biz == 0") + } else { + db.Where("dedicated_biz in (?)", []int{0, o.IntetionBkBizId}) + } +} + +// MatchRsType pick rs type +func (o *SearchContext) MatchRsType(db *gorm.DB) { + // 如果没有指定资源类型，表示只能选择无资源类型标签的资源 + // 没有资源类型标签的资源可以被所有其他类型使用 + if lo.IsEmpty(o.RsType) { + db.Where("rs_type == 'PUBLIC' ") + } else { + db.Where("rs_type in (?)", []string{"PUBLIC", o.RsType}) + } +} + +// MatchOsType match os type +func (o *SearchContext) MatchOsType(db *gorm.DB) { + // os type: Windows, Liunx + osType := o.ObjectDetail.OsType + if cmutil.IsEmpty(o.ObjectDetail.OsType) { + osType = model.LiunxOs + } + db.Where("os_type = ? ", osType) +} + +// MatchOsName TODO +func (o *SearchContext) MatchOsName(db *gorm.DB) { + // match os name like Windows Server 2012 + if len(o.ObjectDetail.OsNames) > 0 { + conditions := []clause.Expression{} + for _, osname := range o.ObjectDetail.OsNames { + conditions = append(conditions, clause.Like{ + Column: "os_name", + Value: "%" + strings.TrimSpace(strings.ToLower(osname)) + "%", + }) + } + if len(conditions) == 1 { + db.Clauses(clause.AndConditions{Exprs: conditions}) + } else { + // 有多个条件，使用or，才会被用（）包括起来所有的or条件 + db.Clauses(clause.OrConditions{Exprs: conditions}) + } + } +} + +// MatchLabels match labels +func (o *SearchContext) MatchLabels(db *gorm.DB) { + if len(o.Labels) > 0 { + db.Where(model.JSONQuery("labels").JointOrContains(o.Labels)) + } else { + // 如果请求没有标签, 只能匹配没有标签的资源 + db.Where(" JSON_TYPE(labels) = 'NULL' OR JSON_LENGTH(labels) < 1 ") + } +} + +// MatchLocationSpec match location parameter func (o *SearchContext) MatchLocationSpec(db *gorm.DB) { if o.LocationSpec.IsEmpty() { return @@ -317,7 +348,7 @@ func (o *SearchContext) MatchLocationSpec(db *gorm.DB) { } } -// MatchStorage 匹配存储参数 +// MatchStorage match storage parameters func (o *SearchContext) MatchStorage(db *gorm.DB) { if len(o.StorageSpecs) == 0 { return @@ -343,11 +374,7 @@ func (o *SearchContext) MatchStorage(db *gorm.DB) { } } -func isWindowsPath(path string) bool { - return strings.Contains(path, "\\") -} - -// MatchSpec TODO +// MatchSpec match spec func (o *SearchContext) MatchSpec(db *gorm.DB) { if len(o.DeviceClass) > 0 { switch { @@ -376,3 +403,37 @@ func (o *SearchContext) UseNetDeviceIsNotEmpty(db *gorm.DB) { func (o *SearchContext) RackIdIsNotEmpty(db *gorm.DB) { db.Where("rack_id is not null and rack_id != ''") } + +func isWindowsPath(path string) bool { + return strings.Contains(path, "\\") +} + +func matchNoMountPointStorage(spec []meta.DiskSpec, sinc map[string]bk.DiskDetail) bool { + mcount := 0 + for _, s := range spec { + for mp, d := range sinc { + if diskDetailMatch(d, s) { + delete(sinc, mp) + mcount++ + break + } + } + } + return mcount == len(spec) +} + +func diskDetailMatch(d bk.DiskDetail, s meta.DiskSpec) bool { + if d.DiskType != s.DiskType && cmutil.IsNotEmpty(s.DiskType) { + logger.Info("disk type not match") + return false + } + if d.Size > s.MaxSize && s.MaxSize > 0 { + logger.Info("max size not match") + return false + } + if d.Size < s.MinSize { + logger.Info("min size not match") + return false + } + return true +} diff --git a/dbm-services/common/db-resource/internal/svr/apply/api.go b/dbm-services/common/db-resource/internal/svr/apply/apply_base.go similarity index 82% rename from dbm-services/common/db-resource/internal/svr/apply/api.go rename to dbm-services/common/db-resource/internal/svr/apply/apply_base.go index 88156d420c..60fe75402c 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/api.go +++ b/dbm-services/common/db-resource/internal/svr/apply/apply_base.go @@ -78,6 +78,23 @@ type RequestInputParam struct { ActionInfo } +// BuildMessage build apply message +func (param RequestInputParam) BuildMessage() (msg string) { + var count int + groupMap := make(map[string]int) + groupCountMap := make(map[string]int) + for _, d := range param.Details { + groupMap[d.Affinity]++ + groupCountMap[d.Affinity] += d.Count + count += d.Count + } + msg = fmt.Sprintf("此次申请分%d组申请%d个机器\n", len(param.Details), count) + for affinity, count := range groupMap { + msg += fmt.Sprintf("按照亲和性%s申请的资源分组%d,总共包含机器数量%d\n", affinity, count, groupCountMap[affinity]) + } + return msg +} + // SortDetails 优先去匹配有明确需求的参数 func (param RequestInputParam) SortDetails() ([]ObjectDetail, error) { if len(param.Details) == 1 { @@ -172,36 +189,38 @@ func (param RequestInputParam) LockKey() string { } const ( - // SAME_SUBZONE_CROSS_SWTICH TODO + // SAME_SUBZONE_CROSS_SWTICH 同城同园区跨交换机跨机架 SAME_SUBZONE_CROSS_SWTICH = "SAME_SUBZONE_CROSS_SWTICH" - // SAME_SUBZONE TODO + // SAME_SUBZONE 同城同园区 SAME_SUBZONE = "SAME_SUBZONE" - // CROS_SUBZONE TODO + // CROS_SUBZONE 同城跨园区 CROS_SUBZONE = "CROS_SUBZONE" - // MAX_EACH_ZONE_EQUAL TODO + // MAX_EACH_ZONE_EQUAL 尽量每个zone分配数量相等 MAX_EACH_ZONE_EQUAL = "MAX_EACH_ZONE_EQUAL" // CROSS_RACK 跨机架 CROSS_RACK = "CROSS_RACK" - // NONE TODO + // NONE 无亲和性 NONE = "NONE" ) -// ObjectDetail TODO +// ObjectDetail 资源申请对象详情 +// 反亲和性 目前只有一种选项,当campus是空的时候，则此值生效 +// SAME_SUBZONE_CROSS_SWTICH: 同城同subzone跨交换机跨机架、 +// SAME_SUBZONE: 同城同subzone +// CROS_SUBZONE：同城跨subzone +// NONE: 无需亲和性处理 type ObjectDetail struct { - BkCloudId int `json:"bk_cloud_id"` - GroupMark string `json:"group_mark" binding:"required" ` // 资源组标记 - Labels map[string]string `json:"labels"` // 标签 + BkCloudId int `json:"bk_cloud_id"` + Hosts Hosts `json:"hosts"` // 主机id + GroupMark string `json:"group_mark" binding:"required" ` // 资源组标记 + Labels []string `json:"labels"` // 标签 // 通过机型规格 或者 资源规格描述来匹配资源 // 这两个条件是 || 关系 DeviceClass []string `json:"device_class"` // 机器类型 "IT5.8XLARGE128" "SA3.2XLARGE32" Spec meta.Spec `json:"spec"` // 规格描述 StorageSpecs []meta.DiskSpec `json:"storage_spec"` LocationSpec meta.LocationSpec `json:"location_spec"` // 地域区间 - // 反亲和性 目前只有一种选项,当campus是空的时候，则此值生效 - // SAME_SUBZONE_CROSS_SWTICH: 同城同subzone跨交换机跨机架、 - // SAME_SUBZONE: 同城同subzone - // CROS_SUBZONE：同城跨subzone - // NONE: 无需亲和性处理 + Affinity string `json:"affinity"` // Windows,Linux OsType string `json:"os_type"` @@ -209,6 +228,24 @@ type ObjectDetail struct { Count int `json:"count" binding:"required,min=1"` // 申请数量 } +// Hosts bk hosts +type Hosts []Host + +// GetBkHostIds get bk host ids +func (a Hosts) GetBkHostIds() []int { + var bkHostIds []int + for _, v := range a { + bkHostIds = append(bkHostIds, v.BkHostId) + } + return bkHostIds +} + +// Host bk host +type Host struct { + BkHostId int `json:"bk_host_id"` + IP string `json:"ip"` +} + // GetDiskMatchInfo get request disk message func (a *ObjectDetail) GetDiskMatchInfo() (message string) { if len(a.StorageSpecs) > 0 { diff --git a/dbm-services/common/db-resource/internal/svr/apply/core.go b/dbm-services/common/db-resource/internal/svr/apply/apply_core.go similarity index 84% rename from dbm-services/common/db-resource/internal/svr/apply/core.go rename to dbm-services/common/db-resource/internal/svr/apply/apply_core.go index 35de15225a..a641500508 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/core.go +++ b/dbm-services/common/db-resource/internal/svr/apply/apply_core.go @@ -34,15 +34,14 @@ type subzone = string // PickerObject TODO type PickerObject struct { - Item string - Count int - PickDistrbute map[string]int - ExistSubZone []subzone // 已存在的园区 - SatisfiedHostIds []int - SelectedResources []*model.TbRpDetail - + Item string + Count int + PickDistrbute map[string]int + // 已存在的园区 + ExistSubZone []subzone + SatisfiedHostIds []int + // SelectedResources []*model.TbRpDetail // 待选择实例 - // PickeElements map[subzone][]InstanceObject // 具备优先级的待选实例列表 PriorityElements map[subzone]*PriorityQueue @@ -187,13 +186,13 @@ func (c *PickerObject) RollbackUnusedInstance() error { return model.UpdateTbRpDetailStatusAtSelling(c.SatisfiedHostIds, model.Unused) } -// CampusNice TODO +// CampusNice build campus type CampusNice struct { Campus string `json:"campus"` Count int `json:"count"` } -// CampusWrapper TODO +// CampusWrapper 园区排序 type CampusWrapper struct { Campus []CampusNice by func(p, q *CampusNice) bool @@ -244,3 +243,45 @@ func (c *PickerObject) InterSectForLinkNetDevice(linkDeviceIds []string) int { } return baseSet.Intersect(myset).Cardinality() } + +// InstanceObject instance object +type InstanceObject struct { + BkHostId int + Equipment string + LinkNetdeviceId []string + Nice int64 + InsDetail *model.TbRpDetail +} + +// GetLinkNetDeviceIdsInterface getLinkNetDeviceIdsInterface +func (c *InstanceObject) GetLinkNetDeviceIdsInterface() []interface{} { + var k []interface{} + for _, v := range c.LinkNetdeviceId { + k = append(k, v) + } + return k +} + +// Wrapper Wrapper +type Wrapper struct { + Instances []InstanceObject + by func(p, q *InstanceObject) bool +} + +// SortBy sortby +type SortBy func(p, q *InstanceObject) bool + +// Len 用于排序 +func (pw Wrapper) Len() int { // 重写 Len() 方法 + return len(pw.Instances) +} + +// Swap 用于排序 +func (pw Wrapper) Swap(i, j int) { // 重写 Swap() 方法 + pw.Instances[i], pw.Instances[j] = pw.Instances[j], pw.Instances[i] +} + +// Less 用于排序 +func (pw Wrapper) Less(i, j int) bool { // 重写 Less() 方法 + return pw.by(&pw.Instances[i], &pw.Instances[j]) +} diff --git a/dbm-services/common/db-resource/internal/svr/apply/match_resource.go b/dbm-services/common/db-resource/internal/svr/apply/apply_match.go similarity index 99% rename from dbm-services/common/db-resource/internal/svr/apply/match_resource.go rename to dbm-services/common/db-resource/internal/svr/apply/apply_match.go index d486fffadf..043b72086b 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/match_resource.go +++ b/dbm-services/common/db-resource/internal/svr/apply/apply_match.go @@ -172,7 +172,6 @@ func (c *PickerObject) pickerOneByPriority(key string, cross_switch bool) bool { } c.ExistEquipmentIds = append(c.ExistEquipmentIds, v.Equipment) c.SatisfiedHostIds = append(c.SatisfiedHostIds, v.BkHostId) - c.SelectedResources = append(c.SelectedResources, v.InsDetail) c.ExistLinkNetdeviceIds = append(c.ExistLinkNetdeviceIds, v.LinkNetdeviceId...) c.PickDistrbute[key]++ return true diff --git a/dbm-services/common/db-resource/internal/svr/apply/priority_queue.go b/dbm-services/common/db-resource/internal/svr/apply/apply_priority_queue.go similarity index 100% rename from dbm-services/common/db-resource/internal/svr/apply/priority_queue.go rename to dbm-services/common/db-resource/internal/svr/apply/apply_priority_queue.go diff --git a/dbm-services/common/db-resource/internal/svr/apply/instance.go b/dbm-services/common/db-resource/internal/svr/apply/instance.go deleted file mode 100644 index 425fb0f795..0000000000 --- a/dbm-services/common/db-resource/internal/svr/apply/instance.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. - * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. - * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at https://opensource.org/licenses/MIT - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package apply - -import "dbm-services/common/db-resource/internal/model" - -// InstanceObject instance object -type InstanceObject struct { - BkHostId int - Equipment string - LinkNetdeviceId []string - Nice int64 - InsDetail *model.TbRpDetail -} - -// GetLinkNetDeviceIdsInterface getLinkNetDeviceIdsInterface -func (c *InstanceObject) GetLinkNetDeviceIdsInterface() []interface{} { - var k []interface{} - for _, v := range c.LinkNetdeviceId { - k = append(k, v) - } - return k -} - -// Wrapper Wrapper -type Wrapper struct { - Instances []InstanceObject - by func(p, q *InstanceObject) bool -} - -// SortBy sortby -type SortBy func(p, q *InstanceObject) bool - -// Len 用于排序 -func (pw Wrapper) Len() int { // 重写 Len() 方法 - return len(pw.Instances) -} - -// Swap 用于排序 -func (pw Wrapper) Swap(i, j int) { // 重写 Swap() 方法 - pw.Instances[i], pw.Instances[j] = pw.Instances[j], pw.Instances[i] -} - -// Less 用于排序 -func (pw Wrapper) Less(i, j int) bool { // 重写 Less() 方法 - return pw.by(&pw.Instances[i], &pw.Instances[j]) -} diff --git a/dbm-services/common/db-resource/internal/svr/bk/cc_test.go b/dbm-services/common/db-resource/internal/svr/bk/cc_test.go index 6986359c27..175f5042eb 100644 --- a/dbm-services/common/db-resource/internal/svr/bk/cc_test.go +++ b/dbm-services/common/db-resource/internal/svr/bk/cc_test.go @@ -66,7 +66,7 @@ func TestReserverCC(t *testing.T) { RsType: "MySQL", Hosts: hosts, } - importResp, err := manage.Doimport(param) + importResp, err := manage.Doimport(param, "") if err != nil { t.Fatal(err) } diff --git a/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_spec.go b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_spec.go index e0ccc71d4c..076cd4edbb 100644 --- a/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_spec.go +++ b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_spec.go @@ -96,6 +96,7 @@ func (c *DbmClient) GetDbmSpec(queryParam map[string]string) (specData []DbmSpec } query.Set("limit", "-1") u.RawQuery = query.Encode() + logger.Info("query url:%s", u.String()) request, err := http.NewRequest("GET", u.String(), nil) if err != nil { return nil, err @@ -111,6 +112,7 @@ func (c *DbmClient) GetDbmSpec(queryParam map[string]string) (specData []DbmSpec logger.Error("read respone body failed %s", err.Error()) return } + logger.Info("GetDbmSpec respone:%s", string(body)) var rpdata DbmBaseResp if err = json.Unmarshal(body, &rpdata); err != nil { logger.Error("unmarshal respone body failed %s", err.Error()) @@ -119,6 +121,7 @@ func (c *DbmClient) GetDbmSpec(queryParam map[string]string) (specData []DbmSpec if rpdata.Code != 0 { return nil, fmt.Errorf("respone code:%d,message:%s", rpdata.Code, rpdata.Message) } + logger.Debug("GetDbmSpec respone:%s", string(rpdata.Data)) var specRespData DbmSpecBaseResp if err = json.Unmarshal(rpdata.Data, &specRespData); err != nil { logger.Error("unmarshal DbmBaseResp body failed %s", err.Error()) diff --git a/dbm-services/common/db-resource/internal/svr/task/inspection_task.go b/dbm-services/common/db-resource/internal/svr/task/inspection_task.go index 8e538d5008..c2b7f9f718 100644 --- a/dbm-services/common/db-resource/internal/svr/task/inspection_task.go +++ b/dbm-services/common/db-resource/internal/svr/task/inspection_task.go @@ -65,10 +65,8 @@ func InspectCheckResource() (err error) { // return err } for _, m := range resp.Data { - if m.BKModuleId == allowCCMouduleInfo.CC_IDLE_MODULE_ID { - continue - } - if m.BKSetId == allowCCMouduleInfo.CC_MANAGE_TOPO.SetId { + if m.BKModuleId == allowCCMouduleInfo.CC_IDLE_MODULE_ID || + m.BKSetId == allowCCMouduleInfo.CC_MANAGE_TOPO.SetId { continue } err = model.DB.Self.Table(model.TbRpDetailName()).Where("bk_biz_id = ? and bk_host_id = ? and status = ? ",