diff --git a/dbm-services/common/db-resource/internal/controller/manage/import.go b/dbm-services/common/db-resource/internal/controller/manage/import.go index 6be8d892f6..3d0309d15a 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/import.go +++ b/dbm-services/common/db-resource/internal/controller/manage/import.go @@ -35,8 +35,8 @@ import ( // ImportMachParam 资源导入请求参数 type ImportMachParam struct { // ForBizs 业务标签,表示这个资源将来给ForBizs这个业务使用 - ForBizs []int `json:"for_bizs"` - RsTypes []string `json:"resource_types"` + ForBiz int `json:"for_biz"` + RsType string `json:"resource_type"` BkBizId int `json:"bk_biz_id" binding:"number"` Hosts []HostBase `json:"hosts" binding:"gt=0,dive,required"` Labels map[string]string `json:"labels"` @@ -120,29 +120,13 @@ type ImportHostResp struct { NotFoundInCCHosts []string `json:"not_found_in_cc_hosts"` } -func (p ImportMachParam) transParamToBytes() (lableJson, bizJson, rstypes json.RawMessage, err error) { +func (p ImportMachParam) transParamToBytes() (lableJson json.RawMessage, err error) { // lableJson = []byte("{}") lableJson, err = json.Marshal(cmutil.CleanStrMap(p.Labels)) if err != nil { logger.Error(fmt.Sprintf("ConverLableToJsonStr Failed,Error:%s", err.Error())) return } - bizJson = []byte("[]") - if len(p.ForBizs) > 0 { - bizJson, err = json.Marshal(cmutil.IntSliceToStrSlice(p.ForBizs)) - if err != nil { - logger.Error(fmt.Sprintf("conver biz json Failed,Error:%s", err.Error())) - return - } - } - rstypes = []byte("[]") - if len(p.RsTypes) > 0 { - rstypes, err = json.Marshal(cmutil.StringsRemoveEmpty(p.RsTypes)) - if err != nil { - logger.Error(fmt.Sprintf("conver resource types Failed,Error:%s", err.Error())) - return - } - } return } @@ -183,7 +167,7 @@ func Doimport(param ImportMachParam) (resp *ImportHostResp, err error) { } resp.SearchDiskErrInfo = diskResp.IpFailedLogMap resp.NotFoundInCCHosts = notFoundHosts - lableJson, bizJson, rstypes, err := param.transParamToBytes() + lableJson, err := param.transParamToBytes() if err != nil { return resp, err } @@ -204,7 +188,7 @@ func Doimport(param ImportMachParam) (resp *ImportHostResp, err error) { } for _, h := range ccHostsInfo { delete(hostsMap, h.InnerIP) - el := transHostInfoToDbModule(h, h.BkCloudId, param.BkBizId, rstypes, bizJson, lableJson) + el := param.transHostInfoToDbModule(h, h.BkCloudId, lableJson) el.SetMore(h.InnerIP, diskResp.IpLogContentMap) // gse agent 1.0的 agent 是用 cloudid:ip gseAgentId := h.BkAgentId @@ -265,16 +249,16 @@ func getCvmMachList(hosts []*cc.Host) []string { } // transHostInfoToDbModule 获取的到的主机信息赋值给db model -func transHostInfoToDbModule(h *cc.Host, bkCloudId, bkBizId int, rstp, biz, label []byte) model.TbRpDetail { +func (p ImportMachParam) transHostInfoToDbModule(h *cc.Host, bkCloudId int, label []byte) model.TbRpDetail { osType := h.BkOsType if cmutil.IsEmpty(osType) { osType = bk.OsLinux } return model.TbRpDetail{ - RsTypes: rstp, - DedicatedBizs: biz, + DedicatedBiz: p.ForBiz, + RsType: p.RsType, BkCloudID: bkCloudId, - BkBizId: bkBizId, + BkBizId: p.BkBizId, AssetID: h.AssetID, BkHostID: h.BKHostId, IP: h.InnerIP, diff --git a/dbm-services/common/db-resource/internal/controller/manage/list.go b/dbm-services/common/db-resource/internal/controller/manage/list.go index b125057c1f..9a152ae362 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/list.go +++ b/dbm-services/common/db-resource/internal/controller/manage/list.go @@ -16,11 +16,12 @@ import ( "strings" rf "github.com/gin-gonic/gin" + "github.com/samber/lo" "gorm.io/gorm" "dbm-services/common/db-resource/internal/model" - "dbm-services/common/db-resource/internal/svr/apply" "dbm-services/common/db-resource/internal/svr/bk" + "dbm-services/common/db-resource/internal/svr/dbmapi" "dbm-services/common/db-resource/internal/svr/meta" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/errno" @@ -30,21 +31,21 @@ import ( // MachineResourceGetterInputParam TODO type MachineResourceGetterInputParam struct { // 专用业务Ids - ForBizs []int `json:"for_bizs"` - City []string `json:"city"` - SubZones []string `json:"subzones"` - DeviceClass []string `json:"device_class"` - Labels map[string]string `json:"labels"` - Hosts []string `json:"hosts"` - BkCloudIds []int `json:"bk_cloud_ids"` - RsTypes []string `json:"resource_types"` - MountPoint string `json:"mount_point"` - Cpu apply.MeasureRange `json:"cpu"` - Mem apply.MeasureRange `json:"mem"` - Disk apply.MeasureRange `json:"disk"` - DiskType string `json:"disk_type"` - OsType string `json:"os_type"` - StorageSpecs []apply.DiskSpec `json:"storage_spec"` + ForBiz int `json:"for_biz"` + City []string `json:"city"` + SubZoneIds []string `json:"subzone_ids"` + DeviceClass []string `json:"device_class"` + Labels map[string]string `json:"labels"` + Hosts []string `json:"hosts"` + BkCloudIds []int `json:"bk_cloud_ids"` + RsType string `json:"resource_type"` + MountPoint string `json:"mount_point"` + Cpu meta.MeasureRange `json:"cpu"` + Mem meta.MeasureRange `json:"mem"` + Disk meta.MeasureRange `json:"disk"` + DiskType string `json:"disk_type"` + OsType string `json:"os_type"` + StorageSpecs []meta.DiskSpec `json:"storage_spec"` // true,false,"" GseAgentAlive string `json:"gse_agent_alive"` Limit int `json:"limit"` @@ -129,7 +130,7 @@ func (c *MachineResourceGetterInputParam) matchStorageSpecs(db *gorm.DB) { func (c *MachineResourceGetterInputParam) getRealCitys() (realCistys []string, err error) { for _, logicCity := range c.City { - rcitys, err := meta.GetIdcCityByLogicCity(logicCity) + rcitys, err := dbmapi.GetIdcCityByLogicCity(logicCity) if err != nil { logger.Error("from %s get real citys failed %s", logicCity, err.Error()) return nil, err @@ -172,13 +173,8 @@ func (c *MachineResourceGetterInputParam) queryBs(db *gorm.DB) (err error) { if len(c.BkCloudIds) > 0 { db.Where("bk_cloud_id in (?) ", c.BkCloudIds) } - if len(c.RsTypes) > 0 { - // 如果参数["all"],表示选择没有任何资源类型标签的资源 - if c.RsTypes[0] == "all" { - db.Where("JSON_LENGTH(rs_types) <= 0") - } else { - db.Where("(?)", model.JSONQuery("rs_types").JointOrContains(c.RsTypes)) - } + if lo.IsNotEmpty(c.RsType) { + db.Where("rs_type = ? ", c.RsType) } c.matchSpec(db) c.matchStorageSpecs(db) @@ -189,17 +185,13 @@ func (c *MachineResourceGetterInputParam) queryBs(db *gorm.DB) (err error) { } db.Where(" city in (?) ", realCitys) } - if len(c.SubZones) > 0 { - db.Where(" sub_zone in (?) ", c.SubZones) + if len(c.SubZoneIds) > 0 { + db.Where(" sub_zone_id in (?) ", c.SubZoneIds) } - if len(c.ForBizs) > 0 { + if c.ForBiz > 0 { // 如果参数[0],表示选择没有任何业务标签的资源 - if c.ForBizs[0] == 0 { - db.Where("JSON_LENGTH(dedicated_bizs) <= 0") - } else { - db.Where("(?)", model.JSONQuery("dedicated_bizs").JointOrContains(cmutil.IntSliceToStrSlice(c.ForBizs))) - } + db.Where("dedicated_biz = ?", c.ForBiz) } if cmutil.IsNotEmpty(c.OsType) { db.Where("os_type = ?", c.OsType) diff --git a/dbm-services/common/db-resource/internal/controller/manage/manage.go b/dbm-services/common/db-resource/internal/controller/manage/manage.go index a29d3dd509..88803cec6b 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/manage.go +++ b/dbm-services/common/db-resource/internal/controller/manage/manage.go @@ -23,6 +23,7 @@ import ( "dbm-services/common/go-pubpkg/logger" rf "github.com/gin-gonic/gin" + "github.com/samber/lo" ) // MachineResourceHandler 主机处理handler @@ -86,8 +87,8 @@ func (c *MachineResourceHandler) Delete(r *rf.Context) { // BatchUpdateMachineInput 批量编辑主机信息请求参数 type BatchUpdateMachineInput struct { BkHostIds []int `json:"bk_host_ids" binding:"required,dive,gt=0" ` - ForBizs []int `json:"for_bizs"` - RsTypes []string `json:"resource_types"` + ForBiz int `json:"for_biz"` + RsType string `json:"resource_type"` RackId string `json:"rack_id"` SetBizEmpty bool `json:"set_empty_biz"` SetRsTypeEmpty bool `json:"set_empty_resource_type"` @@ -111,31 +112,13 @@ func (c *MachineResourceHandler) BatchUpdate(r *rf.Context) { } // update for biz - if len(input.ForBizs) > 0 { - bizJson, err := json.Marshal(cmutil.IntSliceToStrSlice(input.ForBizs)) - if err != nil { - logger.Error(fmt.Sprintf("conver biz json Failed,Error:%s", err.Error())) - c.SendResponse(r, err, requestId, err.Error()) - return - } - updateMap["dedicated_bizs"] = bizJson - } - if input.SetBizEmpty { - updateMap["dedicated_bizs"] = EmptyArryJson + if input.ForBiz > 0 { + updateMap["dedicated_biz"] = input.ForBiz } // update resource type - if len(input.RsTypes) > 0 { - rstypes, err := json.Marshal(input.RsTypes) - if err != nil { - logger.Error(fmt.Sprintf("conver resource types Failed,Error:%s", err.Error())) - c.SendResponse(r, err, requestId, err.Error()) - return - } - updateMap["rs_types"] = rstypes - } - if input.SetRsTypeEmpty { - updateMap["rs_types"] = EmptyArryJson + if lo.IsNotEmpty(input.RsType) { + updateMap["rs_type"] = input.RsType } // update disk @@ -155,7 +138,7 @@ func (c *MachineResourceHandler) BatchUpdate(r *rf.Context) { } // do update - err := model.DB.Self.Table(model.TbRpDetailName()).Select("dedicated_bizs", "rs_types", "storage_device", "rack_id"). + err := model.DB.Self.Table(model.TbRpDetailName()).Select("dedicated_biz", "rs_type", "storage_device", "rack_id"). Where("bk_host_id in (?)", input.BkHostIds).Updates(updateMap).Error if err != nil { c.SendResponse(r, err, requestId, err.Error()) @@ -175,8 +158,8 @@ type MachineResourceInputParam struct { type MachineResource struct { BkHostID int `json:"bk_host_id" binding:"required"` Labels map[string]string `json:"labels"` - ForBizs []int `json:"for_bizs"` - RsTypes []string `json:"resource_types"` + ForBiz int `json:"for_biz"` + RsType string `json:"resource_type"` StorageDevice map[string]bk.DiskDetail `json:"storage_device"` } @@ -199,23 +182,11 @@ func (c *MachineResourceHandler) Update(r *rf.Context) { } updateMap["lable"] = l } - if len(v.ForBizs) > 0 { - bizJson, err := json.Marshal(cmutil.IntSliceToStrSlice(v.ForBizs)) - if err != nil { - logger.Error(fmt.Sprintf("conver biz json Failed,Error:%s", err.Error())) - c.SendResponse(r, err, requestId, err.Error()) - return - } - updateMap["dedicated_bizs"] = bizJson + if v.ForBiz > 0 { + updateMap["dedicated_biz"] = v.ForBiz } - if len(v.RsTypes) > 0 { - rstypes, err := json.Marshal(v.RsTypes) - if err != nil { - logger.Error(fmt.Sprintf("conver resource types Failed,Error:%s", err.Error())) - c.SendResponse(r, err, requestId, err.Error()) - return - } - updateMap["rs_types"] = rstypes + if lo.IsNotEmpty(v.RsType) { + updateMap["rs_type"] = v.RsType } if len(v.StorageDevice) > 0 { storageJson, err := json.Marshal(v.StorageDevice) @@ -226,7 +197,7 @@ func (c *MachineResourceHandler) Update(r *rf.Context) { } updateMap["storage_device"] = storageJson } - err := tx.Model(&model.TbRpDetail{}).Table(model.TbRpDetailName()).Select("dedicated_bizs", "rs_types", + err := tx.Model(&model.TbRpDetail{}).Table(model.TbRpDetailName()).Select("dedicated_biz", "rs_type", "label").Where("bk_host_id=?", v.BkHostID).Updates(updateMap).Error if err != nil { tx.Rollback() diff --git a/dbm-services/common/db-resource/internal/controller/manage/spec.go b/dbm-services/common/db-resource/internal/controller/manage/spec.go index 186f4770bf..833b6cdce8 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/spec.go +++ b/dbm-services/common/db-resource/internal/controller/manage/spec.go @@ -11,13 +11,12 @@ package manage import ( - "strconv" - "github.com/gin-gonic/gin" "dbm-services/common/db-resource/internal/model" "dbm-services/common/db-resource/internal/svr/apply" "dbm-services/common/db-resource/internal/svr/bk" + "dbm-services/common/db-resource/internal/svr/meta" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/errno" "dbm-services/common/go-pubpkg/logger" @@ -33,10 +32,10 @@ type SpecCheckInput struct { // SpecInfo TODO type SpecInfo struct { - GroupMark string `json:"group_mark" binding:"required" ` - DeviceClass []string `json:"device_class"` - Spec apply.Spec `json:"spec"` - StorageSpecs []apply.DiskSpec `json:"storage_spec"` + GroupMark string `json:"group_mark" binding:"required" ` + DeviceClass []string `json:"device_class"` + Spec meta.Spec `json:"spec"` + StorageSpecs []meta.DiskSpec `json:"storage_spec"` } // SpecSum TODO @@ -67,8 +66,7 @@ func (m MachineResourceHandler) SpecSum(r *gin.Context) { // 如果没有指定资源类型，表示只能选择无资源类型标签的资源 // 没有资源类型标签的资源可以被所有其他类型使用 if input.ForbizId > 0 { - db.Where("( ? or JSON_LENGTH(dedicated_bizs)<=0 )", model.JSONQuery("dedicated_bizs").Contains([]string{ - strconv.Itoa(input.ForbizId)})) + db.Where("dedicated_biz = ? ", input.ForbizId) } if cmutil.IsEmpty(input.ResourceType) { db.Where("JSON_LENGTH(rs_types) <= 0") diff --git a/dbm-services/common/db-resource/internal/controller/statistic/statistic.go b/dbm-services/common/db-resource/internal/controller/statistic/statistic.go new file mode 100644 index 0000000000..0d9d64feae --- /dev/null +++ b/dbm-services/common/db-resource/internal/controller/statistic/statistic.go @@ -0,0 +1,432 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +// Package statistic 资源统计相关接口 +package statistic + +import ( + "errors" + "fmt" + "hash/crc32" + "sort" + "strconv" + "strings" + "sync" + + "github.com/gin-gonic/gin" + "github.com/samber/lo" + "gorm.io/gorm" + + "dbm-services/common/db-resource/internal/controller" + "dbm-services/common/db-resource/internal/model" + "dbm-services/common/db-resource/internal/svr/dbmapi" + "dbm-services/common/go-pubpkg/logger" +) + +// Handler statistic handler +type Handler struct { + controller.BaseHandler +} + +const ( + // EmptyDeviceClass 无机型信息 + EmptyDeviceClass = "无机型信息" +) + +// RegisterRouter 注册路由 +func (s *Handler) RegisterRouter(engine *gin.Engine) { + r := engine.Group("statistic") + { + r.POST("/groupby/resource_type", s.CountGroupbyResourceType) + r.POST("/summary", s.ResourceDistribution) + } +} + +// CountGroupbyResourceType 按照资源类型统计 + +// CountGroupbyResourceTypeResult 按照资源类型统计结果数据 +type CountGroupbyResourceTypeResult struct { + RsType string `json:"rs_type"` + Count int `json:"count"` +} + +// CountGroupbyResourceType count groupby resource type +func (s *Handler) CountGroupbyResourceType(c *gin.Context) { + var data []CountGroupbyResourceTypeResult + err := model.DB.Self.Table(model.TbRpDetailName()). + Select("rs_type, count(*) as count").Group("rs_type"). + Find(&data, "status = ? ", model.Unused).Error + if err != nil { + logger.Error("query failed %s", err.Error) + s.SendResponse(c, err, err.Error(), "") + } + s.SendResponse(c, nil, data, "") +} + +// ResourDistributionParam 统计资源分布参数 +type ResourDistributionParam struct { + ForBiz int `json:"for_biz"` + City string `json:"city"` + SubZoneIds []string `json:"subzone_ids"` + GroupBy string `json:"group_by" binding:"required"` + SpecParam DbmSpecParam `json:"spec_param" ` +} + +// DbmSpecParam 规格参数 +type DbmSpecParam struct { + DbType string `json:"db_type"` + MachineType string `json:"machine_type"` + ClusterType string `json:"cluster_type"` + SpecIdList []int `json:"spec_id_list"` +} + +func (m DbmSpecParam) getQueryParam() map[string]string { + p := make(map[string]string) + if lo.IsNotEmpty(m.DbType) { + p["spec_db_type"] = m.DbType + } + if lo.IsNotEmpty(m.MachineType) { + p["spec_machine_type"] = m.MachineType + } + if lo.IsNotEmpty(m.ClusterType) { + p["spec_cluster_type"] = m.ClusterType + } + if len(m.SpecIdList) > 0 { + var specIdStrList []string + for _, specId := range m.SpecIdList { + specIdStrList = append(specIdStrList, strconv.Itoa(specId)) + } + p["spec_ids"] = strings.Join(specIdStrList, ",") + } + return p +} + +// ResourceDistribution 统计资源分布 +func (s *Handler) ResourceDistribution(c *gin.Context) { + var param ResourDistributionParam + var err error + var rsList []model.TbRpDetail + var result interface{} + + if err = s.Prepare(c, ¶m); err != nil { + logger.Error("parse resourDistributionParam failed %s", err.Error) + return + } + + dbmClient := dbmapi.NewDbmClient() + specList, err := dbmClient.GetDbmSpec(param.SpecParam.getQueryParam()) + if err != nil { + logger.Error("get dbm spec failed %s", err.Error) + s.SendResponse(c, err, err.Error(), "") + } + + db := model.DB.Self.Table(model.TbRpDetailName()) + if err = param.dbFilter(db); err != nil { + s.SendResponse(c, err, err.Error(), "") + return + } + if err = db.Find(&rsList).Error; err != nil { + logger.Error("query failed %s", err.Error) + s.SendResponse(c, err, err.Error(), "") + } + + switch param.GroupBy { + case "device_class": + var afterFilterRslist []model.TbRpDetail + if lo.IsNotEmpty(param.SpecParam.ClusterType) { + for _, rs := range rsList { + for _, spec := range specList { + if rs.MatchDbmSpec(spec) { + afterFilterRslist = append(afterFilterRslist, rs) + } + } + } + } else { + afterFilterRslist = rsList + } + result = groupByDeviceClass(afterFilterRslist) + case "spec": + + result, err = groupByDbmSpec(rsList, specList) + if err != nil { + s.SendResponse(c, err, err.Error(), "") + return + } + default: + s.SendResponse(c, errors.New("未知聚合类型"), fmt.Sprintf("未知的聚合类型%s", param.GroupBy), "") + return + } + s.SendResponse(c, nil, result, "") +} + +func (r ResourDistributionParam) dbFilter(db *gorm.DB) (err error) { + db.Where("status = ? ", model.Unused) + if lo.IsNotEmpty(r.City) { + realCitys, err := dbmapi.GetIdcCityByLogicCity(r.City) + if err != nil { + logger.Error("get idc city by logic city failed %s", err.Error()) + return fmt.Errorf("根据逻辑城市%s获取机房城市失败:%v", r.City, err) + } + db.Where("city in (?)", realCitys) + } + if len(r.SubZoneIds) > 0 { + db.Where("sub_zone_id in (?)", r.SubZoneIds) + } + if lo.IsNotEmpty(r.ForBiz) { + db.Where("dedicated_biz = ? ", r.ForBiz) + } + if r.SpecParam.DbType != "" { + db.Where("rs_type in (?) ", []string{model.PUBLIC_RESOURCE_DBTYEP, r.SpecParam.DbType}) + } + return nil +} + +func dealCity(city string) string { + if lo.IsEmpty(city) { + city = "无区域信息" + } + return city +} + +func dealDeviceClass(deviceClass string) string { + if lo.IsEmpty(deviceClass) { + deviceClass = EmptyDeviceClass + } + return deviceClass +} + +// CityGroupCount city分组统计 +type CityGroupCount struct { + City string + Count int +} + +// GroupByDeviceClassResult 按照机型聚合结果 +type GroupByDeviceClassResult struct { + DedicatedBiz int `json:"dedicated_biz"` + City string `json:"city"` + DeviceClass string `json:"device_class"` + DiskSummary string `json:"disk_summary"` + CpuMemSummary string `json:"cpu_mem_summary"` + Count int `json:"count"` + SubZoneDetail map[string]int `json:"sub_zone_detail"` +} + +func groupByDeviceClass(rpList []model.TbRpDetail) (result []GroupByDeviceClassResult) { + result = []GroupByDeviceClassResult{} + groupMap := make(map[string][]model.TbRpDetail) + subZonegroupMap := make(map[string]map[string]int) + diskDetailMap := make(map[string][]string) + cpuMemDetailMap := make(map[string]string) + for _, rs := range rpList { + if err := rs.UnmarshalDiskInfo(); err != nil { + logger.Error("%s: unmarshal disk info failed %s", rs.IP, err.Error()) + continue + } + diskHash32 := crc32.ChecksumIEEE([]byte(rs.ConcatDiskInfoIgnoreDiskId())) + groupKey := fmt.Sprintf("%d:%s:%s:%d", rs.DedicatedBiz, dealCity(rs.City), dealDeviceClass(rs.DeviceClass), + diskHash32) + if _, exist := diskDetailMap[groupKey]; !exist { + for mp, dinfo := range rs.Storages { + diskDetailMap[groupKey] = append(diskDetailMap[groupKey], fmt.Sprintf("%s:%d:%s", mp, dinfo.Size, dinfo.DiskType)) + } + } + if lo.IsNotEmpty(rs.DeviceClass) { + if _, exist := cpuMemDetailMap[rs.DeviceClass]; !exist { + cpuMemDetailMap[groupKey] = fmt.Sprintf("%d核%dG", rs.CPUNum, rs.DramCap) + } + } + groupMap[groupKey] = append(groupMap[groupKey], rs) + if _, exist := subZonegroupMap[groupKey]; !exist { + subZonegroupMap[groupKey] = make(map[string]int) + subZonegroupMap[groupKey][rs.SubZone] = 1 + } else { + subZonegroupMap[groupKey][rs.SubZone]++ + } + } + + cityMap := make(map[string][]GroupByDeviceClassResult) + cityCountMap := make(map[string]int) + for groupKey, grs := range groupMap { + dedicatedBiz, city, deviceClass, _, err := parseDeviceClassKey(groupKey) + if err != nil { + logger.Error("parse device class key failed %s", err.Error()) + continue + } + cpuMem := "无" + if deviceClass != EmptyDeviceClass { + cpuMem = cpuMemDetailMap[groupKey] + } + cityCountMap[city] += len(grs) + cityMap[city] = append(cityMap[city], GroupByDeviceClassResult{ + DedicatedBiz: dedicatedBiz, + City: city, + DeviceClass: deviceClass, + DiskSummary: strings.Join(diskDetailMap[groupKey], ";"), + Count: len(grs), + CpuMemSummary: cpuMem, + SubZoneDetail: subZonegroupMap[groupKey]}) + } + + var citysGroupKey []CityGroupCount + for city, count := range cityCountMap { + citysGroupKey = append(citysGroupKey, CityGroupCount{City: city, Count: count}) + } + // 对城市整体数量进行排序 + sort.SliceStable(citysGroupKey, func(i, j int) bool { + return citysGroupKey[i].Count > citysGroupKey[j].Count + }) + + // 按照subZone数量排序 + for _, city := range citysGroupKey { + part := cityMap[city.City] + sort.SliceStable(part, func(i, j int) bool { + return part[i].Count > part[j].Count + }) + result = append(result, part...) + } + + return result +} + +// GroupBySpecResult 按照规格聚合结果 +type GroupBySpecResult struct { + DedicatedBiz int `json:"dedicated_biz"` + City string `json:"city"` + SpecId int `json:"spec_id"` + SpecName string `json:"spec_name"` + SpecMachineType string `json:"spec_machine_type"` + SpecClusterType string `json:"spec_cluster_type"` + Count int `json:"count"` + SubZoneDetail map[string]int `json:"sub_zone_detail"` +} + +func groupByDbmSpec(rpList []model.TbRpDetail, specList []dbmapi.DbmSpec) (result []GroupBySpecResult, err error) { + result = []GroupBySpecResult{} + specMap := make(map[int]dbmapi.DbmSpec) + for _, spec := range specList { + specMap[spec.SpecId] = spec + } + specMatchrelationMap := rsMatchSpecs(rpList, specList) + groupMap := make(map[string][]model.TbRpDetail) + subZonegroupMap := make(map[string]map[string]int) + for _, rs := range rpList { + matchSpecIds, ok := specMatchrelationMap[rs.BkHostID] + if !ok { + logger.Warn("%s 资源没有匹配到任何规格", rs.IP) + continue + } + for _, specId := range matchSpecIds { + groupKey := fmt.Sprintf("%d:%s:%d", rs.DedicatedBiz, dealCity(rs.City), specId) + groupMap[groupKey] = append(groupMap[groupKey], rs) + if _, exist := subZonegroupMap[groupKey]; !exist { + subZonegroupMap[groupKey] = make(map[string]int) + subZonegroupMap[groupKey][rs.SubZone] = 1 + } else { + subZonegroupMap[groupKey][rs.SubZone]++ + } + } + } + cityMap := make(map[string][]GroupBySpecResult) + cityCountMap := make(map[string]int) + for groupKey, grs := range groupMap { + dedicatedBiz, city, specId, err := parseSpecGroupKey(groupKey) + if err != nil { + logger.Error("parse group key failed %s", err.Error) + continue + } + spec, ok := specMap[specId] + if !ok { + spec = dbmapi.DbmSpec{} + } + + cityCountMap[city] += len(grs) + cityMap[city] = append(cityMap[city], GroupBySpecResult{ + DedicatedBiz: dedicatedBiz, + SpecId: specId, + SpecName: spec.SpecName, + SpecMachineType: spec.SpecMachineType, + SpecClusterType: spec.SpecClusterType, + City: city, + Count: len(grs), + SubZoneDetail: subZonegroupMap[groupKey], + }) + } + var citysGroupKey []CityGroupCount + for city, count := range cityCountMap { + citysGroupKey = append(citysGroupKey, CityGroupCount{City: city, Count: count}) + } + // 对城市整体数量进行排序 + sort.SliceStable(citysGroupKey, func(i, j int) bool { + return citysGroupKey[i].Count > citysGroupKey[j].Count + }) + // 按照subZone数量排序 + for _, city := range citysGroupKey { + part := cityMap[city.City] + sort.SliceStable(part, func(i, j int) bool { + return part[i].Count > part[j].Count + }) + result = append(result, part...) + } + return result, nil +} + +func parseDeviceClassKey(groupKey string) (dedicatedBiz int, city, deviceClass, diskHash string, err error) { + cols := strings.Split(groupKey, ":") + if len(cols) < 4 { + return -1, "", "", "", fmt.Errorf("group key format error, %s", groupKey) + } + dedicatedBiz, err = strconv.Atoi(cols[0]) + if err != nil { + return -1, "", "", "", fmt.Errorf("dedicated biz format error, %s", cols[0]) + } + city = cols[1] + deviceClass = cols[2] + diskHash = cols[3] + return dedicatedBiz, city, deviceClass, diskHash, nil +} + +func parseSpecGroupKey(groupKey string) (dedicatedBiz int, city string, specId int, err error) { + cols := strings.Split(groupKey, ":") + if len(cols) < 3 { + return -1, "", -1, fmt.Errorf("group key format error, %s", groupKey) + } + dedicatedBiz, err = strconv.Atoi(cols[0]) + if err != nil { + return -1, "", -1, fmt.Errorf("dedicated biz format error, %s", cols[0]) + } + city = cols[1] + specId, err = strconv.Atoi(cols[2]) + return dedicatedBiz, city, specId, err +} + +func rsMatchSpecs(rsList []model.TbRpDetail, specList []dbmapi.DbmSpec) map[int][]int { + relationMap := make(map[int][]int) + ctrlChan := make(chan struct{}, 10) + wg := sync.WaitGroup{} + lc := sync.Mutex{} + for _, rs := range rsList { + for _, spec := range specList { + wg.Add(1) + ctrlChan <- struct{}{} + go func(xrs model.TbRpDetail, xspec dbmapi.DbmSpec) { + if xrs.MatchDbmSpec(xspec) { + lc.Lock() + relationMap[xrs.BkHostID] = append(relationMap[xrs.BkHostID], xspec.SpecId) + lc.Unlock() + } + wg.Done() + <-ctrlChan + }(rs, spec) + } + } + wg.Wait() + return relationMap +} diff --git a/dbm-services/common/db-resource/internal/mock/dbm_spec.go b/dbm-services/common/db-resource/internal/mock/dbm_spec.go new file mode 100644 index 0000000000..9f774eb989 --- /dev/null +++ b/dbm-services/common/db-resource/internal/mock/dbm_spec.go @@ -0,0 +1,90 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +// Package mock TODO +package mock + +import ( + "dbm-services/common/db-resource/internal/svr/dbmapi" + "dbm-services/common/db-resource/internal/svr/meta" +) + +// GetDbmSpecListMock 获取dbm_spec列表的mock数据 +func GetDbmSpecListMock() (dbmSpecList []dbmapi.DbmSpec) { + return []dbmapi.DbmSpec{ + { + SpecId: 1, + SpecName: "2核4G100G磁盘", + SpecClusterType: "single", + SpecMachineType: "single", + DeviceClass: []string{}, + Cpu: meta.MeasureRange{ + Min: 2, + Max: 4, + }, + Mem: meta.MeasureRange{ + Min: 2000, + Max: 8000, + }, + StorageSpecs: []meta.DiskSpec{ + { + MountPoint: "/data", + DiskType: "ALL", + MinSize: 10, + }, + }, + }, + { + SpecId: 2, + SpecName: "4核16G500G磁盘", + SpecClusterType: "single", + SpecMachineType: "single", + DeviceClass: []string{}, + Cpu: meta.MeasureRange{ + Min: 4, + Max: 8, + }, + Mem: meta.MeasureRange{ + Min: 6000, + Max: 16000, + }, + + StorageSpecs: []meta.DiskSpec{ + { + MountPoint: "/data", + DiskType: "ALL", + MinSize: 100, + }, + }, + }, + { + SpecId: 3, + SpecName: "16核64G3000G磁盘", + SpecClusterType: "single", + SpecMachineType: "single", + DeviceClass: []string{}, + Cpu: meta.MeasureRange{ + Min: 16, + Max: 16, + }, + Mem: meta.MeasureRange{ + Min: 60000, + Max: 64000, + }, + StorageSpecs: []meta.DiskSpec{ + { + MountPoint: "/data", + DiskType: "ALL", + MinSize: 2900, + }, + }, + }, + } +} diff --git a/dbm-services/common/db-resource/internal/model/TbRpDetail.go b/dbm-services/common/db-resource/internal/model/TbRpDetail.go index 2286f0c19c..75da567921 100644 --- a/dbm-services/common/db-resource/internal/model/TbRpDetail.go +++ b/dbm-services/common/db-resource/internal/model/TbRpDetail.go @@ -17,9 +17,11 @@ import ( "time" "dbm-services/common/db-resource/internal/svr/bk" + "dbm-services/common/db-resource/internal/svr/dbmapi" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/logger" + "github.com/samber/lo" "gorm.io/gorm" ) @@ -36,14 +38,21 @@ const ( UsedByOther = "UsedByOther" ) +const ( + // PUBLIC_RESOURCE_DBTYEP 公共资源DB类型 + PUBLIC_RESOURCE_DBTYEP = "PUBLIC" + // PUBLIC_RESOURCE_BIZ 公共资源业务ID + PUBLIC_RESOURCE_BIZ = 0 +) + // TbRpDetail 机器资源明细表 // nolint type TbRpDetail struct { ID int `gorm:"primary_key;auto_increment;not_null" json:"-"` BkCloudID int `gorm:"uniqueIndex:ip;column:bk_cloud_id;type:int(11);not null;comment:'云区域 ID'" json:"bk_cloud_id"` - BkBizId int `gorm:"column:bk_biz_id;type:int(11);not null;comment:'机器当前所属业务'" json:"bk_biz_id"` - DedicatedBizs json.RawMessage `gorm:"column:dedicated_bizs;type:json;comment:'专属业务,可属于多个'" json:"for_bizs"` - RsTypes json.RawMessage `gorm:"column:rs_types;type:json;comment:'资源类型标签'" json:"resource_types"` + BkBizId int `gorm:"column:bk_biz_id;type:int(11);not null;comment:机器当前所属业务" json:"bk_biz_id"` + DedicatedBiz int `gorm:"column:dedicated_biz;type:int(11);default:0;comment:专属业务" json:"dedicated_biz"` + RsType string `gorm:"column:rs_type;type:varchar(64);default:'PUBLIC';comment:资源专用组件类型" json:"rs_type"` Bizs map[string]string `gorm:"-" json:"-"` BkHostID int `gorm:"index:idx_host_id;column:bk_host_id;type:int(11);not null;comment:'bk主机ID'" json:"bk_host_id"` IP string `gorm:"uniqueIndex:ip;column:ip;type:varchar(20);not null" json:"ip"` @@ -110,6 +119,62 @@ func TbRpDetailName() string { return "tb_rp_detail" } +// MatchDbmSpec 资源是否匹配dbm的规格 +func (t TbRpDetail) MatchDbmSpec(spec dbmapi.DbmSpec) bool { + logger.Info("spec:%+v", spec) + logger.Info("cpu:%d,mem:%d,city:%s,disk:%s", t.CPUNum, t.DramCap, t.City, string(t.StorageDevice)) + if len(spec.DeviceClass) > 0 { + if !lo.Contains(spec.DeviceClass, t.DeviceClass) { + logger.Warn("deviceClass not match, dbmSpec:%+v, detail:%s", spec.SpecName, t.IP) + return false + } + } else { + if !isWithinRange(t.CPUNum, spec.Cpu.Min, spec.Cpu.Max) { + logger.Warn("cpu not match, dbmSpec:%+v, detail:%s", spec.SpecName, t.IP) + return false + } + if !isWithinRange(t.DramCap, spec.Mem.Min*1000, spec.Mem.Max*1000) { + logger.Warn("mem not match, dbmSpec:%+v, detail:%s", spec.SpecName, t.IP) + return false + } + } + if len(spec.StorageSpecs) > 0 { + if err := t.UnmarshalDiskInfo(); err != nil { + logger.Error("unmarshal disk info failed, err:%s") + return false + } + for _, diskSpec := range spec.StorageSpecs { + mp := diskSpec.MountPoint + realDiskInfo, ok := t.Storages[mp] + if !ok { + logger.Warn("disk not found, mp:%s, detail:%s", mp, t.IP) + return false + } + if diskSpec.DiskType != "ALL" && lo.IsNotEmpty(diskSpec.DiskType) { + if diskSpec.DiskType != realDiskInfo.DiskType { + return false + } + } + // 如果规格配置了上限 + switch { + case diskSpec.MaxSize > 0: + if !isWithinRange(realDiskInfo.Size, diskSpec.MinSize, diskSpec.MaxSize) { + return false + } + case diskSpec.MaxSize <= 0 && diskSpec.MinSize > 0: + if realDiskInfo.Size < diskSpec.MinSize { + return false + } + } + } + } + return true +} + +func isWithinRange(value, min, max int) bool { + return value >= min && value <= max +} + // DeviceClassIsLocalSSD TODO func (t TbRpDetail) DeviceClassIsLocalSSD() bool { if cmutil.IsEmpty(t.DeviceClass) { @@ -126,6 +191,14 @@ func (t *TbRpDetail) UnmarshalDiskInfo() (err error) { return } +// ConcatDiskInfoIgnoreDiskId concat disk info +func (t *TbRpDetail) ConcatDiskInfoIgnoreDiskId() (info string) { + for mp, dk := range t.Storages { + info += fmt.Sprintf("%s:%d:%s,", mp, dk.Size, dk.DiskType) + } + return +} + // GetTbRpDetailAll TODO func GetTbRpDetailAll(sqlstr string) ([]TbRpDetail, error) { var m []TbRpDetail diff --git a/dbm-services/common/db-resource/internal/model/TbRpDetailArchive.go b/dbm-services/common/db-resource/internal/model/TbRpDetailArchive.go index 60a466d4c1..36768bb50a 100644 --- a/dbm-services/common/db-resource/internal/model/TbRpDetailArchive.go +++ b/dbm-services/common/db-resource/internal/model/TbRpDetailArchive.go @@ -25,8 +25,8 @@ type TbRpDetailArchive struct { ID int `gorm:"primary_key;auto_increment;not_null" json:"-"` BkCloudID int `gorm:"column:bk_cloud_id;type:int(11);not null;comment:'云区域 ID'" json:"bk_cloud_id"` BkBizId int `gorm:"column:bk_biz_id;type:int(11);not null;comment:'机器当前所属业务'" json:"bk_biz_id"` - DedicatedBizs json.RawMessage `gorm:"column:dedicated_bizs;type:json;comment:'专属业务,可属于多个'" json:"dedicated_bizs"` - RsTypes json.RawMessage `gorm:"column:rs_types;type:json;comment:'资源类型标签'" json:"rs_types"` + DedicatedBiz int `gorm:"column:dedicated_biz;type:int(11);default:0;comment:专属业务" json:"dedicated_biz"` + RsType string `gorm:"column:rs_type;type:varchar(64);default:'PUBLIC';comment:资源专用组件类型" json:"rs_type"` Bizs map[string]string `gorm:"-"` BkHostID int `gorm:"column:bk_host_id;type:int(11);not null;comment:'bk主机ID'" json:"bk_host_id"` IP string `gorm:"column:ip;type:varchar(20);not null" json:"ip"` // svr ip diff --git a/dbm-services/common/db-resource/internal/routers/routers.go b/dbm-services/common/db-resource/internal/routers/routers.go index 02c0b37a06..98924c2b03 100644 --- a/dbm-services/common/db-resource/internal/routers/routers.go +++ b/dbm-services/common/db-resource/internal/routers/routers.go @@ -17,6 +17,7 @@ import ( "dbm-services/common/db-resource/internal/controller" "dbm-services/common/db-resource/internal/controller/apply" "dbm-services/common/db-resource/internal/controller/manage" + "dbm-services/common/db-resource/internal/controller/statistic" "github.com/gin-gonic/gin" ) @@ -31,6 +32,9 @@ func RegisterRoutes(engine *gin.Engine) { // background router background := controller.BackStageHandler{} background.RegisterRouter(engine) + // statistic router + statistic := statistic.Handler{} + statistic.RegisterRouter(engine) engine.Handle("GET", "/ping", func(context *gin.Context) { context.String(http.StatusOK, "pong") }) diff --git a/dbm-services/common/db-resource/internal/svr/apply/api.go b/dbm-services/common/db-resource/internal/svr/apply/api.go index a77a81043e..88156d420c 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/api.go +++ b/dbm-services/common/db-resource/internal/svr/apply/api.go @@ -13,15 +13,13 @@ package apply import ( "encoding/json" "fmt" - "path" "strconv" "time" "github.com/samber/lo" - "gorm.io/gorm" - "gorm.io/gorm/clause" "dbm-services/common/db-resource/internal/model" + "dbm-services/common/db-resource/internal/svr/meta" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/logger" ) @@ -195,10 +193,10 @@ type ObjectDetail struct { Labels map[string]string `json:"labels"` // 标签 // 通过机型规格 或者 资源规格描述来匹配资源 // 这两个条件是 || 关系 - DeviceClass []string `json:"device_class"` // 机器类型 "IT5.8XLARGE128" "SA3.2XLARGE32" - Spec Spec `json:"spec"` // 规格描述 - StorageSpecs []DiskSpec `json:"storage_spec"` - LocationSpec LocationSpec `json:"location_spec"` // 地域区间 + DeviceClass []string `json:"device_class"` // 机器类型 "IT5.8XLARGE128" "SA3.2XLARGE32" + Spec meta.Spec `json:"spec"` // 规格描述 + StorageSpecs []meta.DiskSpec `json:"storage_spec"` + LocationSpec meta.LocationSpec `json:"location_spec"` // 地域区间 // 反亲和性 目前只有一种选项,当campus是空的时候，则此值生效 // SAME_SUBZONE_CROSS_SWTICH: 同城同subzone跨交换机跨机架、 // SAME_SUBZONE: 同城同subzone @@ -273,156 +271,3 @@ func (a *ObjectDetail) GetMessage() (message string) { message += fmt.Sprintf("申请总数: %d \n\r", a.Count) return message } - -// GetEmptyDiskSpec get empty disk spec info -func GetEmptyDiskSpec(ds []DiskSpec) (dms []DiskSpec) { - for _, v := range ds { - if v.MountPointIsEmpty() { - dms = append(dms, v) - } - } - return -} - -// GetDiskSpecMountPoints get disk mount point -func GetDiskSpecMountPoints(ds []DiskSpec) (mountPoints []string) { - for _, v := range ds { - logger.Info("disk info %v", v) - if v.MountPointIsEmpty() { - continue - } - mountPoints = append(mountPoints, path.Clean(v.MountPoint)) - } - return -} - -// Spec cpu memory spec param -type Spec struct { - Cpu MeasureRange `json:"cpu"` // cpu range - Mem MeasureRange `json:"ram"` -} - -// IsEmpty judge spec is empty -func (s Spec) IsEmpty() bool { - return s.Cpu.IsEmpty() && s.Mem.IsEmpty() -} - -// NotEmpty judge spec is not empty -func (s Spec) NotEmpty() bool { - return s.Cpu.IsNotEmpty() || s.Mem.IsNotEmpty() -} - -// MeasureRange cpu spec range -type MeasureRange struct { - Min int `json:"min"` - Max int `json:"max"` -} - -// Iegal determine whether the parameter is legal -func (m MeasureRange) Iegal() bool { - if m.IsNotEmpty() { - return m.Max >= m.Min - } - return true -} - -// MatchTotalStorageSize match total disk capacity -func (m *MeasureRange) MatchTotalStorageSize(db *gorm.DB) { - m.MatchRange(db, "total_storage_cap") -} - -// MatchMem match memory size range -func (m *MeasureRange) MatchMem(db *gorm.DB) { - m.MatchRange(db, "dram_cap") -} - -// MatchCpu match cpu core number range -func (m *MeasureRange) MatchCpu(db *gorm.DB) { - m.MatchRange(db, "cpu_num") -} - -// MatchRange universal range matching -func (m *MeasureRange) MatchRange(db *gorm.DB, col string) { - switch { - case m.Min > 0 && m.Max > 0: - db.Where(col+" >= ? and "+col+" <= ?", m.Min, m.Max) - case m.Max > 0 && m.Min <= 0: - db.Where(col+" <= ?", m.Max) - case m.Max <= 0 && m.Min > 0: - db.Where(col+" >= ?", m.Min) - } -} - -// MatchCpuBuilder cpu builder -func (m *MeasureRange) MatchCpuBuilder() *MeasureRangeBuilder { - return &MeasureRangeBuilder{Col: "cpu_num", MeasureRange: m} -} - -// MatchMemBuilder mem builder -func (m *MeasureRange) MatchMemBuilder() *MeasureRangeBuilder { - return &MeasureRangeBuilder{Col: "dram_cap", MeasureRange: m} -} - -// MeasureRangeBuilder build range sql -type MeasureRangeBuilder struct { - Col string - *MeasureRange -} - -// Build build orm query sql -// nolint -func (m *MeasureRangeBuilder) Build(builder clause.Builder) { - switch { - case m.Min > 0 && m.Max > 0: - builder.WriteQuoted(m.Col) - builder.WriteString(fmt.Sprintf(" >= %d AND ", m.Min)) - builder.WriteQuoted(m.Col) - builder.WriteString(fmt.Sprintf(" <= %d ", m.Max)) - case m.Max > 0 && m.Min <= 0: - builder.WriteQuoted(m.Col) - builder.WriteString(fmt.Sprintf(" <= %d ", m.Max)) - case m.Max <= 0 && m.Min > 0: - builder.WriteQuoted(m.Col) - builder.WriteString(fmt.Sprintf(" >= %d ", m.Min)) - } -} - -// IsNotEmpty is not empty -func (m MeasureRange) IsNotEmpty() bool { - return m.Max > 0 && m.Min > 0 -} - -// IsEmpty is empty -func (m MeasureRange) IsEmpty() bool { - return m.Min == 0 && m.Max == 0 -} - -// DiskSpec disk spec param -type DiskSpec struct { - DiskType string `json:"disk_type"` - MinSize int `json:"min"` - MaxSize int `json:"max"` - MountPoint string `json:"mount_point"` -} - -// MountPointIsEmpty determine whether the disk parameter is empty -func (d DiskSpec) MountPointIsEmpty() bool { - return cmutil.IsEmpty(d.MountPoint) -} - -// LocationSpec location spec param -type LocationSpec struct { - City string `json:"city" validate:"required"` // 所属城市获取地域 - SubZoneIds []string `json:"sub_zone_ids"` - IncludeOrExclude bool `json:"include_or_exclue"` -} - -// IsEmpty whether the address location parameter is blank -func (l LocationSpec) IsEmpty() bool { - return cmutil.IsEmpty(l.City) -} - -// SubZoneIsEmpty determine whether subzone is empty -func (l LocationSpec) SubZoneIsEmpty() bool { - return l.IsEmpty() || len(l.SubZoneIds) == 0 -} diff --git a/dbm-services/common/db-resource/internal/svr/apply/apply.go b/dbm-services/common/db-resource/internal/svr/apply/apply.go index 531f727037..58c2a45910 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/apply.go +++ b/dbm-services/common/db-resource/internal/svr/apply/apply.go @@ -14,12 +14,12 @@ package apply import ( "fmt" "path" - "strconv" "strings" "dbm-services/common/db-resource/internal/config" "dbm-services/common/db-resource/internal/model" "dbm-services/common/db-resource/internal/svr/bk" + "dbm-services/common/db-resource/internal/svr/dbmapi" "dbm-services/common/db-resource/internal/svr/meta" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/errno" @@ -56,7 +56,7 @@ func CycleApply(param RequestInputParam) (pickers []*PickerObject, err error) { if config.AppConfig.RunMode == "dev" { idcCitys = []string{} } else if cmutil.ElementNotInArry(v.Affinity, []string{CROSS_RACK, NONE}) || lo.IsNotEmpty(&v.LocationSpec.City) { - idcCitys, err = meta.GetIdcCityByLogicCity(v.LocationSpec.City) + idcCitys, err = dbmapi.GetIdcCityByLogicCity(v.LocationSpec.City) if err != nil { logger.Error("request real citys by logic city %s from bkdbm api failed:%v", v.LocationSpec.City, err) return pickers, err @@ -134,18 +134,17 @@ func (o *SearchContext) pickBase(db *gorm.DB) { // 如果没有指定资源类型，表示只能选择无资源类型标签的资源 // 没有资源类型标签的资源可以被所有其他类型使用 - if cmutil.IsEmpty(o.RsType) { - db.Where("JSON_LENGTH(rs_types) <= 0") + if lo.IsEmpty(o.RsType) { + db.Where("rs_type == 'PUBLIC' ") } else { - db.Where("? or JSON_LENGTH(rs_types) <= 0 ", model.JSONQuery("rs_types").Contains([]string{o.RsType})) + db.Where("rs_type in (?)", []string{"PUBLIC", o.RsType}) } // 如果没有指定专属业务，就表示只能选用公共的资源 // 不能匹配打了业务标签的资源 if o.IntetionBkBizId <= 0 { - db.Where("JSON_LENGTH(dedicated_bizs) <= 0") + db.Where("dedicated_biz == 0") } else { - db.Where("? or JSON_LENGTH(dedicated_bizs) <= 0", model.JSONQuery("dedicated_bizs").Contains([]string{ - strconv.Itoa(o.IntetionBkBizId)})) + db.Where("dedicated_biz in (?)", []int{0, o.IntetionBkBizId}) } o.MatchLables(db) o.MatchLocationSpec(db) @@ -191,7 +190,7 @@ func (o *SearchContext) PickInstance() (picker *PickerObject, err error) { } // 过滤没有挂载点的磁盘匹配需求 logger.Info("storage spec %v", o.StorageSpecs) - diskSpecs := GetEmptyDiskSpec(o.StorageSpecs) + diskSpecs := meta.GetEmptyDiskSpec(o.StorageSpecs) if len(diskSpecs) > 0 { ts := []model.TbRpDetail{} for _, ins := range items { @@ -201,7 +200,7 @@ func (o *SearchContext) PickInstance() (picker *PickerObject, err error) { } logger.Info("%v", ins.Storages) noUseStorages := make(map[string]bk.DiskDetail) - smp := GetDiskSpecMountPoints(o.StorageSpecs) + smp := meta.GetDiskSpecMountPoints(o.StorageSpecs) for mp, v := range ins.Storages { if cmutil.ElementNotInArry(mp, smp) { noUseStorages[mp] = v @@ -239,7 +238,7 @@ func (o *SearchContext) MatchLables(db *gorm.DB) { db.Where(" JSON_TYPE(label) = 'NULL' OR JSON_LENGTH(label) <= 1 ") } -func matchNoMountPointStorage(spec []DiskSpec, sinc map[string]bk.DiskDetail) bool { +func matchNoMountPointStorage(spec []meta.DiskSpec, sinc map[string]bk.DiskDetail) bool { mcount := 0 for _, s := range spec { for mp, d := range sinc { @@ -253,7 +252,7 @@ func matchNoMountPointStorage(spec []DiskSpec, sinc map[string]bk.DiskDetail) bo return mcount == len(spec) } -func diskDetailMatch(d bk.DiskDetail, s DiskSpec) bool { +func diskDetailMatch(d bk.DiskDetail, s meta.DiskSpec) bool { logger.Info("spec %v", s) logger.Info("detail %v", d) if d.DiskType != s.DiskType && cmutil.IsNotEmpty(s.DiskType) { diff --git a/dbm-services/common/db-resource/internal/svr/apply/core.go b/dbm-services/common/db-resource/internal/svr/apply/core.go index d5eaa8a7d5..35de15225a 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/core.go +++ b/dbm-services/common/db-resource/internal/svr/apply/core.go @@ -146,106 +146,6 @@ func NewPicker(count int, item string) *PickerObject { } } -// PickerSameSubZone 挑选同subzone的资源 -// func (c *PickerObject) PickerSameSubZone(cross_switch bool) { -// sortSubZones := c.sortSubZone(false) -// if len(sortSubZones) == 0 { -// return -// } -// for _, subzone := range sortSubZones { -// logger.Info("PickerSameSubZone:PickeElements: %v", c.PickeElements[subzone]) -// if len(c.PickeElements[subzone]) < c.Count || len(c.PickeElements[subzone]) == 0 { -// c.ProcessLogs = append(c.ProcessLogs, fmt.Sprintf("%s 符合条件的资源有%d,实际需要申请%d,不满足！！！", -// subzone, len(c.PickeElements[subzone]), c.Count)) -// continue -// } -// logger.Info("dbeug %v", subzone) -// logger.Info("dbeug %v", c.PickeElements[subzone]) -// c.SatisfiedHostIds = []int{} -// c.ExistEquipmentIds = []string{} -// c.ExistLinkNetdeviceIds = []string{} -// for idx := range c.PickeElements[subzone] { -// logger.Info("loop %d", idx) -// c.pickerOne(subzone, cross_switch) -// // 匹配资源完成 -// logger.Info(fmt.Sprintf("surplus %s,%d", subzone, len(c.PickeElements[subzone]))) -// logger.Info(fmt.Sprintf("%s,%d,%d", subzone, c.Count, len(c.SatisfiedHostIds))) -// if c.PickerDone() { -// return -// } -// } -// } -// } - -// Picker 筛选，匹配资源 -// -// @receiver c -// @param cross_campus 是否跨园区 -// func (c *PickerObject) Picker(cross_subzone bool) { -// campKeys := c.sortSubZone(cross_subzone) -// if len(campKeys) == 0 { -// return -// } -// subzoneChan := make(chan subzone, len(campKeys)) -// for _, v := range campKeys { -// subzoneChan <- v -// } -// for subzone := range subzoneChan { -// if len(c.PickeElements[subzone]) == 0 { -// delete(c.PickeElements, subzone) -// } -// if len(c.sortSubZone(cross_subzone)) == 0 { -// logger.Info("go out here") -// close(subzoneChan) -// return -// } -// logger.Info(fmt.Sprintf("surplus %s,%d", subzone, len(c.PickeElements[subzone]))) -// logger.Info(fmt.Sprintf("%s,%d,%d", subzone, c.Count, len(c.SatisfiedHostIds))) -// if c.pickerOne(subzone, false) { -// if cross_subzone { -// delete(c.PickeElements, subzone) -// } -// } -// // 匹配资源完成 -// if c.PickerDone() { -// close(subzoneChan) -// return -// } -// // 非跨园区循环读取 -// if !cross_subzone { -// subzoneChan <- subzone -// continue -// } -// // 跨园区 -// if len(subzoneChan) == 0 { -// close(subzoneChan) -// return -// } -// } - -// } - -// func (c *PickerObject) pickerOne(key string, cross_switch bool) bool { -// c.ExistSubZone = append(c.ExistSubZone, key) -// for _, v := range c.PickeElements[key] { -// if cross_switch { -// if !c.CrossRackCheck(v) || !c.CrossSwitchCheck(v) { -// // 如果存在交集,则删除该元素 -// c.deleteElement(key, v.BkHostId) -// continue -// } -// } -// c.ExistEquipmentIds = append(c.ExistEquipmentIds, v.Equipment) -// c.SatisfiedHostIds = append(c.SatisfiedHostIds, v.BkHostId) -// c.SelectedResources = append(c.SelectedResources, v.InsDetail) -// c.ExistLinkNetdeviceIds = append(c.ExistLinkNetdeviceIds, v.LinkNetdeviceId...) -// c.PickDistrbute[key]++ -// c.deleteElement(key, v.BkHostId) -// return true -// } -// return len(c.PickeElements) == 0 -// } - // CrossSwitchCheck 跨交换机检查 func (c *PickerObject) CrossSwitchCheck(v InstanceObject) bool { if len(v.LinkNetdeviceId) == 0 { @@ -269,16 +169,6 @@ func (c *PickerObject) DebugDistrubuteLog() { } } -// func (c *PickerObject) deleteElement(key string, bkhostId int) { -// var k []InstanceObject -// for _, v := range c.PickeElements[key] { -// if v.BkHostId != bkhostId { -// k = append(k, v) -// } -// } -// c.PickeElements[key] = k -// } - // PreselectedSatisfiedInstance TODO func (c *PickerObject) PreselectedSatisfiedInstance() error { affectRows, err := model.UpdateTbRpDetail(c.SatisfiedHostIds, model.Preselected) @@ -324,29 +214,6 @@ func (pw CampusWrapper) Less(i, j int) bool { return pw.by(&pw.Campus[i], &pw.Campus[j]) } -// sortSubZone 根据排序剩下有效的园区 -// func (c *PickerObject) sortSubZone(cross_subzone bool) []string { -// var keys []string -// var campusNice []CampusNice -// for key, campusIntances := range c.PickeElements { -// // keys = append(keys, key) -// if !cross_subzone || cmutil.ElementNotInArry(key, c.ExistSubZone) { -// campusNice = append(campusNice, CampusNice{ -// Campus: key, -// Count: len(campusIntances), -// }) -// } -// } -// // 按照每个园区的数量从大到小排序 -// sort.Sort(CampusWrapper{campusNice, func(p, q *CampusNice) bool { -// return q.Count < p.Count -// }}) -// for _, capmus := range campusNice { -// keys = append(keys, capmus.Campus) -// } -// return keys -// } - // PickerDone TODO func (c *PickerObject) PickerDone() bool { return len(c.SatisfiedHostIds) == c.Count diff --git a/dbm-services/common/db-resource/internal/svr/apply/match_resource.go b/dbm-services/common/db-resource/internal/svr/apply/match_resource.go index 44565b2a2b..d486fffadf 100644 --- a/dbm-services/common/db-resource/internal/svr/apply/match_resource.go +++ b/dbm-services/common/db-resource/internal/svr/apply/match_resource.go @@ -11,7 +11,6 @@ package apply import ( - "encoding/json" "fmt" "sort" "strconv" @@ -200,18 +199,12 @@ const ( ) func (o *SearchContext) setResourcePriority(ins model.TbRpDetail, ele *Item) { - var insDedicatedBizIds []string - var insDedicatedRstyps []string + logger.Info("%v", ins.Storages) if err := ins.UnmarshalDiskInfo(); err != nil { logger.Error("%s umarshal disk failed %s", ins.IP, err.Error()) } - if err := json.Unmarshal(ins.DedicatedBizs, &insDedicatedBizIds); err != nil { - logger.Error("unmarshal dedicate bizs failed %v", err) - } - if err := json.Unmarshal(ins.RsTypes, &insDedicatedRstyps); err != nil { - logger.Error("unmarshal resource type failed %v", err) - } + // 如果请求的磁盘为空，尽量匹配没有磁盘的机器 // 请求参数需要几块盘，如果机器盘数量预制相等，则优先级更高 if len(o.StorageSpecs) == len(ins.Storages) { @@ -222,15 +215,12 @@ func (o *SearchContext) setResourcePriority(ins model.TbRpDetail, ele *Item) { ele.Priority += PriorityP1 } // 如果请求参数请求了专属业务资源，则标记了专用业务的资源优先级更高 - if o.IntetionBkBizId > 0 && lo.Contains(insDedicatedBizIds, strconv.Itoa(o.IntetionBkBizId)) { - ele.Priority += PriorityP2 - } - // 如果请求参数请求了专属db类型，则标记了专用db类型的资源优先级更高 - if lo.IsNotEmpty(o.RsType) && lo.Contains(insDedicatedRstyps, o.RsType) { + if o.IntetionBkBizId > 0 && ins.DedicatedBiz == o.IntetionBkBizId { ele.Priority += PriorityP2 } + // 如果请求参数请求了专属db类型，机器的资源类型标签只有一个，且等于请求的资源的类中，则优先级更高 - if lo.IsNotEmpty(o.RsType) && (len(insDedicatedRstyps) == 1 && insDedicatedRstyps[0] == o.RsType) { + if lo.IsNotEmpty(o.RsType) && (ins.RsType == o.RsType) { ele.Priority += PriorityP2 } // 如果是匹配的资源是redis资源 diff --git a/dbm-services/common/db-resource/internal/svr/bk/cc_test.go b/dbm-services/common/db-resource/internal/svr/bk/cc_test.go index be740211d8..6986359c27 100644 --- a/dbm-services/common/db-resource/internal/svr/bk/cc_test.go +++ b/dbm-services/common/db-resource/internal/svr/bk/cc_test.go @@ -61,9 +61,9 @@ func TestReserverCC(t *testing.T) { }) } param := manage.ImportMachParam{ - ForBizs: []int{1001, 1002}, + ForBiz: 1001, BkBizId: 100443, - RsTypes: []string{"MySQL", "Redis"}, + RsType: "MySQL", Hosts: hosts, } importResp, err := manage.Doimport(param) diff --git a/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_api.go b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_api.go new file mode 100644 index 0000000000..d9de6aae64 --- /dev/null +++ b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_api.go @@ -0,0 +1,21 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package dbmapi + +const ( + // DBMLogicCityApi 查询逻辑城市映射 + DBMLogicCityApi = "/apis/proxypass/dbmeta/bk_city_name/" + // DBMEnvironApi 查询DBM环境变量 + DBMEnvironApi = "/apis/conf/system_settings/environ/" + + // DBMSpecApi 查询规格接口 + DBMSpecApi = "/apis/dbresource/spec/" +) diff --git a/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_env.go b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_env.go new file mode 100644 index 0000000000..13c58d0c7b --- /dev/null +++ b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_env.go @@ -0,0 +1,80 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package dbmapi + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + + "dbm-services/common/go-pubpkg/logger" +) + +// DbmEnvData TODO +type DbmEnvData struct { + BK_DOMAIN string `json:"BK_DOMAIN"` + CC_IDLE_MODULE_ID int `json:"CC_IDLE_MODULE_ID"` + CC_MANAGE_TOPO struct { + SetId int `json:"set_id"` + DirtyModuleId int `json:"dirty_module_id"` + ResourceModuleId int `json:"resource_module_id"` + } `json:"CC_MANAGE_TOPO"` +} + +// GetDbmEnv get dbm env +func GetDbmEnv() (data DbmEnvData, err error) { + c := NewDbmClient() + return c.getDbmEnv() +} + +// getDbmEnv get dbm env +func (c *DbmClient) getDbmEnv() (data DbmEnvData, err error) { + u, err := url.JoinPath(c.EndPoint, DBMEnvironApi) + if err != nil { + return DbmEnvData{}, err + } + logger.Info("request url %s", u) + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return DbmEnvData{}, err + } + c.addCookie(req) + var content []byte + resp, err := http.DefaultClient.Do(req) + if resp.Body != nil { + content, err = io.ReadAll(resp.Body) + if err != nil { + logger.Error("read respone body failed %s", err.Error()) + return data, err + } + } + if err != nil { + return DbmEnvData{}, fmt.Errorf("respone body %s,err:%v", string(content), err) + } + defer resp.Body.Close() + logger.Info("get dbm env respone body %s", string(content)) + + var rpdata DbmBaseResp + if err = json.Unmarshal(content, &rpdata); err != nil { + return DbmEnvData{}, err + } + if rpdata.Code != 0 { + return DbmEnvData{}, errors.New(rpdata.Message) + } + var dbmEnvResp DbmEnvData + if err = json.Unmarshal(rpdata.Data, &dbmEnvResp); err != nil { + return DbmEnvData{}, err + } + return dbmEnvResp, nil +} diff --git a/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_logic_city.go b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_logic_city.go new file mode 100644 index 0000000000..47c1c73c54 --- /dev/null +++ b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_logic_city.go @@ -0,0 +1,94 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package dbmapi + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/url" + "time" + + "dbm-services/common/go-pubpkg/logger" +) + +// GetIdcCityByLogicCityParam TODO +type GetIdcCityByLogicCityParam struct { + LogicCityName string `json:"logic_city_name"` +} + +// IdcCitysResp idc citys respone +type IdcCitysResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data []string `json:"data"` + RequestId string `json:"request_id"` +} + +// GetIdcCityByLogicCity 根据逻辑城市获取实际对应城市列表 +func GetIdcCityByLogicCity(logicCity string) (idcCitys []string, err error) { + var content []byte + cli := NewDbmClient() + u, err := url.JoinPath(cli.EndPoint, DBMLogicCityApi) + if err != nil { + return nil, err + } + p := GetIdcCityByLogicCityParam{ + LogicCityName: logicCity, + } + body, err := json.Marshal(p) + if err != nil { + logger.Error("marshal GetIdcCityByLogicCityParam body failed %s ", err.Error()) + return nil, err + } + request, err := http.NewRequest(http.MethodPost, u, bytes.NewBuffer(body)) + if err != nil { + return nil, err + } + request.Header.Add("content-type", "application/json;charset=utf-8") + cli.addCookie(request) + + f := func() (content []byte, err error) { + resp, err := cli.Client.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + content, err = io.ReadAll(resp.Body) + if err != nil { + logger.Error("read respone body failed %s", err.Error()) + return nil, err + } + return + } + + for i := 0; i <= 3; i++ { + content, err = f() + if err == nil { + break + } + logger.Error("read respone body failed %s", err.Error()) + time.Sleep(1 * time.Second) + } + + if err != nil { + logger.Error("try 3 time get real citys from dbm failed %s", err.Error()) + return nil, err + } + + logger.Info("respone %v", string(content)) + var d IdcCitysResp + if err = json.Unmarshal(content, &d); err != nil { + return nil, err + } + return d.Data, nil +} diff --git a/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_spec.go b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_spec.go new file mode 100644 index 0000000000..de239a6193 --- /dev/null +++ b/dbm-services/common/db-resource/internal/svr/dbmapi/dbm_spec.go @@ -0,0 +1,128 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +// Package dbmapi TODO +package dbmapi + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + + "dbm-services/common/db-resource/internal/config" + "dbm-services/common/db-resource/internal/svr/meta" + + "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/logger" +) + +// DbmSpec Data dbm 规格信息 +type DbmSpec struct { + SpecId int `json:"spec_id"` + SpecName string `json:"spec_name"` + SpecClusterType string `json:"spec_cluster_type"` + SpecMachineType string `json:"spec_machine_type"` + DeviceClass []string `json:"device_class"` + Mem meta.MeasureRange `json:"mem"` + Cpu meta.MeasureRange `json:"cpu"` + StorageSpecs []meta.DiskSpec `json:"storage_spec"` +} + +// DbmSpecBaseResp dbm 规格信息 +type DbmSpecBaseResp struct { + Count int `json:"code"` + Results []DbmSpec `json:"results"` +} + +// DbmBaseResp dbm base api respone data +type DbmBaseResp struct { + Code int `json:"code"` + Message string `json:"message"` + RequestId string `json:"request_id"` + Data json.RawMessage `json:"data"` +} + +// DbmClient request client +type DbmClient struct { + EndPoint string + AppCode string + AppSecret string + Client *http.Client +} + +// NewDbmClient TODO +func NewDbmClient() *DbmClient { + base := config.AppConfig.DbMeta + if cmutil.IsEmpty(config.AppConfig.DbMeta) { + base = "http://bk-dbm" + } + return &DbmClient{ + EndPoint: base, + AppCode: config.AppConfig.BkSecretConfig.BkAppCode, + AppSecret: config.AppConfig.BkSecretConfig.BKAppSecret, + Client: &http.Client{}, + } +} + +// GetDbmSpec 获取dbm规格 +func (c *DbmClient) GetDbmSpec(queryParam map[string]string) (specData []DbmSpec, err error) { + fullUrl, err := url.JoinPath(c.EndPoint, DBMSpecApi) + if err != nil { + return nil, err + } + u, err := url.Parse(fullUrl) + if err != nil { + return nil, err + } + query := u.Query() + for k, v := range queryParam { + query.Set(k, v) + } + query.Set("limit", "-1") + u.RawQuery = query.Encode() + request, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + c.addCookie(request) + resp, err := c.Client.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.Error("read respone body failed %s", err.Error()) + return + } + var rpdata DbmBaseResp + if err = json.Unmarshal(body, &rpdata); err != nil { + logger.Error("unmarshal respone body failed %s", err.Error()) + return + } + if rpdata.Code != 0 { + return nil, fmt.Errorf("respone code:%d,message:%s", rpdata.Code, rpdata.Message) + } + var specRespData DbmSpecBaseResp + if err = json.Unmarshal(rpdata.Data, &specRespData); err != nil { + logger.Error("unmarshal DbmBaseResp body failed %s", err.Error()) + return + } + return specRespData.Results, nil +} + +func (c *DbmClient) addCookie(request *http.Request) { + request.AddCookie(&http.Cookie{Name: "bk_app_code", Path: "/", Value: c.AppCode, + MaxAge: 86400}) + request.AddCookie(&http.Cookie{Name: "bk_app_secret", Path: "/", Value: c.AppSecret, + MaxAge: 86400}) +} diff --git a/dbm-services/common/db-resource/internal/svr/meta/disk.go b/dbm-services/common/db-resource/internal/svr/meta/disk.go new file mode 100644 index 0000000000..7a8c5e8a36 --- /dev/null +++ b/dbm-services/common/db-resource/internal/svr/meta/disk.go @@ -0,0 +1,55 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package meta + +import ( + "path" + + "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/logger" +) + +// MeasureRange cpu spec range + +// DiskSpec disk spec param +type DiskSpec struct { + DiskType string `json:"disk_type"` + MinSize int `json:"min"` + MaxSize int `json:"max"` + MountPoint string `json:"mount_point"` +} + +// MountPointIsEmpty determine whether the disk parameter is empty +func (d DiskSpec) MountPointIsEmpty() bool { + return cmutil.IsEmpty(d.MountPoint) +} + +// GetEmptyDiskSpec get empty disk spec info +func GetEmptyDiskSpec(ds []DiskSpec) (dms []DiskSpec) { + for _, v := range ds { + if v.MountPointIsEmpty() { + dms = append(dms, v) + } + } + return +} + +// GetDiskSpecMountPoints get disk mount point +func GetDiskSpecMountPoints(ds []DiskSpec) (mountPoints []string) { + for _, v := range ds { + logger.Info("disk info %v", v) + if v.MountPointIsEmpty() { + continue + } + mountPoints = append(mountPoints, path.Clean(v.MountPoint)) + } + return +} diff --git a/dbm-services/common/db-resource/internal/svr/meta/location.go b/dbm-services/common/db-resource/internal/svr/meta/location.go new file mode 100644 index 0000000000..bf9a5eb1b8 --- /dev/null +++ b/dbm-services/common/db-resource/internal/svr/meta/location.go @@ -0,0 +1,30 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package meta + +import "dbm-services/common/go-pubpkg/cmutil" + +// LocationSpec location spec param +type LocationSpec struct { + City string `json:"city" validate:"required"` // 所属城市获取地域 + SubZoneIds []string `json:"sub_zone_ids"` + IncludeOrExclude bool `json:"include_or_exclue"` +} + +// IsEmpty whether the address location parameter is blank +func (l LocationSpec) IsEmpty() bool { + return cmutil.IsEmpty(l.City) +} + +// SubZoneIsEmpty determine whether subzone is empty +func (l LocationSpec) SubZoneIsEmpty() bool { + return l.IsEmpty() || len(l.SubZoneIds) == 0 +} diff --git a/dbm-services/common/db-resource/internal/svr/meta/meta.go b/dbm-services/common/db-resource/internal/svr/meta/meta.go index 66ef93a1b0..f9658b7bc1 100644 --- a/dbm-services/common/db-resource/internal/svr/meta/meta.go +++ b/dbm-services/common/db-resource/internal/svr/meta/meta.go @@ -1,3 +1,4 @@ +// Package meta TODO /* * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. @@ -7,165 +8,96 @@ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ - -// Package meta TODO package meta import ( - "bytes" - "encoding/json" - "errors" "fmt" - "io" - "net/http" - "net/url" - "time" - "dbm-services/common/db-resource/internal/config" - "dbm-services/common/go-pubpkg/cmutil" - "dbm-services/common/go-pubpkg/logger" + "gorm.io/gorm" + "gorm.io/gorm/clause" ) -const ( - // DBMCityUrl 查询逻辑城市映射 - DBMCityUrl = "/apis/proxypass/dbmeta/bk_city_name/" - // DBMEnviron 查询DBM环境变量 - DBMEnviron = "/apis/conf/system_settings/environ/" -) +// MeasureRange TODO +type MeasureRange struct { + Min int `json:"min"` + Max int `json:"max"` +} -// GetIdcCityByLogicCityParam TODO -type GetIdcCityByLogicCityParam struct { - LogicCityName string `json:"logic_city_name"` +// Iegal determine whether the parameter is legal +func (m MeasureRange) Iegal() bool { + if m.IsNotEmpty() { + return m.Max >= m.Min + } + return true } -// IdcCitysResp idc citys respone -type IdcCitysResp struct { - Code int `json:"code"` - Message string `json:"message"` - Data []string `json:"data"` - RequestId string `json:"request_id"` +// MatchTotalStorageSize match total disk capacity +func (m *MeasureRange) MatchTotalStorageSize(db *gorm.DB) { + m.MatchRange(db, "total_storage_cap") } -// DbmEnvResp dbm env respone -type DbmEnvResp struct { - Data DbmEnvData `json:"data"` - Code int `json:"code"` - Message string `json:"message"` - RequestId string `json:"request_id"` +// MatchMem match memory size range +func (m *MeasureRange) MatchMem(db *gorm.DB) { + m.MatchRange(db, "dram_cap") } -// DbmEnvData TODO -type DbmEnvData struct { - BK_DOMAIN string `json:"BK_DOMAIN"` - CC_IDLE_MODULE_ID int `json:"CC_IDLE_MODULE_ID"` - CC_MANAGE_TOPO struct { - SetId int `json:"set_id"` - DirtyModuleId int `json:"dirty_module_id"` - ResourceModuleId int `json:"resource_module_id"` - } `json:"CC_MANAGE_TOPO"` +// MatchCpu match cpu core number range +func (m *MeasureRange) MatchCpu(db *gorm.DB) { + m.MatchRange(db, "cpu_num") } -// GetDbmEnv get dbm env -func GetDbmEnv() (data DbmEnvData, err error) { - u, err := getRequestUrl(DBMEnviron) - if err != nil { - return DbmEnvData{}, err - } - logger.Info("request url %s", u) - req, err := http.NewRequest(http.MethodGet, u, nil) - if err != nil { - return DbmEnvData{}, err - } - addCookie(req) - var content []byte - resp, err := http.DefaultClient.Do(req) - if resp.Body != nil { - content, err = io.ReadAll(resp.Body) - if err != nil { - logger.Error("read respone body failed %s", err.Error()) - return data, err - } - } - if err != nil { - return DbmEnvData{}, fmt.Errorf("respone body %s,err:%v", string(content), err) - } - defer resp.Body.Close() - var rpdata DbmEnvResp - if err = json.Unmarshal(content, &rpdata); err != nil { - return DbmEnvData{}, err - } - if rpdata.Code != 0 { - return DbmEnvData{}, errors.New(rpdata.Message) +// MatchRange universal range matching +func (m *MeasureRange) MatchRange(db *gorm.DB, col string) { + switch { + case m.Min > 0 && m.Max > 0: + db.Where(col+" >= ? and "+col+" <= ?", m.Min, m.Max) + case m.Max > 0 && m.Min <= 0: + db.Where(col+" <= ?", m.Max) + case m.Max <= 0 && m.Min > 0: + db.Where(col+" >= ?", m.Min) } - logger.Info("get dbm env respone body %s", string(content)) - return rpdata.Data, nil } -func addCookie(request *http.Request) { - request.AddCookie(&http.Cookie{Name: "bk_app_code", Path: "/", Value: config.AppConfig.BkSecretConfig.BkAppCode, - MaxAge: 86400}) - request.AddCookie(&http.Cookie{Name: "bk_app_secret", Path: "/", Value: config.AppConfig.BkSecretConfig.BKAppSecret, - MaxAge: 86400}) +// MatchCpuBuilder cpu builder +func (m *MeasureRange) MatchCpuBuilder() *MeasureRangeBuilder { + return &MeasureRangeBuilder{Col: "cpu_num", MeasureRange: m} } -func getRequestUrl(apiaddr string) (string, error) { - base := config.AppConfig.DbMeta - if cmutil.IsEmpty(config.AppConfig.DbMeta) { - base = "http://bk-dbm" - } - return url.JoinPath(base, apiaddr) +// MatchMemBuilder mem builder +func (m *MeasureRange) MatchMemBuilder() *MeasureRangeBuilder { + return &MeasureRangeBuilder{Col: "dram_cap", MeasureRange: m} } -// GetIdcCityByLogicCity 根据逻辑城市获取实际对应城市列表 -func GetIdcCityByLogicCity(logicCity string) (idcCitys []string, err error) { - var content []byte - u, err := getRequestUrl(DBMCityUrl) - if err != nil { - return nil, err - } - p := GetIdcCityByLogicCityParam{ - LogicCityName: logicCity, - } - client := &http.Client{} // 客户端,被Get,Head以及Post使用 - body, err := json.Marshal(p) - if err != nil { - logger.Error("marshal GetIdcCityByLogicCityParam body failed %s ", err.Error()) - return nil, err - } - request, err := http.NewRequest(http.MethodPost, u, bytes.NewBuffer(body)) - if err != nil { - return nil, err - } - request.Header.Add("content-type", "application/json;charset=utf-8") - addCookie(request) +// MeasureRangeBuilder build range sql +type MeasureRangeBuilder struct { + Col string + *MeasureRange +} - f := func() (content []byte, err error) { - resp, err := client.Do(request) - if err != nil { - return nil, err - } - defer resp.Body.Close() - content, err = io.ReadAll(resp.Body) - if err != nil { - logger.Error("read respone body failed %s", err.Error()) - return nil, err - } - return +// Build build orm query sql +// nolint +func (m *MeasureRangeBuilder) Build(builder clause.Builder) { + switch { + case m.Min > 0 && m.Max > 0: + builder.WriteQuoted(m.Col) + builder.WriteString(fmt.Sprintf(" >= %d AND ", m.Min)) + builder.WriteQuoted(m.Col) + builder.WriteString(fmt.Sprintf(" <= %d ", m.Max)) + case m.Max > 0 && m.Min <= 0: + builder.WriteQuoted(m.Col) + builder.WriteString(fmt.Sprintf(" <= %d ", m.Max)) + case m.Max <= 0 && m.Min > 0: + builder.WriteQuoted(m.Col) + builder.WriteString(fmt.Sprintf(" >= %d ", m.Min)) } +} - for i := 0; i <= 3; i++ { - content, err = f() - if err == nil { - break - } - logger.Error("read respone body failed %s", err.Error()) - time.Sleep(1 * time.Second) - } +// IsNotEmpty is not empty +func (m MeasureRange) IsNotEmpty() bool { + return m.Max > 0 && m.Min > 0 +} - logger.Info("respone %v", string(content)) - var d IdcCitysResp - if err = json.Unmarshal(content, &d); err != nil { - return nil, err - } - return d.Data, nil +// IsEmpty is empty +func (m MeasureRange) IsEmpty() bool { + return m.Min == 0 && m.Max == 0 } diff --git a/dbm-services/common/db-resource/internal/svr/meta/spec.go b/dbm-services/common/db-resource/internal/svr/meta/spec.go new file mode 100644 index 0000000000..0516a7b288 --- /dev/null +++ b/dbm-services/common/db-resource/internal/svr/meta/spec.go @@ -0,0 +1,27 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package meta + +// Spec cpu memory spec param +type Spec struct { + Cpu MeasureRange `json:"cpu"` // cpu range + Mem MeasureRange `json:"ram"` +} + +// IsEmpty judge spec is empty +func (s Spec) IsEmpty() bool { + return s.Cpu.IsEmpty() && s.Mem.IsEmpty() +} + +// NotEmpty judge spec is not empty +func (s Spec) NotEmpty() bool { + return s.Cpu.IsNotEmpty() || s.Mem.IsNotEmpty() +} diff --git a/dbm-services/common/db-resource/internal/svr/task/inspection_task.go b/dbm-services/common/db-resource/internal/svr/task/inspection_task.go index ceabebbd4a..8e538d5008 100644 --- a/dbm-services/common/db-resource/internal/svr/task/inspection_task.go +++ b/dbm-services/common/db-resource/internal/svr/task/inspection_task.go @@ -13,7 +13,8 @@ package task import ( "dbm-services/common/db-resource/internal/model" "dbm-services/common/db-resource/internal/svr/bk" - "dbm-services/common/db-resource/internal/svr/meta" + "dbm-services/common/db-resource/internal/svr/dbmapi" + "dbm-services/common/go-pubpkg/cc.v3" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/logger" @@ -23,13 +24,13 @@ import ( func InspectCheckResource() (err error) { // 获取空闲机器 var machines []model.TbRpDetail - var allowCCMouduleInfo meta.DbmEnvData + var allowCCMouduleInfo dbmapi.DbmEnvData err = model.DB.Self.Table(model.TbRpDetailName()).Find(&machines, "status = ?", model.Unused).Error if err != nil { logger.Error("get unused machines failed %s", err.Error()) return err } - allowCCMouduleInfo, err = meta.GetDbmEnv() + allowCCMouduleInfo, err = dbmapi.GetDbmEnv() if err != nil { logger.Error("get dbm env failed %s", err.Error()) return err