From b0d5b9922bbe170bd1056f1c6339ba731693a1c8 Mon Sep 17 00:00:00 2001 From: zhuofeng Date: Tue, 10 Sep 2024 19:39:48 +0800 Subject: [PATCH] feat: support profile result filter --- server/querier/profile/config/config.go | 22 ++++-- server/querier/profile/model/model.go | 1 + server/querier/profile/service/profile.go | 89 ++++++++++++++++++++++- server/server.yaml | 5 ++ 4 files changed, 107 insertions(+), 10 deletions(-) diff --git a/server/querier/profile/config/config.go b/server/querier/profile/config/config.go index 5fa37a5cd8c..dbbe0b806ba 100644 --- a/server/querier/profile/config/config.go +++ b/server/querier/profile/config/config.go @@ -22,9 +22,10 @@ import ( "reflect" "regexp" - "github.com/op/go-logging" - "gopkg.in/yaml.v2" "strings" + + logging "github.com/op/go-logging" + yaml "gopkg.in/yaml.v2" ) var log = logging.MustGetLogger("profile") @@ -35,11 +36,12 @@ type Config struct { } type ProfileConfig struct { - LogFile string `default:"/var/log/profile.log" yaml:"log-file"` - LogLevel string `default:"info" yaml:"log-level"` - ListenPort int `default:"20419" yaml:"listen-port"` - FlameQueryLimit int `default:"1000000" yaml:"flame_query_limit"` - Querier Querier `yaml:"querier"` + LogFile string `default:"/var/log/profile.log" yaml:"log-file"` + LogLevel string `default:"info" yaml:"log-level"` + ListenPort int `default:"20419" yaml:"listen-port"` + FlameQueryLimit int `default:"1000000" yaml:"flame_query_limit"` + Querier Querier `yaml:"querier"` + ResultFilter ResultFilter `yaml:"result-filter"` } type Querier struct { @@ -47,6 +49,12 @@ type Querier struct { Port int `default:"20416" yaml:"port"` } +type ResultFilter struct { + Disabled bool `default:"false" yaml:"disabled"` + TotalValuePercent float64 `default:"0.1" yaml:"total-value-percent"` + Depth int `default:"12" yaml:"depth"` +} + func (c *Config) expendEnv() { reConfig := reflect.ValueOf(&c.ProfileConfig) reConfig = reConfig.Elem() diff --git a/server/querier/profile/model/model.go b/server/querier/profile/model/model.go index 8871b3cbc78..4edc2ff310d 100644 --- a/server/querier/profile/model/model.go +++ b/server/querier/profile/model/model.go @@ -43,6 +43,7 @@ type ProfileTreeNode struct { ParentNodeID int SelfValue int TotalValue int + Depth int } type Debug struct { diff --git a/server/querier/profile/service/profile.go b/server/querier/profile/service/profile.go index f54b9e83092..59298e43e2f 100644 --- a/server/querier/profile/service/profile.go +++ b/server/querier/profile/service/profile.go @@ -42,8 +42,9 @@ var log = logging.MustGetLogger("profile") var InstanceProfileEventType = []string{"inuse_objects", "inuse_space", "goroutines", "mem-inuse"} const ( - initLocationCapacity = 1024 - initNodeCapacity = 8192 + initLocationCapacity = 1024 + initNodeCapacity = 8192 + FILTER_TRIGGER_THRESHOLD = 10000 ) func Profile(args model.Profile, cfg *config.QuerierConfig) (result model.ProfileTree, debug interface{}, err error) { @@ -236,6 +237,7 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string for i := range result.FunctionValues.Values { result.FunctionValues.Values[i] = []int{0, 0} } + result.NodeValues.Values = make([][]int, 0, len(nodes)) for _, node := range nodes { locationID := node.LocationID @@ -244,6 +246,11 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string result.NodeValues.Values = append(result.NodeValues.Values, []int{locationID, node.ParentNodeID, node.SelfValue, node.TotalValue}) } + if !cfg.Profile.ResultFilter.Disabled { + totalValueThreshold := int(float64(nodes[0].TotalValue) * cfg.Profile.ResultFilter.TotalValuePercent / 100) + filterResults(nodes, &result, totalValueThreshold, cfg.Profile.ResultFilter.Depth) + } + result.Functions = locations locationTypes := GetLocationType(locations, result.FunctionValues.Values, args.ProfileEventType) result.FunctionTypes = locationTypes @@ -256,6 +263,64 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string return } +func isFilterDiscard(node *model.ProfileTreeNode, totalValueThreshold, depthThreshold int) bool { + return node.TotalValue < totalValueThreshold && node.Depth > depthThreshold +} + +func filterResults(nodes []model.ProfileTreeNode, result *model.ProfileTree, totalValueThreshold, depthThreshold int) { + if len(nodes) < FILTER_TRIGGER_THRESHOLD { + return + } + // get the nodeid that needs to be deleted, + deleteNodeIDs := []int{} + for i, node := range nodes { + calculateDepth(nodes, i) + if !isFilterDiscard(&nodes[i], totalValueThreshold, depthThreshold) { + continue + } + if node.ParentNodeID >= 0 { + nodes[node.ParentNodeID].SelfValue = nodes[node.ParentNodeID].SelfValue + node.TotalValue + } + deleteNodeIDs = append(deleteNodeIDs, i) + } + + // 获取删除 nodeId 后的id和删除前的id的映射表，用于更新 parentNodeID + // obtain the mapping table between the id after nodeId is deleted and the id before deletion, used to update parentNodeID + mapping := getMapping(len(nodes), deleteNodeIDs) + + maxDepth := 0 + result.NodeValues.Values = result.NodeValues.Values[:0] + for i, node := range nodes { + if isFilterDiscard(&nodes[i], totalValueThreshold, depthThreshold) { + continue + } + parentNodeID := node.ParentNodeID + if node.ParentNodeID != -1 { + parentNodeID = mapping[node.ParentNodeID] + } + result.NodeValues.Values = append(result.NodeValues.Values, []int{node.LocationID, parentNodeID, node.SelfValue, node.TotalValue}) + + if node.Depth > maxDepth { + maxDepth = node.Depth + } + } + log.Infof("profile total nodes count %d, total value is %d, value threshold is %d, depth threshold is %d, valid node count %d, maxDepth=%d", len(nodes), nodes[0].TotalValue, totalValueThreshold, depthThreshold, len(result.NodeValues.Values), maxDepth) +} + +func calculateDepth(nodes []model.ProfileTreeNode, thisNodeID int) { + thisNode := &nodes[thisNodeID] + depth := 1 + for thisNode.ParentNodeID >= 0 { + thisNode = &nodes[thisNode.ParentNodeID] + if thisNode.Depth > 0 { + depth += thisNode.Depth + break + } + depth++ + } + nodes[thisNodeID].Depth = depth +} + func newProfileTreeNode(locationID, selfValue, totalValue int) model.ProfileTreeNode { node := model.ProfileTreeNode{} node.LocationID = locationID @@ -275,7 +340,6 @@ func updateAllParentNodes(nodes []model.ProfileTreeNode, thisNodeID, selfValue, thisNode = &nodes[thisNode.ParentNodeID] thisNode.TotalValue += totalValue } - } func CutKernelFunction(profileLocationByteSlice []byte, maxKernelStackDepth int, sep string) ([]byte, bool) { @@ -343,3 +407,22 @@ func GetLocationType(locations []string, locationValues [][]int, profileEventTyp } return locationTypes } + +func getMapping(arrLen int, indicesToRemove []int) map[int]int { + removed := make(map[int]struct{}) + for _, index := range indicesToRemove { + removed[index] = struct{}{} + } + + mapping := make(map[int]int) + newIndex := 0 + + for i := 0; i < arrLen; i++ { + if _, found := removed[i]; !found { + mapping[i] = newIndex + newIndex++ + } + } + + return mapping +} diff --git a/server/server.yaml b/server/server.yaml index 9d42a2392c2..5327ad36bc2 100644 --- a/server/server.yaml +++ b/server/server.yaml @@ -360,6 +360,11 @@ querier: # profile相关配置 profile: flame_query_limit: 1000000 + result-filter: + disabled: false + # nodes that exceed the `depth` layer and have a total-value ratio less than `total-value-percent` are not displayed. + total-value-percent: 0.1 + depth: 12 # trace-map 相关配置 trace-map: