From 6394db6a64b4f24e2f5d361dc7398ae8915a33d7 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 20 Dec 2021 17:26:29 -0600 Subject: [PATCH 1/8] Add support for query struct --- pkg/analyse/grafana.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/analyse/grafana.go b/pkg/analyse/grafana.go index 74b53a182..c2c4b129a 100644 --- a/pkg/analyse/grafana.go +++ b/pkg/analyse/grafana.go @@ -3,6 +3,7 @@ package analyse import ( "encoding/json" "fmt" + "reflect" "regexp" "sort" "strings" @@ -84,7 +85,21 @@ func metricsFromTemplating(templating sdk.Templating, metrics map[string]struct{ if templateVar.Type != "query" { continue } - if query, ok := templateVar.Query.(string); ok { + + query, ok := templateVar.Query.(string) + if !ok { + iter := reflect.ValueOf(templateVar.Query).MapRange() + for iter.Next() { + key := iter.Key().Interface() + value := iter.Value().Interface() + if key == "query" { + query = value.(string) + break + } + } + } + + if query != "" { // label_values if strings.Contains(query, "label_values") { re := regexp.MustCompile(`label_values\(([a-zA-Z0-9_]+)`) From ac4c5a3a8010d644834c162708727cc31891a667 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 20 Dec 2021 17:27:46 -0600 Subject: [PATCH 2/8] Expand list of custom panels --- pkg/analyse/grafana.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/analyse/grafana.go b/pkg/analyse/grafana.go index c2c4b129a..34542cd53 100644 --- a/pkg/analyse/grafana.go +++ b/pkg/analyse/grafana.go @@ -133,7 +133,19 @@ func metricsFromTemplating(templating sdk.Templating, metrics map[string]struct{ // Workaround to support Grafana "timeseries" panel. This should // be implemented in grafana/tools-sdk, and removed from here. func getCustomPanelTargets(panel sdk.Panel) *[]sdk.Target { - if panel.CommonPanel.Type != "timeseries" { + + switch panel.CommonPanel.Type { + case + "timeseries", + "piechart", + "gauge", + "table-old", + "jdbranham-diagram-panel", + "agenty-flowcharting-panel", + "grafana-worldmap-panel", + "digiapulssi-breadcrumb-panel", + "grafana-piechart-panel": + default: return nil } From cc5c0dcc91cb3c64fa29cbb59123d53554af894a Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 20 Dec 2021 17:28:28 -0600 Subject: [PATCH 3/8] Skip panels that don't contain queries --- pkg/analyse/grafana.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/analyse/grafana.go b/pkg/analyse/grafana.go index 34542cd53..c30e5ed88 100644 --- a/pkg/analyse/grafana.go +++ b/pkg/analyse/grafana.go @@ -175,6 +175,20 @@ func getCustomPanelTargets(panel sdk.Panel) *[]sdk.Target { func metricsFromPanel(panel sdk.Panel, metrics map[string]struct{}) []error { var parseErrors []error + switch panel.CommonPanel.Type { + case + "row", + "welcome", + "dashlist", + "news", + "annolist", + "alertlist", + "pluginlist", + "grafana-clock-panel", + "text": + return parseErrors // Let's not throw parse errors...these don't contain queries! + } + targets := panel.GetTargets() if targets == nil { targets = getCustomPanelTargets(panel) From e6f5093eef54bf3cb2744d3dec049202b9f09ae9 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 20 Dec 2021 17:30:05 -0600 Subject: [PATCH 4/8] Allow spaces for label_values regex --- pkg/analyse/grafana.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/analyse/grafana.go b/pkg/analyse/grafana.go index c30e5ed88..895861722 100644 --- a/pkg/analyse/grafana.go +++ b/pkg/analyse/grafana.go @@ -102,7 +102,7 @@ func metricsFromTemplating(templating sdk.Templating, metrics map[string]struct{ if query != "" { // label_values if strings.Contains(query, "label_values") { - re := regexp.MustCompile(`label_values\(([a-zA-Z0-9_]+)`) + re := regexp.MustCompile(`label_values\(\s*([a-zA-Z0-9_]+)`) sm := re.FindStringSubmatch(query) // In case of really gross queries, like - https://github.com/grafana/jsonnet-libs/blob/e97ab17f67ab40d5fe3af7e59151dd43be03f631/hass-mixin/dashboard.libsonnet#L93 if len(sm) > 0 { From 14acc1d829b9ea3d7ec849e114026afe0ae7d0a1 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 20 Dec 2021 17:31:05 -0600 Subject: [PATCH 5/8] Additional replacement of variables for query --- pkg/analyse/grafana.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/analyse/grafana.go b/pkg/analyse/grafana.go index 895861722..782a1b531 100644 --- a/pkg/analyse/grafana.go +++ b/pkg/analyse/grafana.go @@ -216,6 +216,15 @@ func metricsFromPanel(panel sdk.Panel, metrics map[string]struct{}) []error { } func parseQuery(query string, metrics map[string]struct{}) error { + re := regexp.MustCompile(`\[\s*\$(\w+|{\w+})\]`) // variable rate interval + query = re.ReplaceAllString(query, "[1s]") + + re1 := regexp.MustCompile(`offset\s+\$(\w+|{\w+})`) // variable rate interval + query = re1.ReplaceAllString(query, "offset 1s") + + re2 := regexp.MustCompile(`(by\s*\()\$((\w+|{\w+}))`) // variable by clause + query = re2.ReplaceAllString(query, "$1$2") + query = strings.ReplaceAll(query, `$__interval`, "5m") query = strings.ReplaceAll(query, `$interval`, "5m") query = strings.ReplaceAll(query, `$resolution`, "5s") @@ -223,6 +232,13 @@ func parseQuery(query string, metrics map[string]struct{}) error { query = strings.ReplaceAll(query, "$__range", "1d") query = strings.ReplaceAll(query, "${__range_s:glob}", "30") query = strings.ReplaceAll(query, "${__range_s}", "30") + + re3 := regexp.MustCompile(`\$(\w+|{\w+})`) + query = re3.ReplaceAllString(query, "79197919") // Strip out all variables + + re4 := regexp.MustCompile(`\$(__(to|from):date:\w+\b|{__(to|from):date:\w+})`) + query = re4.ReplaceAllString(query, "12") // Replace dates + expr, err := parser.ParseExpr(query) if err != nil { return err @@ -230,6 +246,9 @@ func parseQuery(query string, metrics map[string]struct{}) error { parser.Inspect(expr, func(node parser.Node, path []parser.Node) error { if n, ok := node.(*parser.VectorSelector); ok { + if strings.Contains(n.Name, "79197919") { + return errors.New("Query contains a variable in the metric name") + } metrics[n.Name] = struct{}{} } From 353bf96263b4d9e57d070e2b4490640dfdbfc382 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 20 Dec 2021 17:51:59 -0600 Subject: [PATCH 6/8] Update comments --- pkg/analyse/grafana.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/analyse/grafana.go b/pkg/analyse/grafana.go index 782a1b531..d3deb7a1b 100644 --- a/pkg/analyse/grafana.go +++ b/pkg/analyse/grafana.go @@ -88,7 +88,7 @@ func metricsFromTemplating(templating sdk.Templating, metrics map[string]struct{ query, ok := templateVar.Query.(string) if !ok { - iter := reflect.ValueOf(templateVar.Query).MapRange() + iter := reflect.ValueOf(templateVar.Query).MapRange() // A query struct for iter.Next() { key := iter.Key().Interface() value := iter.Value().Interface() @@ -219,7 +219,7 @@ func parseQuery(query string, metrics map[string]struct{}) error { re := regexp.MustCompile(`\[\s*\$(\w+|{\w+})\]`) // variable rate interval query = re.ReplaceAllString(query, "[1s]") - re1 := regexp.MustCompile(`offset\s+\$(\w+|{\w+})`) // variable rate interval + re1 := regexp.MustCompile(`offset\s+\$(\w+|{\w+})`) // variable offset query = re1.ReplaceAllString(query, "offset 1s") re2 := regexp.MustCompile(`(by\s*\()\$((\w+|{\w+}))`) // variable by clause From 53b4a7b3fbd2cda390ab3961396d02e856c7cd2b Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Tue, 21 Dec 2021 10:10:22 -0600 Subject: [PATCH 7/8] Add comments about magic number --- pkg/analyse/grafana.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/analyse/grafana.go b/pkg/analyse/grafana.go index d3deb7a1b..8dd499007 100644 --- a/pkg/analyse/grafana.go +++ b/pkg/analyse/grafana.go @@ -233,11 +233,13 @@ func parseQuery(query string, metrics map[string]struct{}) error { query = strings.ReplaceAll(query, "${__range_s:glob}", "30") query = strings.ReplaceAll(query, "${__range_s}", "30") - re3 := regexp.MustCompile(`\$(\w+|{\w+})`) - query = re3.ReplaceAllString(query, "79197919") // Strip out all variables + re3 := regexp.MustCompile(`\$(__(to|from):date:\w+\b|{__(to|from):date:\w+})`) + query = re3.ReplaceAllString(query, "12") // Replace dates - re4 := regexp.MustCompile(`\$(__(to|from):date:\w+\b|{__(to|from):date:\w+})`) - query = re4.ReplaceAllString(query, "12") // Replace dates + // Replace *all* variables. Magic number 79197919 is used as it's unlikely to appear in a query. Some queries have + // variable metric names. There is a check a few lines below for this edge case. + re4 := regexp.MustCompile(`\$(\w+|{\w+})`) + query = re4.ReplaceAllString(query, "79197919") expr, err := parser.ParseExpr(query) if err != nil { @@ -246,7 +248,7 @@ func parseQuery(query string, metrics map[string]struct{}) error { parser.Inspect(expr, func(node parser.Node, path []parser.Node) error { if n, ok := node.(*parser.VectorSelector); ok { - if strings.Contains(n.Name, "79197919") { + if strings.Contains(n.Name, "79197919") { // Check for the magic number in the metric name..drop it return errors.New("Query contains a variable in the metric name") } metrics[n.Name] = struct{}{} From 61aca87bf4fe3c8857e82c0c988aee09f3d63d8c Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Wed, 15 Jun 2022 10:57:46 -0500 Subject: [PATCH 8/8] Update grafana-tools dependency --- go.mod | 2 +- go.sum | 4 +- vendor/github.com/grafana-tools/sdk/panel.go | 148 ++++++++++--------- vendor/modules.txt | 4 +- 4 files changed, 83 insertions(+), 75 deletions(-) diff --git a/go.mod b/go.mod index 23736cdac..4d0008f65 100644 --- a/go.mod +++ b/go.mod @@ -73,4 +73,4 @@ replace github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0- replace github.com/hashicorp/consul => github.com/hashicorp/consul v1.8.1 -replace github.com/grafana-tools/sdk => github.com/colega/grafana-tools-sdk v0.0.0-20220323154849-711bca56d13f +replace github.com/grafana-tools/sdk => github.com/colega/grafana-tools-sdk v0.0.0-20220401112130-163cd43fb5e4 diff --git a/go.sum b/go.sum index 116f1e7c8..8968d6619 100644 --- a/go.sum +++ b/go.sum @@ -416,8 +416,8 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/colega/grafana-tools-sdk v0.0.0-20220323154849-711bca56d13f h1:Mc/WpMhT0pzDD5zGjhge7PiO7nkrMME4GuGS1y4HGwk= -github.com/colega/grafana-tools-sdk v0.0.0-20220323154849-711bca56d13f/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= +github.com/colega/grafana-tools-sdk v0.0.0-20220401112130-163cd43fb5e4 h1:u75LBGHfI1jW4gY3gM5h0WFizp7Yues4go+z6ope6vk= +github.com/colega/grafana-tools-sdk v0.0.0-20220401112130-163cd43fb5e4/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= diff --git a/vendor/github.com/grafana-tools/sdk/panel.go b/vendor/github.com/grafana-tools/sdk/panel.go index 34479bd6f..073ea9a2b 100644 --- a/vendor/github.com/grafana-tools/sdk/panel.go +++ b/vendor/github.com/grafana-tools/sdk/panel.go @@ -23,6 +23,7 @@ import ( "bytes" "encoding/json" "errors" + "fmt" ) // Each panel may be one of these types. @@ -1030,78 +1031,85 @@ type probePanel struct { func (p *Panel) UnmarshalJSON(b []byte) (err error) { var probe probePanel - if err = json.Unmarshal(b, &probe); err == nil { - p.CommonPanel = probe.CommonPanel - switch probe.Type { - case "graph": - var graph GraphPanel - p.OfType = GraphType - if err = json.Unmarshal(b, &graph); err == nil { - p.GraphPanel = &graph - } - case "table": - var table TablePanel - p.OfType = TableType - if err = json.Unmarshal(b, &table); err == nil { - p.TablePanel = &table - } - case "text": - var text TextPanel - p.OfType = TextType - if err = json.Unmarshal(b, &text); err == nil { - p.TextPanel = &text - } - case "singlestat": - var singlestat SinglestatPanel - p.OfType = SinglestatType - if err = json.Unmarshal(b, &singlestat); err == nil { - p.SinglestatPanel = &singlestat - } - case "stat": - var stat StatPanel - p.OfType = StatType - if err = json.Unmarshal(b, &stat); err == nil { - p.StatPanel = &stat - } - case "dashlist": - var dashlist DashlistPanel - p.OfType = DashlistType - if err = json.Unmarshal(b, &dashlist); err == nil { - p.DashlistPanel = &dashlist - } - case "bargauge": - var bargauge BarGaugePanel - p.OfType = BarGaugeType - if err = json.Unmarshal(b, &bargauge); err == nil { - p.BarGaugePanel = &bargauge - } - case "heatmap": - var heatmap HeatmapPanel - p.OfType = HeatmapType - if err = json.Unmarshal(b, &heatmap); err == nil { - p.HeatmapPanel = &heatmap - } - case "timeseries": - var timeseries TimeseriesPanel - p.OfType = TimeseriesType - if err = json.Unmarshal(b, ×eries); err == nil { - p.TimeseriesPanel = ×eries - } - case "row": - var rowpanel RowPanel - p.OfType = RowType - if err = json.Unmarshal(b, &rowpanel); err == nil { - p.RowPanel = &rowpanel - } - default: - var custom = make(CustomPanel) - p.OfType = CustomType - if err = json.Unmarshal(b, &custom); err == nil { - p.CustomPanel = &custom - } + if err = json.Unmarshal(b, &probe); err != nil { + return err + } + + p.CommonPanel = probe.CommonPanel + switch probe.Type { + case "graph": + var graph GraphPanel + p.OfType = GraphType + if err = json.Unmarshal(b, &graph); err == nil { + p.GraphPanel = &graph + } + case "table": + var table TablePanel + p.OfType = TableType + if err = json.Unmarshal(b, &table); err == nil { + p.TablePanel = &table + } + case "text": + var text TextPanel + p.OfType = TextType + if err = json.Unmarshal(b, &text); err == nil { + p.TextPanel = &text + } + case "singlestat": + var singlestat SinglestatPanel + p.OfType = SinglestatType + if err = json.Unmarshal(b, &singlestat); err == nil { + p.SinglestatPanel = &singlestat + } + case "stat": + var stat StatPanel + p.OfType = StatType + if err = json.Unmarshal(b, &stat); err == nil { + p.StatPanel = &stat + } + case "dashlist": + var dashlist DashlistPanel + p.OfType = DashlistType + if err = json.Unmarshal(b, &dashlist); err == nil { + p.DashlistPanel = &dashlist + } + case "bargauge": + var bargauge BarGaugePanel + p.OfType = BarGaugeType + if err = json.Unmarshal(b, &bargauge); err == nil { + p.BarGaugePanel = &bargauge + } + case "heatmap": + var heatmap HeatmapPanel + p.OfType = HeatmapType + if err = json.Unmarshal(b, &heatmap); err == nil { + p.HeatmapPanel = &heatmap + } + case "timeseries": + var timeseries TimeseriesPanel + p.OfType = TimeseriesType + if err = json.Unmarshal(b, ×eries); err == nil { + p.TimeseriesPanel = ×eries + } + case "row": + var rowpanel RowPanel + p.OfType = RowType + if err = json.Unmarshal(b, &rowpanel); err == nil { + p.RowPanel = &rowpanel + } + default: + var custom = make(CustomPanel) + p.OfType = CustomType + if err = json.Unmarshal(b, &custom); err == nil { + p.CustomPanel = &custom } } - return + + if err != nil && (probe.Title != "" || probe.Type != "") { + err = fmt.Errorf("%w (panel %q of type %q)", err, probe.Title, probe.Type) + } + + return err } func (p *Panel) MarshalJSON() ([]byte, error) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 62bef95c1..2559b5838 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -450,7 +450,7 @@ github.com/googleapis/gax-go/v2/apierror/internal/proto github.com/gorilla/mux # github.com/gosimple/slug v1.1.1 github.com/gosimple/slug -# github.com/grafana-tools/sdk v0.0.0-20220203092117-edae16afa87b => github.com/colega/grafana-tools-sdk v0.0.0-20220323154849-711bca56d13f +# github.com/grafana-tools/sdk v0.0.0-20220203092117-edae16afa87b => github.com/colega/grafana-tools-sdk v0.0.0-20220401112130-163cd43fb5e4 ## explicit github.com/grafana-tools/sdk # github.com/grafana/dskit v0.0.0-20211103155626-4e784973d341 @@ -1169,4 +1169,4 @@ rsc.io/binaryregexp/syntax # github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 # github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab # github.com/hashicorp/consul => github.com/hashicorp/consul v1.8.1 -# github.com/grafana-tools/sdk => github.com/colega/grafana-tools-sdk v0.0.0-20220323154849-711bca56d13f +# github.com/grafana-tools/sdk => github.com/colega/grafana-tools-sdk v0.0.0-20220401112130-163cd43fb5e4