From 4d2259fe158585b9fc001bd7affe5c1c11d92166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Benkovsk=C3=BD?= Date: Thu, 30 Mar 2023 17:53:56 +0200 Subject: [PATCH 1/3] JSC-5168 : saramaprom adjustments --- .golangci.yml | 39 ++++++++++++ Makefile | 33 ++++++++++ README.md | 93 ++-------------------------- exporter.go | 81 +++++++++--------------- go.mod | 28 +++++++-- go.sum | 150 ++++++++++++++++----------------------------- saramaprom.go | 78 +++++++++-------------- saramaprom_test.go | 75 ++++++++--------------- scheduler.go | 37 +++++++++++ scheduler_test.go | 21 +++++++ tools.go | 10 +++ 11 files changed, 304 insertions(+), 341 deletions(-) create mode 100644 .golangci.yml create mode 100644 Makefile create mode 100644 scheduler.go create mode 100644 scheduler_test.go create mode 100644 tools.go diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..91d8583 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,39 @@ +run: + timeout: 5m + modules-download-mode: readonly +linters: + disable-all: true + enable: + - gosimple + - govet + - ineffassign + - staticcheck + - typecheck + - unused + - gofmt + - revive + - gci + - gofumpt + - whitespace + - godot + - unparam + - gocritic + - gosec + +issues: + include: + - EXC0012 # disable excluding of issues about comments from revive + exclude-rules: + - path: _test\.go + linters: + - gosec + +linters-settings: + godot: + # list of regexps for excluding particular comment lines from check + exclude: + - '^ @.*' # swaggo comments like // @title + - '^ (\d+)(\.|\)).*' # enumeration comments like // 1. or // 1) + gosec: + global: + audit: true diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..883660f --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +export GO111MODULE := on +export GOPROXY := go-proxy.oss.wandera.net +export GONOSUMDB := github.com/wandera/* + +all: check test + +MAKEFLAGS += --no-print-directory + +prepare: + @echo "Downloading tools" + @cat tools.go | grep _ | cut -f2 -d " " | xargs -tI % sh -c "go install %" + +check: prepare + @echo "Running check" +ifeq (, $(shell which golangci-lint)) + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.50.1 +endif + golangci-lint run + go mod tidy + +test: prepare + @echo "Running tests" + mkdir -p report + go test -race -v ./... -coverprofile=report/coverage.txt | tee report/report.txt + go-junit-report -set-exit-code < report/report.txt > report/report.xml + gocov convert report/coverage.txt | gocov-xml > report/coverage.xml + go mod tidy + +clean: + @echo "Running clean" + rm -rf "report/" + +.PHONY: all check test prepare diff --git a/README.md b/README.md index 0bb5464..1f75739 100644 --- a/README.md +++ b/README.md @@ -1,88 +1,7 @@ # saramaprom -[![GoDoc](https://godoc.org/github.com/iimos/saramaprom?status.png)](http://godoc.org/github.com/iimos/saramaprom) -[![Go Report](https://goreportcard.com/badge/github.com/iimos/saramaprom)](https://goreportcard.com/report/github.com/iimos/saramaprom) - -This is a prometheus metrics reporter for the [sarama](https://github.com/Shopify/sarama) library. -It is based on https://github.com/deathowl/go-metrics-prometheus library. - -## Why -Because `go-metrics-prometheus` is a general solution it reports metrics with no labels so it's hard to use. Thus a sarama specific solution was made, it reports metrics with labels for brokers, topics and consumer/producer instance. - -## Installation -```console -go get github.com/iimos/saramaprom -``` - -## Usage - -```go -import ( - "context" - "github.com/Shopify/sarama" - "github.com/iimos/saramaprom" -) - -ctx := context.Background() -cfg := sarama.NewConfig() -err := saramaprom.ExportMetrics(ctx, cfg.MetricRegistry, saramaprom.Options{}) -``` - -Posible options: -```go -type Options struct { - // PrometheusRegistry is prometheus registry. Default prometheus.DefaultRegisterer. - PrometheusRegistry prometheus.Registerer - - // Namespace and Subsystem form the metric name prefix. - // Default Subsystem is "sarama". - Namespace string - Subsystem string - - // Label specifies value of "label" label. Default "". - Label string - - // FlushInterval specifies interval between updating metrics. Default 1s. - FlushInterval time.Duration - - // OnError is error handler. Default handler panics when error occurred. - OnError func(err error) - - // Debug turns on debug logging. - Debug bool -} -``` - -Metric names by default: -``` -Gauges: -sarama_batch_size -sarama_compression_ratio -sarama_incoming_byte_rate -sarama_outgoing_byte_rate -sarama_record_send_rate -sarama_records_per_request -sarama_request_latency_in_ms -sarama_request_rate -sarama_request_size -sarama_requests_in_flight -sarama_response_rate -sarama_response_size - -Histograms: -sarama_batch_size_histogram -sarama_compression_ratio_histogram -sarama_records_per_request_histogram -sarama_request_latency_in_ms_histogram -sarama_request_size_histogram -sarama_response_size_histogram -``` - -Every metric have three labels: -* broker – kafka broker id -* topic – kafka topic name -* label – custom label to distinguish different consumers/producers - - -## Requirements - -Go 1.13 or above. +is a library for exporting sarama metrics (provided through [go-metrics](https://github.com/rcrowley/go-metrics)) to Prometheus. It is +a fork of [saramaprom](https://github.com/iimos/saramaprom/tree/ab69b9d3b9e65611e5377c2fd40882124e491f50) with few fixes +and tweaks: +* go-metrics histograms are registered as Prometheus summaries (to better present client side quantiles) +* removed histogram and timer words from metric names +* removed configuration of optional labels from saramaprom (we never configure it and it was creating additional unnecessary dimension to metrics due to bad implementation) diff --git a/exporter.go b/exporter.go index 924f5f8..8605dfd 100644 --- a/exporter.go +++ b/exporter.go @@ -12,14 +12,14 @@ import ( ) type exporter struct { - opt Options - registry MetricsRegistry - promRegistry prometheus.Registerer - gauges map[string]prometheus.Gauge - customMetrics map[string]*customCollector - histogramBuckets []float64 - timerBuckets []float64 - mutex *sync.Mutex + opt Options + registry MetricsRegistry + promRegistry prometheus.Registerer + gauges map[string]prometheus.Gauge + customMetrics map[string]*customCollector + histogramQuantiles []float64 + timerQuantiles []float64 + mutex *sync.Mutex } func (c *exporter) sanitizeName(key string) string { @@ -41,9 +41,6 @@ func (c *exporter) createKey(name string) string { func (c *exporter) gaugeFromNameAndValue(name string, val float64) error { shortName, labels, skip := c.metricNameAndLabels(name) if skip { - if c.opt.Debug { - fmt.Printf("[saramaprom] skip metric %q because there is no broker or topic labels\n", name) - } return nil } @@ -57,7 +54,6 @@ func (c *exporter) gaugeFromNameAndValue(name string, val float64) error { Namespace: c.sanitizeName(c.opt.Namespace), Subsystem: c.sanitizeName(c.opt.Subsystem), Name: c.sanitizeName(shortName), - Help: shortName, }, labelNames) if err := c.promRegistry.Register(g); err != nil { @@ -100,7 +96,6 @@ func (c *exporter) metricNameAndLabels(metricName string) (newName string, label labels = map[string]string{ "broker": broker, "topic": topic, - "label": c.opt.Label, } return newName, labels, false } @@ -119,7 +114,7 @@ func parseMetricName(name string) (newName, broker, topic string) { return name, "", "" } -func (c *exporter) histogramFromNameAndMetric(name string, goMetric interface{}, buckets []float64) error { +func (c *exporter) summaryFromNameAndMetric(name string, goMetric interface{}, quantiles []float64) error { key := c.createKey(name) collector, exists := c.customMetrics[key] if !exists { @@ -131,28 +126,25 @@ func (c *exporter) histogramFromNameAndMetric(name string, goMetric interface{}, var ps []float64 var count uint64 var sum float64 - var typeName string switch metric := goMetric.(type) { case metrics.Histogram: snapshot := metric.Snapshot() - ps = snapshot.Percentiles(buckets) + ps = snapshot.Percentiles(quantiles) count = uint64(snapshot.Count()) sum = float64(snapshot.Sum()) - typeName = "histogram" case metrics.Timer: snapshot := metric.Snapshot() - ps = snapshot.Percentiles(buckets) + ps = snapshot.Percentiles(quantiles) count = uint64(snapshot.Count()) sum = float64(snapshot.Sum()) - typeName = "timer" default: return fmt.Errorf("unexpected metric type %T", goMetric) } - bucketVals := make(map[float64]uint64) - for ii, bucket := range buckets { - bucketVals[bucket] = uint64(ps[ii]) + quantilesVals := make(map[float64]float64) + for i, quantile := range quantiles { + quantilesVals[quantile] = ps[i] } name, labels, skip := c.metricNameAndLabels(name) @@ -164,27 +156,19 @@ func (c *exporter) histogramFromNameAndMetric(name string, goMetric interface{}, prometheus.BuildFQName( c.sanitizeName(c.opt.Namespace), c.sanitizeName(c.opt.Subsystem), - c.sanitizeName(name)+"_"+typeName, + c.sanitizeName(name), ), c.sanitizeName(name), nil, labels, ) - hist, err := prometheus.NewConstHistogram(desc, count, sum, bucketVals) - if err != nil { - return err - } - c.mutex.Lock() - collector.metric = hist - c.mutex.Unlock() + collector.setMetric(prometheus.MustNewConstSummary(desc, count, sum, quantilesVals)) + return nil } func (c *exporter) update() error { - if c.opt.Debug { - fmt.Print("[saramaprom] update()\n") - } var err error c.registry.Each(func(name string, i interface{}) { switch metric := i.(type) { @@ -193,31 +177,20 @@ func (c *exporter) update() error { case metrics.Gauge: err = c.gaugeFromNameAndValue(name, float64(metric.Value())) case metrics.GaugeFloat64: - err = c.gaugeFromNameAndValue(name, float64(metric.Value())) - case metrics.Histogram: // sarama - samples := metric.Snapshot().Sample().Values() - if len(samples) > 0 { - lastSample := samples[len(samples)-1] - err = c.gaugeFromNameAndValue(name, float64(lastSample)) - } - if err == nil { - err = c.histogramFromNameAndMetric(name, metric, c.histogramBuckets) - } - case metrics.Meter: // sarama + err = c.gaugeFromNameAndValue(name, metric.Value()) + case metrics.Histogram: + err = c.summaryFromNameAndMetric(name, metric, c.histogramQuantiles) + case metrics.Meter: lastSample := metric.Snapshot().Rate1() - err = c.gaugeFromNameAndValue(name, float64(lastSample)) + err = c.gaugeFromNameAndValue(name, lastSample) case metrics.Timer: - lastSample := metric.Snapshot().Rate1() - err = c.gaugeFromNameAndValue(name, float64(lastSample)) - if err == nil { - err = c.histogramFromNameAndMetric(name, metric, c.timerBuckets) - } + err = c.summaryFromNameAndMetric(name, metric, c.timerQuantiles) } }) return err } -// for collecting prometheus.constHistogram objects +// customCollector is a collector of prometheus.constSummary objects. type customCollector struct { prometheus.Collector @@ -231,6 +204,12 @@ func newCustomCollector(mu *sync.Mutex) *customCollector { } } +func (c *customCollector) setMetric(metric prometheus.Metric) { + c.mutex.Lock() + c.metric = metric + c.mutex.Unlock() +} + func (c *customCollector) Collect(ch chan<- prometheus.Metric) { c.mutex.Lock() if c.metric != nil { diff --git a/go.mod b/go.mod index 66c51f3..9a7d807 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,29 @@ module github.com/iimos/saramaprom -go 1.13 +go 1.19 require ( - github.com/prometheus/client_golang v1.7.1 - github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 - github.com/stretchr/testify v1.4.0 + github.com/AlekSi/gocov-xml v1.1.0 + github.com/axw/gocov v1.1.0 + github.com/jstemmer/go-junit-report v1.0.0 + github.com/prometheus/client_golang v1.14.0 + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 + github.com/stretchr/testify v1.8.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/tools v0.0.0-20190617190820-da514acc4774 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e7151df..5492577 100644 --- a/go.sum +++ b/go.sum @@ -1,113 +1,67 @@ -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/AlekSi/gocov-xml v1.1.0 h1:iElWGi7s/MuL8/d8WDtI2fOAsN3ap9x8nK5RrAhaDng= +github.com/AlekSi/gocov-xml v1.1.0/go.mod h1:g1dRVOCHjKkMtlPfW6BokJ/qxoeZ1uPNAK7A/ii3CUo= +github.com/axw/gocov v1.1.0 h1:y5U1krExoJDlb/kNtzxyZQmNRprFOFCutWbNjcQvmVM= +github.com/axw/gocov v1.1.0/go.mod h1:H9G4tivgdN3pYSSVrTFBr6kGDCmAkgbJhtxFzAvgcdw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/jstemmer/go-junit-report v1.0.0 h1:8X1gzZpR+nVQLAht+L/foqOeX2l9DTZoaIPbEQHxsds= +github.com/jstemmer/go-junit-report v1.0.0/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774 h1:CQVOmarCBFzTx0kbOU0ru54Cvot8SdSrNYjZPhQl+gk= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/saramaprom.go b/saramaprom.go index 33f7175..00367c4 100644 --- a/saramaprom.go +++ b/saramaprom.go @@ -1,14 +1,15 @@ package saramaprom import ( - "context" - "fmt" "sync" "time" "github.com/prometheus/client_golang/prometheus" ) +// StopFunc represents function for stopping scheduled task. +type StopFunc func() + // Options holds optional params for ExportMetrics. type Options struct { // PrometheusRegistry is prometheus registry. Default prometheus.DefaultRegisterer. @@ -19,73 +20,50 @@ type Options struct { Namespace string Subsystem string - // Label specifies value of "label" label. Default "". - Label string - - // FlushInterval specifies interval between updating metrics. Default 1s. - FlushInterval time.Duration - - // OnError is error handler. Default handler panics when error occurred. - OnError func(err error) - - // Debug turns on debug logging. - Debug bool + // RefreshInterval specifies interval between updating metrics. Default 1s. + RefreshInterval time.Duration } -// ExportMetrics exports metrics from go-metrics to prometheus. -func ExportMetrics(ctx context.Context, metricsRegistry MetricsRegistry, opt Options) error { +// ExportMetrics exports metrics from go-metrics to prometheus by starting background task, +// which periodically sync sarama metrics to prometheus registry. +func ExportMetrics(metricsRegistry MetricsRegistry, opt Options) StopFunc { if opt.PrometheusRegistry == nil { opt.PrometheusRegistry = prometheus.DefaultRegisterer } if opt.Subsystem == "" { opt.Subsystem = "sarama" } - if opt.FlushInterval == 0 { - opt.FlushInterval = time.Second - } - if opt.OnError != nil { - opt.OnError = func(err error) { - panic(fmt.Errorf("saramaprom: %w", err)) - } + if opt.RefreshInterval == 0 { + opt.RefreshInterval = time.Second } exp := &exporter{ - opt: opt, - registry: metricsRegistry, - promRegistry: opt.PrometheusRegistry, - gauges: make(map[string]prometheus.Gauge), - customMetrics: make(map[string]*customCollector), - histogramBuckets: []float64{0.05, 0.1, 0.25, 0.50, 0.75, 0.9, 0.95, 0.99}, - timerBuckets: []float64{0.50, 0.95, 0.99, 0.999}, - mutex: new(sync.Mutex), + opt: opt, + registry: metricsRegistry, + promRegistry: opt.PrometheusRegistry, + gauges: make(map[string]prometheus.Gauge), + customMetrics: make(map[string]*customCollector), + histogramQuantiles: []float64{0.05, 0.1, 0.25, 0.50, 0.75, 0.9, 0.95, 0.99}, + timerQuantiles: []float64{0.50, 0.95, 0.99, 0.999}, + mutex: new(sync.Mutex), } err := exp.update() if err != nil { - return fmt.Errorf("saramaprom: %w", err) + panic(err) } - go func() { - t := time.NewTicker(opt.FlushInterval) - defer t.Stop() - - for { - select { - case <-t.C: - err := exp.update() - if err != nil { - opt.OnError(err) - } - case <-ctx.Done(): - if err := exp.unregisterGauges(); err != nil { - opt.OnError(err) - } - return - } + scheduler := StartScheduler(opt.RefreshInterval, func() { + err := exp.update() + if err != nil { + panic(err) } - }() + }) - return nil + return func() { + _ = exp.unregisterGauges() + scheduler.Stop() + } } // MetricsRegistry is an interface for 'github.com/rcrowley/go-metrics'.Registry diff --git a/saramaprom_test.go b/saramaprom_test.go index 4dec95f..7d13fbe 100644 --- a/saramaprom_test.go +++ b/saramaprom_test.go @@ -1,17 +1,15 @@ package saramaprom_test import ( - "context" "fmt" "testing" "time" + "github.com/iimos/saramaprom" "github.com/prometheus/client_golang/prometheus" "github.com/rcrowley/go-metrics" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/iimos/saramaprom" ) func TestMetricCreation(t *testing.T) { @@ -21,10 +19,7 @@ func TestMetricCreation(t *testing.T) { err := metricsRegistry.Register("counter-for-broker-123", metrics.NewCounter()) require.NoError(t, err) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - err = saramaprom.ExportMetrics(ctx, metricsRegistry, saramaprom.Options{ + saramaprom.ExportMetrics(metricsRegistry, saramaprom.Options{ Namespace: "test", Subsystem: "subsys", PrometheusRegistry: promRegistry, @@ -36,7 +31,7 @@ func TestMetricCreation(t *testing.T) { Subsystem: "subsys", Name: "counter", Help: "counter", - }, []string{"label", "broker", "topic"}) + }, []string{"broker", "topic"}) err = promRegistry.Register(gauge) require.Error(t, err, "Go-metrics registry didn't get registered to prometheus registry") @@ -55,21 +50,16 @@ func TestLabels(t *testing.T) { err = metricsRegistry.Register("skip-counter", metrics.NewCounter()) require.NoError(t, err) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - err = saramaprom.ExportMetrics(ctx, metricsRegistry, saramaprom.Options{ + saramaprom.ExportMetrics(metricsRegistry, saramaprom.Options{ Namespace: "test", Subsystem: "subsys", - Label: "test-label", PrometheusRegistry: promRegistry, }) - require.NoError(t, err) t.Run("counter1-for-broker-123", func(t *testing.T) { want := []gaugeDetails{{ name: "test_subsys_counter1", - labels: map[string]string{"broker": "123", "label": "test-label", "topic": ""}, + labels: map[string]string{"broker": "123", "topic": ""}, gaugeValues: []float64{0}, }} got := getMetricDetails(promRegistry, "test_subsys_counter1") @@ -78,7 +68,7 @@ func TestLabels(t *testing.T) { t.Run("counter2-for-topic-abc", func(t *testing.T) { want := []gaugeDetails{{ name: "test_subsys_counter2", - labels: map[string]string{"broker": "", "label": "test-label", "topic": "abc"}, + labels: map[string]string{"broker": "", "topic": "abc"}, gaugeValues: []float64{0}, }} got := getMetricDetails(promRegistry, "test_subsys_counter2") @@ -98,21 +88,18 @@ func TestMetricUpdate(t *testing.T) { err := metricsRegistry.Register("counter-for-broker-5", counter) require.NoError(t, err) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - err = saramaprom.ExportMetrics(ctx, metricsRegistry, saramaprom.Options{ + saramaprom.ExportMetrics(metricsRegistry, saramaprom.Options{ Namespace: "test", Subsystem: "subsys", PrometheusRegistry: promRegistry, - FlushInterval: 100 * time.Millisecond, + RefreshInterval: 100 * time.Millisecond, }) require.NoError(t, err) t.Run("by default metric is 0", func(t *testing.T) { want := []gaugeDetails{{ name: "test_subsys_counter", - labels: map[string]string{"broker": "5", "label": "", "topic": ""}, + labels: map[string]string{"broker": "5", "topic": ""}, gaugeValues: []float64{0}, }} got := getMetricDetails(promRegistry, "test_subsys_counter") @@ -125,7 +112,7 @@ func TestMetricUpdate(t *testing.T) { t.Run("after 1st increment", func(t *testing.T) { want := []gaugeDetails{{ name: "test_subsys_counter", - labels: map[string]string{"broker": "5", "label": "", "topic": ""}, + labels: map[string]string{"broker": "5", "topic": ""}, gaugeValues: []float64{10}, }} got := getMetricDetails(promRegistry, "test_subsys_counter") @@ -138,7 +125,7 @@ func TestMetricUpdate(t *testing.T) { t.Run("after 2nd increment", func(t *testing.T) { want := []gaugeDetails{{ name: "test_subsys_counter", - labels: map[string]string{"broker": "5", "label": "", "topic": ""}, + labels: map[string]string{"broker": "5", "topic": ""}, gaugeValues: []float64{20}, }} got := getMetricDetails(promRegistry, "test_subsys_counter") @@ -146,7 +133,7 @@ func TestMetricUpdate(t *testing.T) { }) } -func TestHistogram(t *testing.T) { +func TestSummary(t *testing.T) { promRegistry := prometheus.NewRegistry() metricsRegistry := metrics.NewRegistry() @@ -162,36 +149,22 @@ func TestHistogram(t *testing.T) { } gm.Update(10) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - err = saramaprom.ExportMetrics(ctx, metricsRegistry, saramaprom.Options{ + saramaprom.ExportMetrics(metricsRegistry, saramaprom.Options{ Namespace: "test", Subsystem: "subsys", PrometheusRegistry: promRegistry, - FlushInterval: 100 * time.Millisecond, + RefreshInterval: 100 * time.Millisecond, }) require.NoError(t, err) time.Sleep(time.Second) metricFamilies, err := promRegistry.Gather() require.NoError(t, err) - require.Equal(t, 2, len(metricFamilies), "prometheus was unable to register the metric") - - t.Run("check gauge", func(t *testing.T) { - want := []gaugeDetails{{ - name: "test_subsys_metric", - labels: map[string]string{"broker": "", "label": "", "topic": "x"}, - gaugeValues: []float64{10}, - }} - got := getMetricDetails(promRegistry, "test_subsys_metric") - assert.Equal(t, want, got) - }) - t.Run("check histogram", func(t *testing.T) { - t.Log(metricFamilies[1].GetMetric()[0].GetHistogram()) - got := fmt.Sprint(metricFamilies[1]) - want := `name:"test_subsys_metric_histogram" help:"metric" type:HISTOGRAM metric: label: label: histogram: bucket: bucket: bucket: bucket: bucket: bucket: bucket: > > ` + t.Run("check summary", func(t *testing.T) { + t.Log(metricFamilies[0].GetMetric()[0].GetSummary()) + got := fmt.Sprint(metricFamilies[0]) + want := `name:"test_subsys_metric" help:"metric" type:SUMMARY metric: label: summary: quantile: quantile: quantile: quantile: quantile: quantile: quantile: > > ` assert.Equal(t, want, got) }) } @@ -225,13 +198,13 @@ func getMetricDetails(pr *prometheus.Registry, fullName string) []gaugeDetails { case "GAUGE": gd.gaugeValues = append(gd.gaugeValues, m.GetGauge().GetValue()) case "HISTOGRAM": - //TODO - //buckets := make(map[float64]uint64) - //m.GetHistogram().GetSampleSum() - //m.GetHistogram().GetSampleCount() - //for _, b := range m.GetHistogram().GetBucket() { + // TODO + // buckets := make(map[float64]uint64) + // m.GetHistogram().GetSampleSum() + // m.GetHistogram().GetSampleCount() + // for _, b := range m.GetHistogram().GetBucket() { // buckets[b.GetUpperBound()] = b.GetCumulativeCount() - //} + // } } ret = append(ret, gd) } diff --git a/scheduler.go b/scheduler.go new file mode 100644 index 0000000..9ce3b6c --- /dev/null +++ b/scheduler.go @@ -0,0 +1,37 @@ +package saramaprom + +import ( + "time" +) + +// Scheduler is used to run jobs in specific intervals. +type Scheduler struct { + stop chan<- struct{} +} + +// StartScheduler starts goroutine that will run given job in given intervals until Stop() is called. +func StartScheduler(interval time.Duration, job func()) Scheduler { + ticker := time.NewTicker(interval) + stop := make(chan struct{}, 1) + go func() { + for { + select { + case <-stop: + ticker.Stop() + return + case <-ticker.C: + job() + } + } + }() + + return Scheduler{stop} +} + +// Stop stops the goroutine running scheduled job. +func (s Scheduler) Stop() { + select { + case s.stop <- struct{}{}: + default: + } +} diff --git a/scheduler_test.go b/scheduler_test.go new file mode 100644 index 0000000..2c46f18 --- /dev/null +++ b/scheduler_test.go @@ -0,0 +1,21 @@ +package saramaprom + +import ( + "testing" + "time" +) + +func TestStartScheduler(t *testing.T) { + t.Run("Execute job", func(t *testing.T) { + interval := time.Millisecond * 5 + executed := make(chan struct{}) + s := StartScheduler(interval, func() { executed <- struct{}{} }) + + select { + case <-executed: + case <-time.After(interval * 2): + t.Error("Expected job to be executed but was not.") + } + s.Stop() + }) +} diff --git a/tools.go b/tools.go new file mode 100644 index 0000000..2ad1b85 --- /dev/null +++ b/tools.go @@ -0,0 +1,10 @@ +//go:build tools +// +build tools + +package tools + +import ( + _ "github.com/AlekSi/gocov-xml" + _ "github.com/axw/gocov/gocov" + _ "github.com/jstemmer/go-junit-report" +) From 570da37a6044a2459776c84010a950f9e2c4f2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Benkovsk=C3=BD?= Date: Mon, 3 Apr 2023 10:12:55 +0200 Subject: [PATCH 2/3] fixup! JSC-5168 : saramaprom adjustments --- go.mod | 2 +- saramaprom_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9a7d807..2c9c6e2 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/iimos/saramaprom +module github.com/wandera/saramaprom go 1.19 diff --git a/saramaprom_test.go b/saramaprom_test.go index 7d13fbe..d31079b 100644 --- a/saramaprom_test.go +++ b/saramaprom_test.go @@ -5,11 +5,11 @@ import ( "testing" "time" - "github.com/iimos/saramaprom" "github.com/prometheus/client_golang/prometheus" "github.com/rcrowley/go-metrics" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/wandera/saramaprom" ) func TestMetricCreation(t *testing.T) { From d19e5c3191108bc48bab56461da2a60198f8ac45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Benkovsk=C3=BD?= Date: Mon, 3 Apr 2023 10:33:23 +0200 Subject: [PATCH 3/3] fixup! JSC-5168 : saramaprom adjustments --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 883660f..7173bc2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ -export GO111MODULE := on export GOPROXY := go-proxy.oss.wandera.net export GONOSUMDB := github.com/wandera/* @@ -13,7 +12,7 @@ prepare: check: prepare @echo "Running check" ifeq (, $(shell which golangci-lint)) - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.50.1 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.52.2 endif golangci-lint run go mod tidy