From 5eef5bd6985bc9d06fa065762668dfdb732e56ad Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Tue, 12 Nov 2024 19:38:53 +0000 Subject: [PATCH 01/10] Split EMF log with larger than 100 buckets. --- exporter/awsemfexporter/datapoint.go | 191 +++++-- exporter/awsemfexporter/datapoint_test.go | 476 +++++++++++++++++- exporter/awsemfexporter/grouped_metric.go | 3 +- .../awsemfexporter/grouped_metric_test.go | 44 +- exporter/awsemfexporter/metric_translator.go | 2 + 5 files changed, 669 insertions(+), 47 deletions(-) diff --git a/exporter/awsemfexporter/datapoint.go b/exporter/awsemfexporter/datapoint.go index 1104a9e59e41..031dd43e2eb8 100644 --- a/exporter/awsemfexporter/datapoint.go +++ b/exporter/awsemfexporter/datapoint.go @@ -109,6 +109,12 @@ type summaryMetricEntry struct { count uint64 } +type dataPointSplit struct { + cWMetricHistogram *cWMetricHistogram + length int + capacity int +} + // CalculateDeltaDatapoints retrieves the NumberDataPoint at the given index and performs rate/delta calculation if necessary. func (dps numberDataPointSlice) CalculateDeltaDatapoints(i int, instrumentationScopeName string, _ bool, calculators *emfCalculators) ([]dataPoint, bool) { metric := dps.NumberDataPointSlice.At(i) @@ -193,58 +199,159 @@ func (dps histogramDataPointSlice) IsStaleNaNInf(i int) (bool, pcommon.Map) { } // CalculateDeltaDatapoints retrieves the ExponentialHistogramDataPoint at the given index. +// As CloudWatch EMF logs allows in maximum of 100 target members, the exponential histogram metric are split into multiple data points as needed, +// each containing a maximum of 100 buckets, to comply with CloudWatch EMF log constraints. +// Note that the number of values and counts in each split may not be less than splitThreshold as we are only adding non-zero bucket counts. +// +// For each split data point: +// - Min and Max values are recalculated based on the bucket boundary within that specific split. +// - Sum is only assigned to the first split to ensure the total sum of the datapoints after aggregation is correct. +// - Count is accumulated based on the bucket counts within each split. func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, instrumentationScopeName string, _ bool, _ *emfCalculators) ([]dataPoint, bool) { metric := dps.ExponentialHistogramDataPointSlice.At(idx) + const splitThreshold = 100 + var currentBucketIndex = 0 + var datapoints []dataPoint + var currentPositiveIndex = metric.Positive().BucketCounts().Len() - 1 + var currentZeroIndex = 0 + var currentNegativeIndex = 0 + totalBucketLen := metric.Positive().BucketCounts().Len() + metric.Negative().BucketCounts().Len() + if metric.ZeroCount() > 0 { + totalBucketLen++ + } + + if totalBucketLen == 0 { + return []dataPoint{{ + name: dps.metricName, + value: &cWMetricHistogram{ + Values: []float64{}, + Counts: []float64{}, + Count: metric.Count(), + Sum: metric.Sum(), + Max: metric.Max(), + Min: metric.Min(), + }, + labels: createLabels(metric.Attributes(), instrumentationScopeName), + timestampMs: unixNanoToMilliseconds(metric.Timestamp()), + }}, true + } + + for currentBucketIndex < totalBucketLen { + // Create a new dataPointSplit with a capacity of up to splitThreshold buckets + split := dataPointSplit{ + cWMetricHistogram: &cWMetricHistogram{ + Values: []float64{}, + Counts: []float64{}, + Max: metric.Max(), + Min: metric.Min(), + Count: 0, + Sum: 0, + }, + length: 0, + capacity: splitThreshold, + } + + // Only assign `Sum` if this is the first split to make sure the total sum of the datapoints after aggregation is correct. + if currentBucketIndex == 0 { + split.cWMetricHistogram.Sum = metric.Sum() + } + + if totalBucketLen-currentBucketIndex < splitThreshold { + split.capacity = totalBucketLen - currentBucketIndex + } + + // Set mid-point of positive buckets in values/counts array. + currentBucketIndex, currentPositiveIndex = iteratePositiveBuckets(&split, metric, currentBucketIndex, currentPositiveIndex, totalBucketLen) + // Set count of zero bucket in values/counts array. + currentBucketIndex, currentZeroIndex = iterateZeroBucket(&split, metric, currentBucketIndex, currentZeroIndex, totalBucketLen) + // Set mid-point of negative buckets in values/counts array. + currentBucketIndex, currentNegativeIndex = iterateNegativeBuckets(&split, metric, currentBucketIndex, currentNegativeIndex, totalBucketLen) + + // Add the current split to the datapoints list + datapoints = append(datapoints, dataPoint{ + name: dps.metricName, + value: split.cWMetricHistogram, + labels: createLabels(metric.Attributes(), instrumentationScopeName), + timestampMs: unixNanoToMilliseconds(metric.Timestamp()), + }) + } + return datapoints, true +} + +func iteratePositiveBuckets(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentPositiveIndex int, totalBucketLen int) (int, int) { scale := metric.Scale() base := math.Pow(2, math.Pow(2, float64(-scale))) - arrayValues := []float64{} - arrayCounts := []float64{} - var bucketBegin float64 - var bucketEnd float64 - - // Set mid-point of positive buckets in values/counts array. positiveBuckets := metric.Positive() positiveOffset := positiveBuckets.Offset() positiveBucketCounts := positiveBuckets.BucketCounts() - bucketBegin = 0 - bucketEnd = 0 - for i := 0; i < positiveBucketCounts.Len(); i++ { - index := i + int(positiveOffset) - if bucketBegin == 0 { - bucketBegin = math.Pow(base, float64(index)) + bucketBegin := 0.0 + bucketEnd := 0.0 + + for split.length < split.capacity && currentPositiveIndex >= 0 { + index := currentPositiveIndex + int(positiveOffset) + if bucketEnd == 0 { + bucketEnd = math.Pow(base, float64(index+1)) } else { - bucketBegin = bucketEnd + bucketEnd = bucketBegin } - bucketEnd = math.Pow(base, float64(index+1)) + bucketBegin = math.Pow(base, float64(index)) metricVal := (bucketBegin + bucketEnd) / 2 - count := positiveBucketCounts.At(i) + count := positiveBucketCounts.At(currentPositiveIndex) if count > 0 { - arrayValues = append(arrayValues, metricVal) - arrayCounts = append(arrayCounts, float64(count)) + split.cWMetricHistogram.Values = append(split.cWMetricHistogram.Values, metricVal) + split.cWMetricHistogram.Counts = append(split.cWMetricHistogram.Counts, float64(count)) + split.length++ + split.cWMetricHistogram.Count += count + if split.length == 1 && currentBucketIndex != 0 { + split.cWMetricHistogram.Max = bucketEnd + } + if split.length == split.capacity && currentBucketIndex != totalBucketLen-1 { + split.cWMetricHistogram.Min = bucketBegin + } } + currentBucketIndex++ + currentPositiveIndex-- } - // Set count of zero bucket in values/counts array. - if metric.ZeroCount() > 0 { - arrayValues = append(arrayValues, 0) - arrayCounts = append(arrayCounts, float64(metric.ZeroCount())) + return currentBucketIndex, currentPositiveIndex +} + +func iterateZeroBucket(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentZeroIndex int, totalBucketLen int) (int, int) { + if metric.ZeroCount() > 0 && split.length < split.capacity && currentZeroIndex == 0 { + split.cWMetricHistogram.Values = append(split.cWMetricHistogram.Values, 0) + split.cWMetricHistogram.Counts = append(split.cWMetricHistogram.Counts, float64(metric.ZeroCount())) + split.length++ + split.cWMetricHistogram.Count += metric.ZeroCount() + if split.length == 1 && currentBucketIndex != 0 { + split.cWMetricHistogram.Max = 0 + } + if split.length == split.capacity && currentBucketIndex != totalBucketLen-1 { + split.cWMetricHistogram.Min = 0 + } + currentZeroIndex++ + currentBucketIndex++ } - // Set mid-point of negative buckets in values/counts array. + return currentBucketIndex, currentZeroIndex +} + +func iterateNegativeBuckets(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentNegativeIndex int, totalBucketLen int) (int, int) { // According to metrics spec, the value in histogram is expected to be non-negative. // https://opentelemetry.io/docs/specs/otel/metrics/api/#histogram // However, the negative support is defined in metrics data model. // https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exponentialhistogram // The negative is also supported but only verified with unit test. - + scale := metric.Scale() + base := math.Pow(2, math.Pow(2, float64(-scale))) negativeBuckets := metric.Negative() negativeOffset := negativeBuckets.Offset() negativeBucketCounts := negativeBuckets.BucketCounts() - bucketBegin = 0 - bucketEnd = 0 - for i := 0; i < negativeBucketCounts.Len(); i++ { - index := i + int(negativeOffset) + bucketBegin := 0.0 + bucketEnd := 0.0 + + for split.length < split.capacity && currentNegativeIndex < metric.Negative().BucketCounts().Len() { + index := currentNegativeIndex + int(negativeOffset) if bucketEnd == 0 { bucketEnd = -math.Pow(base, float64(index)) } else { @@ -252,26 +359,24 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, } bucketBegin = -math.Pow(base, float64(index+1)) metricVal := (bucketBegin + bucketEnd) / 2 - count := negativeBucketCounts.At(i) + count := negativeBucketCounts.At(currentNegativeIndex) if count > 0 { - arrayValues = append(arrayValues, metricVal) - arrayCounts = append(arrayCounts, float64(count)) + split.cWMetricHistogram.Values = append(split.cWMetricHistogram.Values, metricVal) + split.cWMetricHistogram.Counts = append(split.cWMetricHistogram.Counts, float64(count)) + split.length++ + split.cWMetricHistogram.Count += count + if split.length == 1 && currentBucketIndex != 0 { + split.cWMetricHistogram.Max = bucketEnd + } + if split.length == split.capacity && currentBucketIndex != totalBucketLen-1 { + split.cWMetricHistogram.Min = bucketBegin + } } + currentBucketIndex++ + currentNegativeIndex++ } - return []dataPoint{{ - name: dps.metricName, - value: &cWMetricHistogram{ - Values: arrayValues, - Counts: arrayCounts, - Count: metric.Count(), - Sum: metric.Sum(), - Max: metric.Max(), - Min: metric.Min(), - }, - labels: createLabels(metric.Attributes(), instrumentationScopeName), - timestampMs: unixNanoToMilliseconds(metric.Timestamp()), - }}, true + return currentBucketIndex, currentNegativeIndex } func (dps exponentialHistogramDataPointSlice) IsStaleNaNInf(i int) (bool, pcommon.Map) { diff --git a/exporter/awsemfexporter/datapoint_test.go b/exporter/awsemfexporter/datapoint_test.go index 597f1d4b091c..a3004608b0c8 100644 --- a/exporter/awsemfexporter/datapoint_test.go +++ b/exporter/awsemfexporter/datapoint_test.go @@ -244,6 +244,35 @@ func generateTestExponentialHistogramMetricWithInfs(name string) pmetric.Metrics return otelMetrics } +func generateTestExponentialHistogramMetricWithLongBuckets(name string) pmetric.Metrics { + otelMetrics := pmetric.NewMetrics() + rs := otelMetrics.ResourceMetrics().AppendEmpty() + metrics := rs.ScopeMetrics().AppendEmpty().Metrics() + metric := metrics.AppendEmpty() + metric.SetName(name) + metric.SetUnit("Seconds") + exponentialHistogramMetric := metric.SetEmptyExponentialHistogram() + + exponentialHistogramDatapoint := exponentialHistogramMetric.DataPoints().AppendEmpty() + exponentialHistogramDatapoint.SetCount(3662) + exponentialHistogramDatapoint.SetSum(1000) + exponentialHistogramDatapoint.SetMin(-9e+17) + exponentialHistogramDatapoint.SetMax(9e+17) + exponentialHistogramDatapoint.SetZeroCount(2) + posBucketCounts := make([]uint64, 60) + for i := range posBucketCounts { + posBucketCounts[i] = uint64(i + 1) + } + exponentialHistogramDatapoint.Positive().BucketCounts().FromRaw(posBucketCounts) + negBucketCounts := make([]uint64, 60) + for i := range negBucketCounts { + negBucketCounts[i] = uint64(i + 1) + } + exponentialHistogramDatapoint.Negative().BucketCounts().FromRaw(negBucketCounts) + exponentialHistogramDatapoint.Attributes().PutStr("label1", "value1") + return otelMetrics +} + func generateTestSummaryMetric(name string) pmetric.Metrics { otelMetrics := pmetric.NewMetrics() rs := otelMetrics.ResourceMetrics().AppendEmpty() @@ -850,7 +879,7 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSlice(t *testing. }(), expectedDatapoint: dataPoint{ name: "foo", - value: &cWMetricHistogram{Values: []float64{1.5, 3, 6, 0, -1.5, -3, -6}, Counts: []float64{1, 2, 3, 4, 1, 2, 3}}, + value: &cWMetricHistogram{Values: []float64{6, 3, 1.5, 0, -1.5, -3, -6}, Counts: []float64{3, 2, 1, 4, 1, 2, 3}, Count: 16}, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, }, @@ -871,7 +900,7 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSlice(t *testing. }(), expectedDatapoint: dataPoint{ name: "foo", - value: &cWMetricHistogram{Values: []float64{0.625, 2.5, 10, 0, -0.625, -2.5, -10}, Counts: []float64{1, 2, 3, 4, 1, 2, 3}}, + value: &cWMetricHistogram{Values: []float64{10, 2.5, 0.625, 0, -0.625, -2.5, -10}, Counts: []float64{3, 2, 1, 4, 1, 2, 3}, Count: 16}, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1", "label2": "value2"}, }, }, @@ -895,6 +924,449 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSlice(t *testing. } +func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDataPoints(t *testing.T) { + dmd := generateDeltaMetricMetadata(false, "foo", false) + + testCases := []struct { + name string + histogramDPS pmetric.ExponentialHistogramDataPointSlice + expectedDatapoints []dataPoint + }{ + { + name: "Exponential histogram with more than 100 buckets, including positive, negative and zero buckets", + histogramDPS: func() pmetric.ExponentialHistogramDataPointSlice { + histogramDPS := pmetric.NewExponentialHistogramDataPointSlice() + histogramDP := histogramDPS.AppendEmpty() + posBucketCounts := make([]uint64, 60) + for i := range posBucketCounts { + posBucketCounts[i] = uint64(i + 1) + } + histogramDP.Positive().BucketCounts().FromRaw(posBucketCounts) + histogramDP.SetZeroCount(2) + negBucketCounts := make([]uint64, 60) + for i := range negBucketCounts { + negBucketCounts[i] = uint64(i + 1) + } + histogramDP.Negative().BucketCounts().FromRaw(negBucketCounts) + histogramDP.SetSum(1000) + histogramDP.SetMin(-9e+17) + histogramDP.SetMax(9e+17) + histogramDP.SetCount(uint64(3662)) + histogramDP.Attributes().PutStr("label1", "value1") + return histogramDPS + }(), + expectedDatapoints: []dataPoint{ + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, 2.7021597764222976e+16, + 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 1.688849860263936e+15, 8.44424930131968e+14, 4.22212465065984e+14, + 2.11106232532992e+14, 1.05553116266496e+14, 5.2776558133248e+13, 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, + 1.649267441664e+12, 8.24633720832e+11, 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, + 1.2884901888e+10, 6.442450944e+09, 3.221225472e+09, 1.610612736e+09, 8.05306368e+08, 4.02653184e+08, 2.01326592e+08, 1.00663296e+08, + 5.0331648e+07, 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06, 786432, 393216, 196608, 98304, 49152, 24576, + 12288, 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5, 0, -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, -3072, + -6144, -12288, -24576, -49152, -98304, -196608, -393216, -786432, -1.572864e+06, -3.145728e+06, -6.291456e+06, -1.2582912e+07, -2.5165824e+07, + -5.0331648e+07, -1.00663296e+08, -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, -1.610612736e+09, -3.221225472e+09, -6.442450944e+09, + -1.2884901888e+10, -2.5769803776e+10, -5.1539607552e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11}, + Counts: []float64{60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, + 6, 5, 4, 3, 2, 1, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39}, + Sum: 1000, Count: 2612, Min: -5.49755813888e+11, Max: 9e+17}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{-8.24633720832e+11, -1.649267441664e+12, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -5.2776558133248e+13, + -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, -1.688849860263936e+15, -3.377699720527872e+15, + -6.755399441055744e+15, -1.3510798882111488e+16, -2.7021597764222976e+16, -5.404319552844595e+16, -1.080863910568919e+17, -2.161727821137838e+17, + -4.323455642275676e+17, -8.646911284551352e+17}, + Counts: []float64{40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60}, + Sum: 0, Count: 1050, Min: -9e+17, Max: -5.49755813888e+11}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + }, + }, + { + name: "Exponential histogram with more than 100 buckets, including positive and zero buckets", + histogramDPS: func() pmetric.ExponentialHistogramDataPointSlice { + histogramDPS := pmetric.NewExponentialHistogramDataPointSlice() + histogramDP := histogramDPS.AppendEmpty() + posBucketCounts := make([]uint64, 120) + for i := range posBucketCounts { + posBucketCounts[i] = uint64(i + 1) + } + histogramDP.Positive().BucketCounts().FromRaw(posBucketCounts) + histogramDP.SetZeroCount(2) + histogramDP.SetSum(10000) + histogramDP.SetMin(0) + histogramDP.SetMax(9e+36) + histogramDP.SetCount(uint64(7262)) + histogramDP.Attributes().PutStr("label1", "value1") + return histogramDPS + }(), + expectedDatapoints: []dataPoint{ + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{9.969209968386869e+35, 4.9846049841934345e+35, 2.4923024920967173e+35, 1.2461512460483586e+35, 6.230756230241793e+34, + 3.1153781151208966e+34, 1.5576890575604483e+34, 7.788445287802241e+33, 3.894222643901121e+33, 1.9471113219505604e+33, + 9.735556609752802e+32, 4.867778304876401e+32, 2.4338891524382005e+32, 1.2169445762191002e+32, 6.084722881095501e+31, + 3.0423614405477506e+31, 1.5211807202738753e+31, 7.605903601369376e+30, 3.802951800684688e+30, 1.901475900342344e+30, + 9.50737950171172e+29, 4.75368975085586e+29, 2.37684487542793e+29, 1.188422437713965e+29, 5.942112188569825e+28, + 2.9710560942849127e+28, 1.4855280471424563e+28, 7.427640235712282e+27, 3.713820117856141e+27, 1.8569100589280704e+27, + 9.284550294640352e+26, 4.642275147320176e+26, 2.321137573660088e+26, 1.160568786830044e+26, 5.80284393415022e+25, + 2.90142196707511e+25, 1.450710983537555e+25, 7.253554917687775e+24, 3.6267774588438875e+24, 1.8133887294219438e+24, + 9.066943647109719e+23, 4.5334718235548594e+23, 2.2667359117774297e+23, 1.1333679558887149e+23, 5.666839779443574e+22, + 2.833419889721787e+22, 1.4167099448608936e+22, 7.083549724304468e+21, 3.541774862152234e+21, 1.770887431076117e+21, + 8.854437155380585e+20, 4.4272185776902924e+20, 2.2136092888451462e+20, 1.1068046444225731e+20, 5.5340232221128655e+19, + 2.7670116110564327e+19, 1.3835058055282164e+19, 6.917529027641082e+18, 3.458764513820541e+18, 1.7293822569102705e+18, + 8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, + 2.7021597764222976e+16, 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 1.688849860263936e+15, + 8.44424930131968e+14, 4.22212465065984e+14, 2.11106232532992e+14, 1.05553116266496e+14, 5.2776558133248e+13, + 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 1.649267441664e+12, 8.24633720832e+11, + 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, 1.2884901888e+10, + 6.442450944e+09, 3.221225472e+09, 1.610612736e+09, 8.05306368e+08, 4.02653184e+08, 2.01326592e+08, 1.00663296e+08, + 5.0331648e+07, 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06}, + Counts: []float64{120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 99, + 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, + 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, + 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21}, + Sum: 10000, Count: 7050, Min: 1.048576e+06, Max: 9e+36}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{786432, 393216, 196608, 98304, 49152, 24576, 12288, 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5, 0}, + Counts: []float64{20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 2}, + Sum: 0, Count: 212, Min: 0, Max: 1.048576e+06}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + }, + }, + { + name: "Exponential histogram with more than 100 buckets, including negative and zero buckets", + histogramDPS: func() pmetric.ExponentialHistogramDataPointSlice { + histogramDPS := pmetric.NewExponentialHistogramDataPointSlice() + histogramDP := histogramDPS.AppendEmpty() + negBucketCounts := make([]uint64, 120) + for i := range negBucketCounts { + negBucketCounts[i] = uint64(i + 1) + } + histogramDP.Negative().BucketCounts().FromRaw(negBucketCounts) + histogramDP.SetZeroCount(2) + histogramDP.SetSum(10000) + histogramDP.SetMin(-9e+36) + histogramDP.SetMax(0) + histogramDP.SetCount(uint64(7262)) + histogramDP.Attributes().PutStr("label1", "value1") + return histogramDPS + }(), + expectedDatapoints: []dataPoint{ + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{0, -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, -3072, -6144, -12288, -24576, + -49152, -98304, -196608, -393216, -786432, -1.572864e+06, -3.145728e+06, -6.291456e+06, -1.2582912e+07, + -2.5165824e+07, -5.0331648e+07, -1.00663296e+08, -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, + -1.610612736e+09, -3.221225472e+09, -6.442450944e+09, -1.2884901888e+10, -2.5769803776e+10, + -5.1539607552e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, -8.24633720832e+11, + -1.649267441664e+12, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, + -5.2776558133248e+13, -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, + -1.688849860263936e+15, -3.377699720527872e+15, -6.755399441055744e+15, -1.3510798882111488e+16, + -2.7021597764222976e+16, -5.404319552844595e+16, -1.080863910568919e+17, -2.161727821137838e+17, + -4.323455642275676e+17, -8.646911284551352e+17, -1.7293822569102705e+18, -3.458764513820541e+18, + -6.917529027641082e+18, -1.3835058055282164e+19, -2.7670116110564327e+19, -5.5340232221128655e+19, + -1.1068046444225731e+20, -2.2136092888451462e+20, -4.4272185776902924e+20, -8.854437155380585e+20, + -1.770887431076117e+21, -3.541774862152234e+21, -7.083549724304468e+21, -1.4167099448608936e+22, + -2.833419889721787e+22, -5.666839779443574e+22, -1.1333679558887149e+23, -2.2667359117774297e+23, + -4.5334718235548594e+23, -9.066943647109719e+23, -1.8133887294219438e+24, -3.6267774588438875e+24, + -7.253554917687775e+24, -1.450710983537555e+25, -2.90142196707511e+25, -5.80284393415022e+25, + -1.160568786830044e+26, -2.321137573660088e+26, -4.642275147320176e+26, -9.284550294640352e+26, + -1.8569100589280704e+27, -3.713820117856141e+27, -7.427640235712282e+27, -1.4855280471424563e+28, + -2.9710560942849127e+28, -5.942112188569825e+28, -1.188422437713965e+29, -2.37684487542793e+29, -4.75368975085586e+29}, + Counts: []float64{2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99}, + Sum: 10000, Count: 4952, Min: -6.338253001141147e+29, Max: 0}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{-9.50737950171172e+29, -1.901475900342344e+30, -3.802951800684688e+30, -7.605903601369376e+30, + -1.5211807202738753e+31, -3.0423614405477506e+31, -6.084722881095501e+31, -1.2169445762191002e+32, + -2.4338891524382005e+32, -4.867778304876401e+32, -9.735556609752802e+32, -1.9471113219505604e+33, -3.894222643901121e+33, + -7.788445287802241e+33, -1.5576890575604483e+34, -3.1153781151208966e+34, -6.230756230241793e+34, -1.2461512460483586e+35, + -2.4923024920967173e+35, -4.9846049841934345e+35, -9.969209968386869e+35}, + Counts: []float64{100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120}, + Sum: 0, Count: 2310, Min: -9e+36, Max: -6.338253001141147e+29}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + }, + }, + { + name: "Exponential histogram with more than 100 buckets, including positive and negative buckets", + histogramDPS: func() pmetric.ExponentialHistogramDataPointSlice { + histogramDPS := pmetric.NewExponentialHistogramDataPointSlice() + histogramDP := histogramDPS.AppendEmpty() + posBucketCounts := make([]uint64, 60) + for i := range posBucketCounts { + posBucketCounts[i] = uint64(i + 1) + } + histogramDP.Positive().BucketCounts().FromRaw(posBucketCounts) + negBucketCounts := make([]uint64, 60) + for i := range negBucketCounts { + negBucketCounts[i] = uint64(i + 1) + } + histogramDP.Negative().BucketCounts().FromRaw(negBucketCounts) + histogramDP.SetSum(1000) + histogramDP.SetMin(-9e+17) + histogramDP.SetMax(9e+17) + histogramDP.SetCount(uint64(3660)) + histogramDP.Attributes().PutStr("label1", "value1") + return histogramDPS + }(), + expectedDatapoints: []dataPoint{ + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, 2.7021597764222976e+16, + 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 1.688849860263936e+15, 8.44424930131968e+14, 4.22212465065984e+14, + 2.11106232532992e+14, 1.05553116266496e+14, 5.2776558133248e+13, 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, + 1.649267441664e+12, 8.24633720832e+11, 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, + 1.2884901888e+10, 6.442450944e+09, 3.221225472e+09, 1.610612736e+09, 8.05306368e+08, 4.02653184e+08, 2.01326592e+08, 1.00663296e+08, + 5.0331648e+07, 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06, 786432, 393216, 196608, 98304, 49152, 24576, + 12288, 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5, -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, -3072, + -6144, -12288, -24576, -49152, -98304, -196608, -393216, -786432, -1.572864e+06, -3.145728e+06, -6.291456e+06, -1.2582912e+07, -2.5165824e+07, + -5.0331648e+07, -1.00663296e+08, -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, -1.610612736e+09, -3.221225472e+09, -6.442450944e+09, + -1.2884901888e+10, -2.5769803776e+10, -5.1539607552e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, -8.24633720832e+11}, + Counts: []float64{60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, + 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40}, + Sum: 1000, Count: 2650, Min: -1.099511627776e+12, Max: 9e+17}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{-1.649267441664e+12, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -5.2776558133248e+13, + -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, -1.688849860263936e+15, -3.377699720527872e+15, + -6.755399441055744e+15, -1.3510798882111488e+16, -2.7021597764222976e+16, -5.404319552844595e+16, -1.080863910568919e+17, -2.161727821137838e+17, + -4.323455642275676e+17, -8.646911284551352e+17}, + Counts: []float64{41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60}, + Sum: 0, Count: 1010, Min: -9e+17, Max: -1.099511627776e+12}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + }, + }, + { + name: "Exponential histogram with exact 200 buckets, including positive, negative buckets", + histogramDPS: func() pmetric.ExponentialHistogramDataPointSlice { + histogramDPS := pmetric.NewExponentialHistogramDataPointSlice() + histogramDP := histogramDPS.AppendEmpty() + posBucketCounts := make([]uint64, 100) + for i := range posBucketCounts { + posBucketCounts[i] = uint64(i + 1) + } + histogramDP.Positive().BucketCounts().FromRaw(posBucketCounts) + negBucketCounts := make([]uint64, 100) + for i := range negBucketCounts { + negBucketCounts[i] = uint64(i + 1) + } + histogramDP.Negative().BucketCounts().FromRaw(negBucketCounts) + histogramDP.SetSum(100000) + histogramDP.SetMin(-9e+36) + histogramDP.SetMax(9e+36) + histogramDP.SetCount(uint64(3662)) + histogramDP.Attributes().PutStr("label1", "value1") + return histogramDPS + }(), + expectedDatapoints: []dataPoint{ + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{ + 9.50737950171172e+29, 4.75368975085586e+29, 2.37684487542793e+29, 1.188422437713965e+29, 5.942112188569825e+28, + 2.9710560942849127e+28, 1.4855280471424563e+28, 7.427640235712282e+27, 3.713820117856141e+27, 1.8569100589280704e+27, + 9.284550294640352e+26, 4.642275147320176e+26, 2.321137573660088e+26, 1.160568786830044e+26, 5.80284393415022e+25, + 2.90142196707511e+25, 1.450710983537555e+25, 7.253554917687775e+24, 3.6267774588438875e+24, 1.8133887294219438e+24, + 9.066943647109719e+23, 4.5334718235548594e+23, 2.2667359117774297e+23, 1.1333679558887149e+23, 5.666839779443574e+22, + 2.833419889721787e+22, 1.4167099448608936e+22, 7.083549724304468e+21, 3.541774862152234e+21, 1.770887431076117e+21, + 8.854437155380585e+20, 4.4272185776902924e+20, 2.2136092888451462e+20, 1.1068046444225731e+20, 5.5340232221128655e+19, + 2.7670116110564327e+19, 1.3835058055282164e+19, 6.917529027641082e+18, 3.458764513820541e+18, 1.7293822569102705e+18, + 8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, + 2.7021597764222976e+16, 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 1.688849860263936e+15, + 8.44424930131968e+14, 4.22212465065984e+14, 2.11106232532992e+14, 1.05553116266496e+14, 5.2776558133248e+13, + 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 1.649267441664e+12, 8.24633720832e+11, + 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, 1.2884901888e+10, 6.442450944e+09, + 3.221225472e+09, 1.610612736e+09, 8.05306368e+08, 4.02653184e+08, 2.01326592e+08, 1.00663296e+08, 5.0331648e+07, + 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06, 786432, 393216, 196608, 98304, 49152, 24576, 12288, + 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5}, + Counts: []float64{ + 100, 99, 98, 97, 96, 95, 94, + 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, + 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}, + Sum: 100000, Count: 5050, Min: 1, Max: 9e+36}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{ + -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, -3072, -6144, -12288, -24576, -49152, -98304, -196608, -393216, + -786432, -1.572864e+06, -3.145728e+06, -6.291456e+06, -1.2582912e+07, -2.5165824e+07, -5.0331648e+07, -1.00663296e+08, + -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, -1.610612736e+09, -3.221225472e+09, -6.442450944e+09, -1.2884901888e+10, + -2.5769803776e+10, -5.1539607552e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, -8.24633720832e+11, + -1.649267441664e+12, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -5.2776558133248e+13, + -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, -1.688849860263936e+15, + -3.377699720527872e+15, -6.755399441055744e+15, -1.3510798882111488e+16, -2.7021597764222976e+16, + -5.404319552844595e+16, -1.080863910568919e+17, -2.161727821137838e+17, -4.323455642275676e+17, -8.646911284551352e+17, + -1.7293822569102705e+18, -3.458764513820541e+18, -6.917529027641082e+18, -1.3835058055282164e+19, -2.7670116110564327e+19, + -5.5340232221128655e+19, -1.1068046444225731e+20, -2.2136092888451462e+20, -4.4272185776902924e+20, -8.854437155380585e+20, + -1.770887431076117e+21, -3.541774862152234e+21, -7.083549724304468e+21, -1.4167099448608936e+22, -2.833419889721787e+22, + -5.666839779443574e+22, -1.1333679558887149e+23, -2.2667359117774297e+23, -4.5334718235548594e+23, -9.066943647109719e+23, + -1.8133887294219438e+24, -3.6267774588438875e+24, -7.253554917687775e+24, -1.450710983537555e+25, -2.90142196707511e+25, + -5.80284393415022e+25, -1.160568786830044e+26, -2.321137573660088e+26, -4.642275147320176e+26, -9.284550294640352e+26, + -1.8569100589280704e+27, -3.713820117856141e+27, -7.427640235712282e+27, -1.4855280471424563e+28, -2.9710560942849127e+28, + -5.942112188569825e+28, -1.188422437713965e+29, -2.37684487542793e+29, -4.75368975085586e+29, -9.50737950171172e+29}, + Counts: []float64{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, + 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100}, + Sum: 0, Count: 5050, Min: -9e+36, Max: -1}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + }, + }, + { + name: "Exponential histogram with more than 200 buckets, including positive, negative and zero buckets", + histogramDPS: func() pmetric.ExponentialHistogramDataPointSlice { + histogramDPS := pmetric.NewExponentialHistogramDataPointSlice() + histogramDP := histogramDPS.AppendEmpty() + posBucketCounts := make([]uint64, 120) + for i := range posBucketCounts { + posBucketCounts[i] = uint64(i + 1) + } + histogramDP.Positive().BucketCounts().FromRaw(posBucketCounts) + histogramDP.SetZeroCount(2) + negBucketCounts := make([]uint64, 120) + for i := range negBucketCounts { + negBucketCounts[i] = uint64(i + 1) + } + histogramDP.Negative().BucketCounts().FromRaw(negBucketCounts) + histogramDP.SetSum(100000) + histogramDP.SetMin(-9e+36) + histogramDP.SetMax(9e+36) + histogramDP.SetCount(uint64(3662)) + histogramDP.Attributes().PutStr("label1", "value1") + return histogramDPS + }(), + expectedDatapoints: []dataPoint{ + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{ + 9.969209968386869e+35, 4.9846049841934345e+35, 2.4923024920967173e+35, 1.2461512460483586e+35, 6.230756230241793e+34, + 3.1153781151208966e+34, 1.5576890575604483e+34, 7.788445287802241e+33, 3.894222643901121e+33, 1.9471113219505604e+33, + 9.735556609752802e+32, 4.867778304876401e+32, 2.4338891524382005e+32, 1.2169445762191002e+32, 6.084722881095501e+31, + 3.0423614405477506e+31, 1.5211807202738753e+31, 7.605903601369376e+30, 3.802951800684688e+30, 1.901475900342344e+30, + 9.50737950171172e+29, 4.75368975085586e+29, 2.37684487542793e+29, 1.188422437713965e+29, 5.942112188569825e+28, + 2.9710560942849127e+28, 1.4855280471424563e+28, 7.427640235712282e+27, 3.713820117856141e+27, 1.8569100589280704e+27, + 9.284550294640352e+26, 4.642275147320176e+26, 2.321137573660088e+26, 1.160568786830044e+26, 5.80284393415022e+25, + 2.90142196707511e+25, 1.450710983537555e+25, 7.253554917687775e+24, 3.6267774588438875e+24, 1.8133887294219438e+24, + 9.066943647109719e+23, 4.5334718235548594e+23, 2.2667359117774297e+23, 1.1333679558887149e+23, 5.666839779443574e+22, + 2.833419889721787e+22, 1.4167099448608936e+22, 7.083549724304468e+21, 3.541774862152234e+21, 1.770887431076117e+21, + 8.854437155380585e+20, 4.4272185776902924e+20, 2.2136092888451462e+20, 1.1068046444225731e+20, 5.5340232221128655e+19, + 2.7670116110564327e+19, 1.3835058055282164e+19, 6.917529027641082e+18, 3.458764513820541e+18, 1.7293822569102705e+18, + 8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, + 2.7021597764222976e+16, 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 1.688849860263936e+15, + 8.44424930131968e+14, 4.22212465065984e+14, 2.11106232532992e+14, 1.05553116266496e+14, 5.2776558133248e+13, + 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 1.649267441664e+12, 8.24633720832e+11, + 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, 1.2884901888e+10, + 6.442450944e+09, 3.221225472e+09, 1.610612736e+09, 8.05306368e+08, 4.02653184e+08, 2.01326592e+08, 1.00663296e+08, 5.0331648e+07, + 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06}, + Counts: []float64{ + 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 99, 98, 97, 96, 95, 94, + 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, + 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, + 27, 26, 25, 24, 23, 22, 21}, + Sum: 100000, Count: 7050, Min: 1048576, Max: 9e+36}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{ + 786432, 393216, 196608, 98304, 49152, 24576, 12288, 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, + 12, 6, 3, 1.5, 0, -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, + -3072, -6144, -12288, -24576, -49152, -98304, -196608, -393216, -786432, -1.572864e+06, -3.145728e+06, -6.291456e+06, + -1.2582912e+07, -2.5165824e+07, -5.0331648e+07, -1.00663296e+08, -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, + -1.610612736e+09, -3.221225472e+09, -6.442450944e+09, -1.2884901888e+10, -2.5769803776e+10, -5.1539607552e+10, + -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, -8.24633720832e+11, -1.649267441664e+12, + -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -5.2776558133248e+13, + -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, + -1.688849860263936e+15, -3.377699720527872e+15, -6.755399441055744e+15, -1.3510798882111488e+16, + -2.7021597764222976e+16, -5.404319552844595e+16, -1.080863910568919e+17, -2.161727821137838e+17, + -4.323455642275676e+17, -8.646911284551352e+17, -1.7293822569102705e+18, -3.458764513820541e+18, + -6.917529027641082e+18, -1.3835058055282164e+19, -2.7670116110564327e+19, -5.5340232221128655e+19, + -1.1068046444225731e+20, -2.2136092888451462e+20, -4.4272185776902924e+20, -8.854437155380585e+20, + -1.770887431076117e+21, -3.541774862152234e+21, -7.083549724304468e+21, -1.4167099448608936e+22, + -2.833419889721787e+22, -5.666839779443574e+22, -1.1333679558887149e+23, -2.2667359117774297e+23, + -4.5334718235548594e+23}, + Counts: []float64{ + 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79}, + Sum: 0, Count: 3372, Min: -6.044629098073146e+23, Max: 1048576}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{ + -9.066943647109719e+23, -1.8133887294219438e+24, -3.6267774588438875e+24, -7.253554917687775e+24, -1.450710983537555e+25, + -2.90142196707511e+25, -5.80284393415022e+25, -1.160568786830044e+26, -2.321137573660088e+26, -4.642275147320176e+26, + -9.284550294640352e+26, -1.8569100589280704e+27, -3.713820117856141e+27, -7.427640235712282e+27, -1.4855280471424563e+28, + -2.9710560942849127e+28, -5.942112188569825e+28, -1.188422437713965e+29, -2.37684487542793e+29, -4.75368975085586e+29, + -9.50737950171172e+29, -1.901475900342344e+30, -3.802951800684688e+30, -7.605903601369376e+30, -1.5211807202738753e+31, + -3.0423614405477506e+31, -6.084722881095501e+31, -1.2169445762191002e+32, -2.4338891524382005e+32, -4.867778304876401e+32, + -9.735556609752802e+32, -1.9471113219505604e+33, -3.894222643901121e+33, -7.788445287802241e+33, -1.5576890575604483e+34, + -3.1153781151208966e+34, -6.230756230241793e+34, -1.2461512460483586e+35, -2.4923024920967173e+35, -4.9846049841934345e+35, + -9.969209968386869e+35}, + Counts: []float64{ + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120}, + Sum: 0, Count: 4100, Min: -9e+36, Max: -6.044629098073146e+23}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(_ *testing.T) { + exponentialHistogramDatapointSlice := exponentialHistogramDataPointSlice{dmd, tc.histogramDPS} + emfCalcs := setupEmfCalculators() + defer require.NoError(t, shutdownEmfCalculators(emfCalcs)) + dps, retained := exponentialHistogramDatapointSlice.CalculateDeltaDatapoints(0, instrLibName, false, emfCalcs) + + assert.True(t, retained) + assert.Equal(t, 1, exponentialHistogramDatapointSlice.Len()) + assert.Equal(t, len(tc.expectedDatapoints), len(dps)) + for i, expectedDP := range tc.expectedDatapoints { + assert.Equal(t, expectedDP, dps[i], "datapoint mismatch at index %d", i) + } + }) + } + +} + func TestIsStaleNaNInf_ExponentialHistogramDataPointSlice(t *testing.T) { testCases := []struct { diff --git a/exporter/awsemfexporter/grouped_metric.go b/exporter/awsemfexporter/grouped_metric.go index dd3426cdc2e9..bd896e9c4194 100644 --- a/exporter/awsemfexporter/grouped_metric.go +++ b/exporter/awsemfexporter/grouped_metric.go @@ -57,7 +57,7 @@ func addToGroupedMetric( continue } - for _, dp := range dps { + for i, dp := range dps { labels := dp.labels if metricType, ok := labels["Type"]; ok { @@ -87,6 +87,7 @@ func addToGroupedMetric( } // Extra params to use when grouping metrics + metadata.groupedMetricMetadata.batchIndex = i groupKey := aws.NewKey(metadata.groupedMetricMetadata, labels) if _, ok := groupedMetrics[groupKey]; ok { // if MetricName already exists in metrics map, print warning log diff --git a/exporter/awsemfexporter/grouped_metric_test.go b/exporter/awsemfexporter/grouped_metric_test.go index d2b1a86610f0..6e09d82b7a3e 100644 --- a/exporter/awsemfexporter/grouped_metric_test.go +++ b/exporter/awsemfexporter/grouped_metric_test.go @@ -406,6 +406,43 @@ func TestAddToGroupedMetric(t *testing.T) { assert.Equal(t, expectedLogs, logs.AllUntimed()) }) + t.Run("Duplicate metric names with different metricIndex", func(t *testing.T) { + emfCalcs := setupEmfCalculators() + defer require.NoError(t, shutdownEmfCalculators(emfCalcs)) + groupedMetrics := make(map[any]*groupedMetric) + generateMetrics := []pmetric.Metrics{ + generateTestExponentialHistogramMetricWithLongBuckets("test_multiBucket_metric"), + } + finalOtelMetrics := generateOtelTestMetrics(generateMetrics...) + + rms := finalOtelMetrics.ResourceMetrics() + ilms := rms.At(0).ScopeMetrics() + metrics := ilms.At(0).Metrics() + assert.Equal(t, 1, metrics.Len()) + + for i := 0; i < metrics.Len(); i++ { + err := addToGroupedMetric(metrics.At(i), + groupedMetrics, + generateTestMetricMetadata(namespace, timestamp, logGroup, logStreamName, instrumentationLibName, metrics.At(i).Type()), + true, + nil, + testCfg, + emfCalcs, + ) + assert.NoError(t, err) + } + assert.Equal(t, 2, len(groupedMetrics)) + expectedLabels := map[string]string{oTellibDimensionKey: instrumentationLibName, "label1": "value1"} + idx := 0 + for _, v := range groupedMetrics { + assert.Equal(t, 1, len(v.metrics)) + assert.Equal(t, 2, len(v.labels)) + assert.Equal(t, generateTestMetricMetadata(namespace, timestamp, logGroup, logStreamName, instrumentationLibName, metrics.At(0).Type(), idx), v.metadata) + assert.Equal(t, expectedLabels, v.labels) + idx++ + } + }) + } func TestAddKubernetesWrapper(t *testing.T) { @@ -508,7 +545,11 @@ func TestTranslateUnit(t *testing.T) { assert.Equal(t, "Count", v) } -func generateTestMetricMetadata(namespace string, timestamp int64, logGroup, logStreamName, instrumentationScopeName string, metricType pmetric.MetricType) cWMetricMetadata { +func generateTestMetricMetadata(namespace string, timestamp int64, logGroup, logStreamName, instrumentationScopeName string, metricType pmetric.MetricType, batchIndex ...int) cWMetricMetadata { + mIndex := 0 + if len(batchIndex) > 0 { + mIndex = batchIndex[0] + } return cWMetricMetadata{ receiver: prometheusReceiver, groupedMetricMetadata: groupedMetricMetadata{ @@ -517,6 +558,7 @@ func generateTestMetricMetadata(namespace string, timestamp int64, logGroup, log logGroup: logGroup, logStream: logStreamName, metricDataType: metricType, + batchIndex: mIndex, }, instrumentationScopeName: instrumentationScopeName, } diff --git a/exporter/awsemfexporter/metric_translator.go b/exporter/awsemfexporter/metric_translator.go index 26aba7bd368e..3b35979a5394 100644 --- a/exporter/awsemfexporter/metric_translator.go +++ b/exporter/awsemfexporter/metric_translator.go @@ -75,6 +75,7 @@ type groupedMetricMetadata struct { logGroup string logStream string metricDataType pmetric.MetricType + batchIndex int retainInitialValueForDelta bool } @@ -140,6 +141,7 @@ func (mt metricTranslator) translateOTelToGroupedMetric(rm pmetric.ResourceMetri logGroup: logGroup, logStream: logStream, metricDataType: metric.Type(), + batchIndex: 0, retainInitialValueForDelta: deltaInitialValue, }, instrumentationScopeName: instrumentationScopeName, From a4e0a1a3ad165ebdbf14558a7fb209161e010e60 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Tue, 12 Nov 2024 20:01:42 +0000 Subject: [PATCH 02/10] Remove un-used code. --- exporter/awsemfexporter/datapoint.go | 133 +++++++++++++--------- exporter/awsemfexporter/datapoint_test.go | 45 ++++++++ 2 files changed, 127 insertions(+), 51 deletions(-) diff --git a/exporter/awsemfexporter/datapoint.go b/exporter/awsemfexporter/datapoint.go index 031dd43e2eb8..e501a99e483a 100644 --- a/exporter/awsemfexporter/datapoint.go +++ b/exporter/awsemfexporter/datapoint.go @@ -115,6 +115,25 @@ type dataPointSplit struct { capacity int } +func (split *dataPointSplit) isNotFull() bool { + return split.length < split.capacity +} + +func (split *dataPointSplit) setMax(maxVal float64) { + split.cWMetricHistogram.Max = maxVal +} + +func (split *dataPointSplit) setMin(minVal float64) { + split.cWMetricHistogram.Min = minVal +} + +func (split *dataPointSplit) appendMetricData(metricVal float64, count uint64) { + split.cWMetricHistogram.Values = append(split.cWMetricHistogram.Values, metricVal) + split.cWMetricHistogram.Counts = append(split.cWMetricHistogram.Counts, float64(count)) + split.length++ + split.cWMetricHistogram.Count += count +} + // CalculateDeltaDatapoints retrieves the NumberDataPoint at the given index and performs rate/delta calculation if necessary. func (dps numberDataPointSlice) CalculateDeltaDatapoints(i int, instrumentationScopeName string, _ bool, calculators *emfCalculators) ([]dataPoint, bool) { metric := dps.NumberDataPointSlice.At(i) @@ -211,11 +230,11 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, metric := dps.ExponentialHistogramDataPointSlice.At(idx) const splitThreshold = 100 - var currentBucketIndex = 0 + currentBucketIndex := 0 + currentPositiveIndex := metric.Positive().BucketCounts().Len() - 1 + currentZeroIndex := 0 + currentNegativeIndex := 0 var datapoints []dataPoint - var currentPositiveIndex = metric.Positive().BucketCounts().Len() - 1 - var currentZeroIndex = 0 - var currentNegativeIndex = 0 totalBucketLen := metric.Positive().BucketCounts().Len() + metric.Negative().BucketCounts().Len() if metric.ZeroCount() > 0 { totalBucketLen++ @@ -239,6 +258,17 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, for currentBucketIndex < totalBucketLen { // Create a new dataPointSplit with a capacity of up to splitThreshold buckets + capacity := splitThreshold + if totalBucketLen-currentBucketIndex < splitThreshold { + capacity = totalBucketLen - currentBucketIndex + } + + sum := 0.0 + // Only assign `Sum` if this is the first split to make sure the total sum of the datapoints after aggregation is correct. + if currentBucketIndex == 0 { + sum = metric.Sum() + } + split := dataPointSplit{ cWMetricHistogram: &cWMetricHistogram{ Values: []float64{}, @@ -246,27 +276,18 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, Max: metric.Max(), Min: metric.Min(), Count: 0, - Sum: 0, + Sum: sum, }, length: 0, - capacity: splitThreshold, + capacity: capacity, } - // Only assign `Sum` if this is the first split to make sure the total sum of the datapoints after aggregation is correct. - if currentBucketIndex == 0 { - split.cWMetricHistogram.Sum = metric.Sum() - } - - if totalBucketLen-currentBucketIndex < splitThreshold { - split.capacity = totalBucketLen - currentBucketIndex - } - - // Set mid-point of positive buckets in values/counts array. - currentBucketIndex, currentPositiveIndex = iteratePositiveBuckets(&split, metric, currentBucketIndex, currentPositiveIndex, totalBucketLen) - // Set count of zero bucket in values/counts array. - currentBucketIndex, currentZeroIndex = iterateZeroBucket(&split, metric, currentBucketIndex, currentZeroIndex, totalBucketLen) - // Set mid-point of negative buckets in values/counts array. - currentBucketIndex, currentNegativeIndex = iterateNegativeBuckets(&split, metric, currentBucketIndex, currentNegativeIndex, totalBucketLen) + // Set collect values from positive buckets and save into split. + currentBucketIndex, currentPositiveIndex = collectDatapointsWithPositiveBuckets(&split, metric, currentBucketIndex, currentPositiveIndex) + // Set collect values from zero buckets and save into split. + currentBucketIndex, currentZeroIndex = collectDatapointsWithZeroBucket(&split, metric, currentBucketIndex, currentZeroIndex) + // Set collect values from negative buckets and save into split. + currentBucketIndex, currentNegativeIndex = collectDatapointsWithNegativeBuckets(&split, metric, currentBucketIndex, currentNegativeIndex) // Add the current split to the datapoints list datapoints = append(datapoints, dataPoint{ @@ -276,10 +297,19 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, timestampMs: unixNanoToMilliseconds(metric.Timestamp()), }) } + + // Override the min and max values of the first and last splits with the raw data of the metric. + datapoints[0].value.(*cWMetricHistogram).Max = metric.Max() + datapoints[len(datapoints)-1].value.(*cWMetricHistogram).Min = metric.Min() + return datapoints, true } -func iteratePositiveBuckets(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentPositiveIndex int, totalBucketLen int) (int, int) { +func collectDatapointsWithPositiveBuckets(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentPositiveIndex int) (int, int) { + if !split.isNotFull() || currentPositiveIndex < 0 { + return currentBucketIndex, currentPositiveIndex + } + scale := metric.Scale() base := math.Pow(2, math.Pow(2, float64(-scale))) positiveBuckets := metric.Positive() @@ -288,7 +318,7 @@ func iteratePositiveBuckets(split *dataPointSplit, metric pmetric.ExponentialHis bucketBegin := 0.0 bucketEnd := 0.0 - for split.length < split.capacity && currentPositiveIndex >= 0 { + for split.isNotFull() && currentPositiveIndex >= 0 { index := currentPositiveIndex + int(positiveOffset) if bucketEnd == 0 { bucketEnd = math.Pow(base, float64(index+1)) @@ -299,15 +329,14 @@ func iteratePositiveBuckets(split *dataPointSplit, metric pmetric.ExponentialHis metricVal := (bucketBegin + bucketEnd) / 2 count := positiveBucketCounts.At(currentPositiveIndex) if count > 0 { - split.cWMetricHistogram.Values = append(split.cWMetricHistogram.Values, metricVal) - split.cWMetricHistogram.Counts = append(split.cWMetricHistogram.Counts, float64(count)) - split.length++ - split.cWMetricHistogram.Count += count - if split.length == 1 && currentBucketIndex != 0 { - split.cWMetricHistogram.Max = bucketEnd + split.appendMetricData(metricVal, count) + + // The value are append from high to low, set Max from the first bucket (highest value) and Min from the last bucket (lowest value) + if split.length == 1 { + split.setMax(bucketEnd) } - if split.length == split.capacity && currentBucketIndex != totalBucketLen-1 { - split.cWMetricHistogram.Min = bucketBegin + if !split.isNotFull() { + split.setMin(bucketBegin) } } currentBucketIndex++ @@ -317,17 +346,16 @@ func iteratePositiveBuckets(split *dataPointSplit, metric pmetric.ExponentialHis return currentBucketIndex, currentPositiveIndex } -func iterateZeroBucket(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentZeroIndex int, totalBucketLen int) (int, int) { - if metric.ZeroCount() > 0 && split.length < split.capacity && currentZeroIndex == 0 { - split.cWMetricHistogram.Values = append(split.cWMetricHistogram.Values, 0) - split.cWMetricHistogram.Counts = append(split.cWMetricHistogram.Counts, float64(metric.ZeroCount())) - split.length++ - split.cWMetricHistogram.Count += metric.ZeroCount() - if split.length == 1 && currentBucketIndex != 0 { - split.cWMetricHistogram.Max = 0 +func collectDatapointsWithZeroBucket(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentZeroIndex int) (int, int) { + if metric.ZeroCount() > 0 && split.isNotFull() && currentZeroIndex == 0 { + split.appendMetricData(0, metric.ZeroCount()) + + // The value are append from high to low, set Max from the first bucket (highest value) and Min from the last bucket (lowest value) + if split.length == 1 { + split.setMax(0) } - if split.length == split.capacity && currentBucketIndex != totalBucketLen-1 { - split.cWMetricHistogram.Min = 0 + if !split.isNotFull() { + split.setMin(0) } currentZeroIndex++ currentBucketIndex++ @@ -336,12 +364,16 @@ func iterateZeroBucket(split *dataPointSplit, metric pmetric.ExponentialHistogra return currentBucketIndex, currentZeroIndex } -func iterateNegativeBuckets(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentNegativeIndex int, totalBucketLen int) (int, int) { +func collectDatapointsWithNegativeBuckets(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentNegativeIndex int) (int, int) { // According to metrics spec, the value in histogram is expected to be non-negative. // https://opentelemetry.io/docs/specs/otel/metrics/api/#histogram // However, the negative support is defined in metrics data model. // https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exponentialhistogram // The negative is also supported but only verified with unit test. + if !split.isNotFull() || currentNegativeIndex >= metric.Negative().BucketCounts().Len() { + return currentBucketIndex, currentNegativeIndex + } + scale := metric.Scale() base := math.Pow(2, math.Pow(2, float64(-scale))) negativeBuckets := metric.Negative() @@ -350,7 +382,7 @@ func iterateNegativeBuckets(split *dataPointSplit, metric pmetric.ExponentialHis bucketBegin := 0.0 bucketEnd := 0.0 - for split.length < split.capacity && currentNegativeIndex < metric.Negative().BucketCounts().Len() { + for split.isNotFull() && currentNegativeIndex < metric.Negative().BucketCounts().Len() { index := currentNegativeIndex + int(negativeOffset) if bucketEnd == 0 { bucketEnd = -math.Pow(base, float64(index)) @@ -361,15 +393,14 @@ func iterateNegativeBuckets(split *dataPointSplit, metric pmetric.ExponentialHis metricVal := (bucketBegin + bucketEnd) / 2 count := negativeBucketCounts.At(currentNegativeIndex) if count > 0 { - split.cWMetricHistogram.Values = append(split.cWMetricHistogram.Values, metricVal) - split.cWMetricHistogram.Counts = append(split.cWMetricHistogram.Counts, float64(count)) - split.length++ - split.cWMetricHistogram.Count += count - if split.length == 1 && currentBucketIndex != 0 { - split.cWMetricHistogram.Max = bucketEnd + split.appendMetricData(metricVal, count) + + // The value are append from high to low, set Max from the first bucket (highest value) and Min from the last bucket (lowest value) + if split.length == 1 { + split.setMax(bucketEnd) } - if split.length == split.capacity && currentBucketIndex != totalBucketLen-1 { - split.cWMetricHistogram.Min = bucketBegin + if !split.isNotFull() { + split.setMin(bucketBegin) } } currentBucketIndex++ diff --git a/exporter/awsemfexporter/datapoint_test.go b/exporter/awsemfexporter/datapoint_test.go index a3004608b0c8..d61a599d8632 100644 --- a/exporter/awsemfexporter/datapoint_test.go +++ b/exporter/awsemfexporter/datapoint_test.go @@ -1347,6 +1347,51 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat }, }, }, + { + name: "Exponential histogram with more than 100 buckets, including positive, negative and zero buckets with zero counts", + histogramDPS: func() pmetric.ExponentialHistogramDataPointSlice { + histogramDPS := pmetric.NewExponentialHistogramDataPointSlice() + histogramDP := histogramDPS.AppendEmpty() + posBucketCounts := make([]uint64, 60) + for i := range posBucketCounts { + posBucketCounts[i] = uint64(i % 5) + } + histogramDP.Positive().BucketCounts().FromRaw(posBucketCounts) + histogramDP.SetZeroCount(2) + negBucketCounts := make([]uint64, 60) + for i := range negBucketCounts { + negBucketCounts[i] = uint64(i % 5) + } + histogramDP.Negative().BucketCounts().FromRaw(negBucketCounts) + histogramDP.SetSum(1000) + histogramDP.SetMin(-9e+17) + histogramDP.SetMax(9e+17) + histogramDP.SetCount(uint64(3662)) + histogramDP.Attributes().PutStr("label1", "value1") + return histogramDPS + }(), + expectedDatapoints: []dataPoint{ + { + name: "foo", + value: &cWMetricHistogram{ + Values: []float64{8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 2.7021597764222976e+16, + 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 8.44424930131968e+14, 4.22212465065984e+14, 2.11106232532992e+14, + 1.05553116266496e+14, 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 8.24633720832e+11, 4.12316860416e+11, + 2.06158430208e+11, 1.03079215104e+11, 2.5769803776e+10, 1.2884901888e+10, 6.442450944e+09, 3.221225472e+09, 8.05306368e+08, 4.02653184e+08, + 2.01326592e+08, 1.00663296e+08, 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 786432, 393216, 196608, 98304, 24576, 12288, 6144, 3072, + 768, 384, 192, 96, 24, 12, 6, 3, 0, -3, -6, -12, -24, -96, -192, -384, -768, -3072, -6144, -12288, -24576, -98304, -196608, -393216, -786432, + -3.145728e+06, -6.291456e+06, -1.2582912e+07, -2.5165824e+07, -1.00663296e+08, -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, -3.221225472e+09, + -6.442450944e+09, -1.2884901888e+10, -2.5769803776e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, -8.24633720832e+11, -3.298534883328e+12, + -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, + -3.377699720527872e+15, -6.755399441055744e+15, -1.3510798882111488e+16, -2.7021597764222976e+16, -1.080863910568919e+17, -2.161727821137838e+17, + -4.323455642275676e+17, -8.646911284551352e+17}, + Counts: []float64{4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, + 2, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4}, + Sum: 1000, Count: 242, Min: -9e+17, Max: 9e+17}, + labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, + }, + }, + }, } for _, tc := range testCases { From ed126c7c75e2db20f9542ac056a5a69a88981ddf Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Tue, 12 Nov 2024 23:59:56 +0000 Subject: [PATCH 03/10] Add changelog. --- ...-emf-log-when-buckets-larger-than-100.yaml | 27 +++++++++++++++++++ .../awsemfexporter/grouped_metric_test.go | 6 ++--- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 .chloggen/split-emf-log-when-buckets-larger-than-100.yaml diff --git a/.chloggen/split-emf-log-when-buckets-larger-than-100.yaml b/.chloggen/split-emf-log-when-buckets-larger-than-100.yaml new file mode 100644 index 000000000000..da57f8624ea9 --- /dev/null +++ b/.chloggen/split-emf-log-when-buckets-larger-than-100.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: awsemfexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Split EMF log to multiple log splits when buckets larger than 100. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [242] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/awsemfexporter/grouped_metric_test.go b/exporter/awsemfexporter/grouped_metric_test.go index 6e09d82b7a3e..08cb72c3151e 100644 --- a/exporter/awsemfexporter/grouped_metric_test.go +++ b/exporter/awsemfexporter/grouped_metric_test.go @@ -431,12 +431,12 @@ func TestAddToGroupedMetric(t *testing.T) { ) assert.NoError(t, err) } - assert.Equal(t, 2, len(groupedMetrics)) + assert.Len(t, groupedMetrics, 2) expectedLabels := map[string]string{oTellibDimensionKey: instrumentationLibName, "label1": "value1"} idx := 0 for _, v := range groupedMetrics { - assert.Equal(t, 1, len(v.metrics)) - assert.Equal(t, 2, len(v.labels)) + assert.Len(t, v.metrics, 1) + assert.Len(t, v.labels, 2) assert.Equal(t, generateTestMetricMetadata(namespace, timestamp, logGroup, logStreamName, instrumentationLibName, metrics.At(0).Type(), idx), v.metadata) assert.Equal(t, expectedLabels, v.labels) idx++ From 0eecd3fbd2d641f6aedcf5c3705be1d4cb70084b Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Wed, 13 Nov 2024 04:46:16 +0000 Subject: [PATCH 04/10] Eliminate zero splits. --- exporter/awsemfexporter/datapoint.go | 45 +++++++++++++++------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/exporter/awsemfexporter/datapoint.go b/exporter/awsemfexporter/datapoint.go index 361eee3ca42b..b60d7122920e 100644 --- a/exporter/awsemfexporter/datapoint.go +++ b/exporter/awsemfexporter/datapoint.go @@ -240,22 +240,6 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, totalBucketLen++ } - if totalBucketLen == 0 { - return []dataPoint{{ - name: dps.metricName, - value: &cWMetricHistogram{ - Values: []float64{}, - Counts: []float64{}, - Count: metric.Count(), - Sum: metric.Sum(), - Max: metric.Max(), - Min: metric.Min(), - }, - labels: createLabels(metric.Attributes(), instrumentationScopeName), - timestampMs: unixNanoToMilliseconds(metric.Timestamp()), - }}, true - } - for currentBucketIndex < totalBucketLen { // Create a new dataPointSplit with a capacity of up to splitThreshold buckets capacity := splitThreshold @@ -289,13 +273,32 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, // Set collect values from negative buckets and save into split. currentBucketIndex, currentNegativeIndex = collectDatapointsWithNegativeBuckets(&split, metric, currentBucketIndex, currentNegativeIndex) - // Add the current split to the datapoints list - datapoints = append(datapoints, dataPoint{ - name: dps.metricName, - value: split.cWMetricHistogram, + if split.length > 0 { + // Add the current split to the datapoints list + datapoints = append(datapoints, dataPoint{ + name: dps.metricName, + value: split.cWMetricHistogram, + labels: createLabels(metric.Attributes(), instrumentationScopeName), + timestampMs: unixNanoToMilliseconds(metric.Timestamp()), + }) + + } + } + + if len(datapoints) == 0 { + return []dataPoint{{ + name: dps.metricName, + value: &cWMetricHistogram{ + Values: []float64{}, + Counts: []float64{}, + Count: metric.Count(), + Sum: metric.Sum(), + Max: metric.Max(), + Min: metric.Min(), + }, labels: createLabels(metric.Attributes(), instrumentationScopeName), timestampMs: unixNanoToMilliseconds(metric.Timestamp()), - }) + }}, true } // Override the min and max values of the first and last splits with the raw data of the metric. From 9b942c8eb434a8ed8e302127f38fe7e09c1c2964 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Thu, 28 Nov 2024 16:38:18 +0000 Subject: [PATCH 05/10] Add exponential histrogram LongBuckets cases into benchmark tests. --- exporter/awsemfexporter/datapoint_test.go | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/exporter/awsemfexporter/datapoint_test.go b/exporter/awsemfexporter/datapoint_test.go index 905e99f54679..dd6865a3060c 100644 --- a/exporter/awsemfexporter/datapoint_test.go +++ b/exporter/awsemfexporter/datapoint_test.go @@ -273,6 +273,35 @@ func generateTestExponentialHistogramMetricWithLongBuckets(name string) pmetric. return otelMetrics } +func generateTestExponentialHistogramMetricWithThousandBuckets(name string) pmetric.Metrics { + otelMetrics := pmetric.NewMetrics() + rs := otelMetrics.ResourceMetrics().AppendEmpty() + metrics := rs.ScopeMetrics().AppendEmpty().Metrics() + metric := metrics.AppendEmpty() + metric.SetName(name) + metric.SetUnit("Seconds") + exponentialHistogramMetric := metric.SetEmptyExponentialHistogram() + + exponentialHistogramDatapoint := exponentialHistogramMetric.DataPoints().AppendEmpty() + exponentialHistogramDatapoint.SetCount(250550) + exponentialHistogramDatapoint.SetSum(10000) + exponentialHistogramDatapoint.SetMin(-9e+20) + exponentialHistogramDatapoint.SetMax(9e+20) + exponentialHistogramDatapoint.SetZeroCount(50) + posBucketCounts := make([]uint64, 500) + for i := range posBucketCounts { + posBucketCounts[i] = uint64(i + 1) + } + exponentialHistogramDatapoint.Positive().BucketCounts().FromRaw(posBucketCounts) + negBucketCounts := make([]uint64, 500) + for i := range negBucketCounts { + negBucketCounts[i] = uint64(i + 1) + } + exponentialHistogramDatapoint.Negative().BucketCounts().FromRaw(negBucketCounts) + exponentialHistogramDatapoint.Attributes().PutStr("label1", "value1") + return otelMetrics +} + func generateTestSummaryMetric(name string) pmetric.Metrics { otelMetrics := pmetric.NewMetrics() rs := otelMetrics.ResourceMetrics().AppendEmpty() @@ -2043,6 +2072,8 @@ func BenchmarkGetAndCalculateDeltaDataPoints(b *testing.B) { generateTestGaugeMetric("int-gauge", doubleValueType), generateTestHistogramMetric("histogram"), generateTestExponentialHistogramMetric("exponential-histogram"), + generateTestExponentialHistogramMetricWithLongBuckets("exponential-histogram-long-buckets"), + generateTestExponentialHistogramMetricWithThousandBuckets("exponential-histogram-thousand-buckets"), generateTestSumMetric("int-sum", intValueType), generateTestSumMetric("double-sum", doubleValueType), generateTestSummaryMetric("summary"), From c4c84fad5fb85395ff8a399e9e94c73af8cc6b00 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Tue, 3 Dec 2024 17:56:39 +0000 Subject: [PATCH 06/10] Address comments. --- exporter/awsemfexporter/datapoint.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/exporter/awsemfexporter/datapoint.go b/exporter/awsemfexporter/datapoint.go index b60d7122920e..d09e52e3e945 100644 --- a/exporter/awsemfexporter/datapoint.go +++ b/exporter/awsemfexporter/datapoint.go @@ -109,14 +109,15 @@ type summaryMetricEntry struct { count uint64 } +// dataPointSplit is not thread-safe. Ensure proper synchronization if concurrent access is required. type dataPointSplit struct { cWMetricHistogram *cWMetricHistogram length int capacity int } -func (split *dataPointSplit) isNotFull() bool { - return split.length < split.capacity +func (split *dataPointSplit) isFull() bool { + return split.length >= split.capacity } func (split *dataPointSplit) setMax(maxVal float64) { @@ -242,10 +243,7 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, for currentBucketIndex < totalBucketLen { // Create a new dataPointSplit with a capacity of up to splitThreshold buckets - capacity := splitThreshold - if totalBucketLen-currentBucketIndex < splitThreshold { - capacity = totalBucketLen - currentBucketIndex - } + capacity := min(splitThreshold, totalBucketLen-currentBucketIndex) sum := 0.0 // Only assign `Sum` if this is the first split to make sure the total sum of the datapoints after aggregation is correct. @@ -309,7 +307,7 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, } func collectDatapointsWithPositiveBuckets(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentPositiveIndex int) (int, int) { - if !split.isNotFull() || currentPositiveIndex < 0 { + if split.isFull() || currentPositiveIndex < 0 { return currentBucketIndex, currentPositiveIndex } @@ -321,7 +319,7 @@ func collectDatapointsWithPositiveBuckets(split *dataPointSplit, metric pmetric. bucketBegin := 0.0 bucketEnd := 0.0 - for split.isNotFull() && currentPositiveIndex >= 0 { + for !split.isFull() && currentPositiveIndex >= 0 { index := currentPositiveIndex + int(positiveOffset) if bucketEnd == 0 { bucketEnd = math.Pow(base, float64(index+1)) @@ -338,7 +336,7 @@ func collectDatapointsWithPositiveBuckets(split *dataPointSplit, metric pmetric. if split.length == 1 { split.setMax(bucketEnd) } - if !split.isNotFull() { + if split.isFull() { split.setMin(bucketBegin) } } @@ -350,14 +348,14 @@ func collectDatapointsWithPositiveBuckets(split *dataPointSplit, metric pmetric. } func collectDatapointsWithZeroBucket(split *dataPointSplit, metric pmetric.ExponentialHistogramDataPoint, currentBucketIndex int, currentZeroIndex int) (int, int) { - if metric.ZeroCount() > 0 && split.isNotFull() && currentZeroIndex == 0 { + if metric.ZeroCount() > 0 && !split.isFull() && currentZeroIndex == 0 { split.appendMetricData(0, metric.ZeroCount()) // The value are append from high to low, set Max from the first bucket (highest value) and Min from the last bucket (lowest value) if split.length == 1 { split.setMax(0) } - if !split.isNotFull() { + if split.isFull() { split.setMin(0) } currentZeroIndex++ @@ -373,7 +371,7 @@ func collectDatapointsWithNegativeBuckets(split *dataPointSplit, metric pmetric. // However, the negative support is defined in metrics data model. // https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exponentialhistogram // The negative is also supported but only verified with unit test. - if !split.isNotFull() || currentNegativeIndex >= metric.Negative().BucketCounts().Len() { + if split.isFull() || currentNegativeIndex >= metric.Negative().BucketCounts().Len() { return currentBucketIndex, currentNegativeIndex } @@ -385,7 +383,7 @@ func collectDatapointsWithNegativeBuckets(split *dataPointSplit, metric pmetric. bucketBegin := 0.0 bucketEnd := 0.0 - for split.isNotFull() && currentNegativeIndex < metric.Negative().BucketCounts().Len() { + for !split.isFull() && currentNegativeIndex < metric.Negative().BucketCounts().Len() { index := currentNegativeIndex + int(negativeOffset) if bucketEnd == 0 { bucketEnd = -math.Pow(base, float64(index)) @@ -402,7 +400,7 @@ func collectDatapointsWithNegativeBuckets(split *dataPointSplit, metric pmetric. if split.length == 1 { split.setMax(bucketEnd) } - if !split.isNotFull() { + if split.isFull() { split.setMin(bucketBegin) } } From 186cc4683557d22b7f081fd552a24613977bf77e Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Wed, 4 Dec 2024 09:55:07 -0800 Subject: [PATCH 07/10] refine comments. --- exporter/awsemfexporter/datapoint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exporter/awsemfexporter/datapoint.go b/exporter/awsemfexporter/datapoint.go index d09e52e3e945..9f1f23854a44 100644 --- a/exporter/awsemfexporter/datapoint.go +++ b/exporter/awsemfexporter/datapoint.go @@ -109,7 +109,8 @@ type summaryMetricEntry struct { count uint64 } -// dataPointSplit is not thread-safe. Ensure proper synchronization if concurrent access is required. +// dataPointSplit is a structure used to manage segments of data points split from a histogram. +// It is not safe for concurrent use. type dataPointSplit struct { cWMetricHistogram *cWMetricHistogram length int From 298ac463ad35ef765d83e38fe0683c7ec755fd81 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Wed, 4 Dec 2024 20:48:32 +0000 Subject: [PATCH 08/10] Update benchemark tests to include 100, 200, 300, 500 buckets. --- exporter/awsemfexporter/datapoint_test.go | 54 ++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/exporter/awsemfexporter/datapoint_test.go b/exporter/awsemfexporter/datapoint_test.go index dd6865a3060c..0acc12451898 100644 --- a/exporter/awsemfexporter/datapoint_test.go +++ b/exporter/awsemfexporter/datapoint_test.go @@ -7,6 +7,7 @@ import ( "fmt" "math" "reflect" + "strconv" "testing" "time" @@ -273,7 +274,8 @@ func generateTestExponentialHistogramMetricWithLongBuckets(name string) pmetric. return otelMetrics } -func generateTestExponentialHistogramMetricWithThousandBuckets(name string) pmetric.Metrics { +func generateTestExponentialHistogramMetricWithSpecifiedNumberOfBuckets(name string, bucketLength int) pmetric.Metrics { + halfBucketLength := bucketLength / 2 otelMetrics := pmetric.NewMetrics() rs := otelMetrics.ResourceMetrics().AppendEmpty() metrics := rs.ScopeMetrics().AppendEmpty().Metrics() @@ -288,12 +290,12 @@ func generateTestExponentialHistogramMetricWithThousandBuckets(name string) pmet exponentialHistogramDatapoint.SetMin(-9e+20) exponentialHistogramDatapoint.SetMax(9e+20) exponentialHistogramDatapoint.SetZeroCount(50) - posBucketCounts := make([]uint64, 500) + posBucketCounts := make([]uint64, halfBucketLength) for i := range posBucketCounts { posBucketCounts[i] = uint64(i + 1) } exponentialHistogramDatapoint.Positive().BucketCounts().FromRaw(posBucketCounts) - negBucketCounts := make([]uint64, 500) + negBucketCounts := make([]uint64, halfBucketLength) for i := range negBucketCounts { negBucketCounts[i] = uint64(i + 1) } @@ -2072,8 +2074,6 @@ func BenchmarkGetAndCalculateDeltaDataPoints(b *testing.B) { generateTestGaugeMetric("int-gauge", doubleValueType), generateTestHistogramMetric("histogram"), generateTestExponentialHistogramMetric("exponential-histogram"), - generateTestExponentialHistogramMetricWithLongBuckets("exponential-histogram-long-buckets"), - generateTestExponentialHistogramMetricWithThousandBuckets("exponential-histogram-thousand-buckets"), generateTestSumMetric("int-sum", intValueType), generateTestSumMetric("double-sum", doubleValueType), generateTestSummaryMetric("summary"), @@ -2096,3 +2096,47 @@ func BenchmarkGetAndCalculateDeltaDataPoints(b *testing.B) { } } } + +func benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b *testing.B, bucketLength int) { + // Generate metrics with the specified number of buckets + generateMetrics := []pmetric.Metrics{ + generateTestExponentialHistogramMetricWithSpecifiedNumberOfBuckets( + "exponential-histogram-thousand-buckets-"+strconv.Itoa(bucketLength), bucketLength), + } + + finalOtelMetrics := generateOtelTestMetrics(generateMetrics...) + rms := finalOtelMetrics.ResourceMetrics() + metrics := rms.At(0).ScopeMetrics().At(0).Metrics() + + emfCalcs := setupEmfCalculators() + defer require.NoError(b, shutdownEmfCalculators(emfCalcs)) + + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < metrics.Len(); i++ { + metadata := generateTestMetricMetadata( + "namespace", time.Now().UnixNano()/int64(time.Millisecond), "log-group", "log-stream", "cloudwatch-otel", metrics.At(i).Type()) + dps := getDataPoints(metrics.At(i), metadata, zap.NewNop()) + + for j := 0; j < dps.Len(); j++ { + dps.CalculateDeltaDatapoints(j, "", false, emfCalcs) + } + } + } +} + +func BenchmarkGetAndCalculateDeltaDataPointsWith100Buckets(b *testing.B) { + benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b, 100) +} + +func BenchmarkGetAndCalculateDeltaDataPointsWith200Buckets(b *testing.B) { + benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b, 200) +} + +func BenchmarkGetAndCalculateDeltaDataPointsWith300Buckets(b *testing.B) { + benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b, 300) +} + +func BenchmarkGetAndCalculateDeltaDataPointsWith500Buckets(b *testing.B) { + benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b, 500) +} From baa5f3f3bb2fd3049bbe0906c0ed15ff8762d431 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Wed, 4 Dec 2024 21:23:28 +0000 Subject: [PATCH 09/10] Refine benchmark tests. --- exporter/awsemfexporter/datapoint_test.go | 48 ++++++----------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/exporter/awsemfexporter/datapoint_test.go b/exporter/awsemfexporter/datapoint_test.go index 0acc12451898..aeb1421f6424 100644 --- a/exporter/awsemfexporter/datapoint_test.go +++ b/exporter/awsemfexporter/datapoint_test.go @@ -2068,12 +2068,14 @@ func TestGetDataPoints(t *testing.T) { }) } -func BenchmarkGetAndCalculateDeltaDataPoints(b *testing.B) { +func benchmarkGetAndCalculateDeltaDataPoints(b *testing.B, bucketLength int) { generateMetrics := []pmetric.Metrics{ generateTestGaugeMetric("int-gauge", intValueType), generateTestGaugeMetric("int-gauge", doubleValueType), generateTestHistogramMetric("histogram"), generateTestExponentialHistogramMetric("exponential-histogram"), + generateTestExponentialHistogramMetricWithSpecifiedNumberOfBuckets( + "exponential-histogram-buckets-"+strconv.Itoa(bucketLength), bucketLength), generateTestSumMetric("int-sum", intValueType), generateTestSumMetric("double-sum", doubleValueType), generateTestSummaryMetric("summary"), @@ -2097,46 +2099,18 @@ func BenchmarkGetAndCalculateDeltaDataPoints(b *testing.B) { } } -func benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b *testing.B, bucketLength int) { - // Generate metrics with the specified number of buckets - generateMetrics := []pmetric.Metrics{ - generateTestExponentialHistogramMetricWithSpecifiedNumberOfBuckets( - "exponential-histogram-thousand-buckets-"+strconv.Itoa(bucketLength), bucketLength), - } - - finalOtelMetrics := generateOtelTestMetrics(generateMetrics...) - rms := finalOtelMetrics.ResourceMetrics() - metrics := rms.At(0).ScopeMetrics().At(0).Metrics() - - emfCalcs := setupEmfCalculators() - defer require.NoError(b, shutdownEmfCalculators(emfCalcs)) - - b.ResetTimer() - for n := 0; n < b.N; n++ { - for i := 0; i < metrics.Len(); i++ { - metadata := generateTestMetricMetadata( - "namespace", time.Now().UnixNano()/int64(time.Millisecond), "log-group", "log-stream", "cloudwatch-otel", metrics.At(i).Type()) - dps := getDataPoints(metrics.At(i), metadata, zap.NewNop()) - - for j := 0; j < dps.Len(); j++ { - dps.CalculateDeltaDatapoints(j, "", false, emfCalcs) - } - } - } -} - -func BenchmarkGetAndCalculateDeltaDataPointsWith100Buckets(b *testing.B) { - benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b, 100) +func BenchmarkGetAndCalculateDeltaDataPointsInclude100Buckets(b *testing.B) { + benchmarkGetAndCalculateDeltaDataPoints(b, 100) } -func BenchmarkGetAndCalculateDeltaDataPointsWith200Buckets(b *testing.B) { - benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b, 200) +func BenchmarkGetAndCalculateDeltaDataPointsInclude200Buckets(b *testing.B) { + benchmarkGetAndCalculateDeltaDataPoints(b, 200) } -func BenchmarkGetAndCalculateDeltaDataPointsWith300Buckets(b *testing.B) { - benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b, 300) +func BenchmarkGetAndCalculateDeltaDataPointsInclude300Buckets(b *testing.B) { + benchmarkGetAndCalculateDeltaDataPoints(b, 300) } -func BenchmarkGetAndCalculateDeltaDataPointsWith500Buckets(b *testing.B) { - benchmarkGetAndCalculateDeltaDataPointsWithBuckets(b, 500) +func BenchmarkGetAndCalculateDeltaDataPointsInclude500Buckets(b *testing.B) { + benchmarkGetAndCalculateDeltaDataPoints(b, 500) } From 3269d4a465004b0078a7ef681448e8c7db480be2 Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Thu, 5 Dec 2024 19:20:50 +0000 Subject: [PATCH 10/10] Apply gofumpt. --- exporter/awsemfexporter/config.go | 14 +- exporter/awsemfexporter/datapoint.go | 1 - exporter/awsemfexporter/datapoint_test.go | 152 +++++++---- exporter/awsemfexporter/emf_exporter.go | 1 - exporter/awsemfexporter/metric_translator.go | 1 - .../awsemfexporter/metric_translator_test.go | 236 +++++++++--------- 6 files changed, 225 insertions(+), 180 deletions(-) diff --git a/exporter/awsemfexporter/config.go b/exporter/awsemfexporter/config.go index 969a69e67798..7495277e1c2a 100644 --- a/exporter/awsemfexporter/config.go +++ b/exporter/awsemfexporter/config.go @@ -14,11 +14,9 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry" ) -var ( - // eMFSupportedUnits contains the unit collection supported by CloudWatch backend service. - // https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html - eMFSupportedUnits = newEMFSupportedUnits() -) +// eMFSupportedUnits contains the unit collection supported by CloudWatch backend service. +// https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html +var eMFSupportedUnits = newEMFSupportedUnits() // Config defines configuration for AWS EMF exporter. type Config struct { @@ -152,11 +150,13 @@ func (config *Config) isAppSignalsEnabled() bool { func newEMFSupportedUnits() map[string]any { unitIndexer := map[string]any{} - for _, unit := range []string{"Seconds", "Microseconds", "Milliseconds", "Bytes", "Kilobytes", "Megabytes", + for _, unit := range []string{ + "Seconds", "Microseconds", "Milliseconds", "Bytes", "Kilobytes", "Megabytes", "Gigabytes", "Terabytes", "Bits", "Kilobits", "Megabits", "Gigabits", "Terabits", "Percent", "Count", "Bytes/Second", "Kilobytes/Second", "Megabytes/Second", "Gigabytes/Second", "Terabytes/Second", "Bits/Second", "Kilobits/Second", - "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", "None"} { + "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", "None", + } { unitIndexer[unit] = nil } return unitIndexer diff --git a/exporter/awsemfexporter/datapoint.go b/exporter/awsemfexporter/datapoint.go index 9f1f23854a44..36d16b62de88 100644 --- a/exporter/awsemfexporter/datapoint.go +++ b/exporter/awsemfexporter/datapoint.go @@ -280,7 +280,6 @@ func (dps exponentialHistogramDataPointSlice) CalculateDeltaDatapoints(idx int, labels: createLabels(metric.Attributes(), instrumentationScopeName), timestampMs: unixNanoToMilliseconds(metric.Timestamp()), }) - } } diff --git a/exporter/awsemfexporter/datapoint_test.go b/exporter/awsemfexporter/datapoint_test.go index aeb1421f6424..fb02857c6fb0 100644 --- a/exporter/awsemfexporter/datapoint_test.go +++ b/exporter/awsemfexporter/datapoint_test.go @@ -393,6 +393,7 @@ func generateOtelTestMetrics(generatedOtelMetrics ...pmetric.Metrics) pmetric.Me } return finalOtelMetrics } + func generateDeltaMetricMetadata(adjustToDelta bool, metricName string, retainInitialValueForDelta bool) deltaMetricMetadata { return deltaMetricMetadata{ adjustToDelta: adjustToDelta, @@ -979,7 +980,8 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat { name: "foo", value: &cWMetricHistogram{ - Values: []float64{8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, 2.7021597764222976e+16, + Values: []float64{ + 8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, 2.7021597764222976e+16, 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 1.688849860263936e+15, 8.44424930131968e+14, 4.22212465065984e+14, 2.11106232532992e+14, 1.05553116266496e+14, 5.2776558133248e+13, 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 1.649267441664e+12, 8.24633720832e+11, 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, @@ -988,22 +990,29 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat 12288, 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5, 0, -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, -3072, -6144, -12288, -24576, -49152, -98304, -196608, -393216, -786432, -1.572864e+06, -3.145728e+06, -6.291456e+06, -1.2582912e+07, -2.5165824e+07, -5.0331648e+07, -1.00663296e+08, -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, -1.610612736e+09, -3.221225472e+09, -6.442450944e+09, - -1.2884901888e+10, -2.5769803776e+10, -5.1539607552e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11}, - Counts: []float64{60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, + -1.2884901888e+10, -2.5769803776e+10, -5.1539607552e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, + }, + Counts: []float64{ + 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39}, - Sum: 1000, Count: 2612, Min: -5.49755813888e+11, Max: 9e+17}, + 34, 35, 36, 37, 38, 39, + }, + Sum: 1000, Count: 2612, Min: -5.49755813888e+11, Max: 9e+17, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, { name: "foo", value: &cWMetricHistogram{ - Values: []float64{-8.24633720832e+11, -1.649267441664e+12, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -5.2776558133248e+13, + Values: []float64{ + -8.24633720832e+11, -1.649267441664e+12, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -5.2776558133248e+13, -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, -1.688849860263936e+15, -3.377699720527872e+15, -6.755399441055744e+15, -1.3510798882111488e+16, -2.7021597764222976e+16, -5.404319552844595e+16, -1.080863910568919e+17, -2.161727821137838e+17, - -4.323455642275676e+17, -8.646911284551352e+17}, + -4.323455642275676e+17, -8.646911284551352e+17, + }, Counts: []float64{40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60}, - Sum: 0, Count: 1050, Min: -9e+17, Max: -5.49755813888e+11}, + Sum: 0, Count: 1050, Min: -9e+17, Max: -5.49755813888e+11, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, }, @@ -1030,7 +1039,8 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat { name: "foo", value: &cWMetricHistogram{ - Values: []float64{9.969209968386869e+35, 4.9846049841934345e+35, 2.4923024920967173e+35, 1.2461512460483586e+35, 6.230756230241793e+34, + Values: []float64{ + 9.969209968386869e+35, 4.9846049841934345e+35, 2.4923024920967173e+35, 1.2461512460483586e+35, 6.230756230241793e+34, 3.1153781151208966e+34, 1.5576890575604483e+34, 7.788445287802241e+33, 3.894222643901121e+33, 1.9471113219505604e+33, 9.735556609752802e+32, 4.867778304876401e+32, 2.4338891524382005e+32, 1.2169445762191002e+32, 6.084722881095501e+31, 3.0423614405477506e+31, 1.5211807202738753e+31, 7.605903601369376e+30, 3.802951800684688e+30, 1.901475900342344e+30, @@ -1048,12 +1058,16 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 1.649267441664e+12, 8.24633720832e+11, 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, 1.2884901888e+10, 6.442450944e+09, 3.221225472e+09, 1.610612736e+09, 8.05306368e+08, 4.02653184e+08, 2.01326592e+08, 1.00663296e+08, - 5.0331648e+07, 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06}, - Counts: []float64{120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 99, + 5.0331648e+07, 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06, + }, + Counts: []float64{ + 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, - 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21}, - Sum: 10000, Count: 7050, Min: 1.048576e+06, Max: 9e+36}, + 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, + }, + Sum: 10000, Count: 7050, Min: 1.048576e+06, Max: 9e+36, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, { @@ -1061,7 +1075,8 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat value: &cWMetricHistogram{ Values: []float64{786432, 393216, 196608, 98304, 49152, 24576, 12288, 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5, 0}, Counts: []float64{20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 2}, - Sum: 0, Count: 212, Min: 0, Max: 1.048576e+06}, + Sum: 0, Count: 212, Min: 0, Max: 1.048576e+06, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, }, @@ -1088,7 +1103,8 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat { name: "foo", value: &cWMetricHistogram{ - Values: []float64{0, -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, -3072, -6144, -12288, -24576, + Values: []float64{ + 0, -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, -3072, -6144, -12288, -24576, -49152, -98304, -196608, -393216, -786432, -1.572864e+06, -3.145728e+06, -6.291456e+06, -1.2582912e+07, -2.5165824e+07, -5.0331648e+07, -1.00663296e+08, -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, -1.610612736e+09, -3.221225472e+09, -6.442450944e+09, -1.2884901888e+10, -2.5769803776e+10, @@ -1106,24 +1122,31 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat -7.253554917687775e+24, -1.450710983537555e+25, -2.90142196707511e+25, -5.80284393415022e+25, -1.160568786830044e+26, -2.321137573660088e+26, -4.642275147320176e+26, -9.284550294640352e+26, -1.8569100589280704e+27, -3.713820117856141e+27, -7.427640235712282e+27, -1.4855280471424563e+28, - -2.9710560942849127e+28, -5.942112188569825e+28, -1.188422437713965e+29, -2.37684487542793e+29, -4.75368975085586e+29}, - Counts: []float64{2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + -2.9710560942849127e+28, -5.942112188569825e+28, -1.188422437713965e+29, -2.37684487542793e+29, -4.75368975085586e+29, + }, + Counts: []float64{ + 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99}, - Sum: 10000, Count: 4952, Min: -6.338253001141147e+29, Max: 0}, + 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + }, + Sum: 10000, Count: 4952, Min: -6.338253001141147e+29, Max: 0, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, { name: "foo", value: &cWMetricHistogram{ - Values: []float64{-9.50737950171172e+29, -1.901475900342344e+30, -3.802951800684688e+30, -7.605903601369376e+30, + Values: []float64{ + -9.50737950171172e+29, -1.901475900342344e+30, -3.802951800684688e+30, -7.605903601369376e+30, -1.5211807202738753e+31, -3.0423614405477506e+31, -6.084722881095501e+31, -1.2169445762191002e+32, -2.4338891524382005e+32, -4.867778304876401e+32, -9.735556609752802e+32, -1.9471113219505604e+33, -3.894222643901121e+33, -7.788445287802241e+33, -1.5576890575604483e+34, -3.1153781151208966e+34, -6.230756230241793e+34, -1.2461512460483586e+35, - -2.4923024920967173e+35, -4.9846049841934345e+35, -9.969209968386869e+35}, + -2.4923024920967173e+35, -4.9846049841934345e+35, -9.969209968386869e+35, + }, Counts: []float64{100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120}, - Sum: 0, Count: 2310, Min: -9e+36, Max: -6.338253001141147e+29}, + Sum: 0, Count: 2310, Min: -9e+36, Max: -6.338253001141147e+29, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, }, @@ -1154,7 +1177,8 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat { name: "foo", value: &cWMetricHistogram{ - Values: []float64{8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, 2.7021597764222976e+16, + Values: []float64{ + 8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 5.404319552844595e+16, 2.7021597764222976e+16, 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 1.688849860263936e+15, 8.44424930131968e+14, 4.22212465065984e+14, 2.11106232532992e+14, 1.05553116266496e+14, 5.2776558133248e+13, 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 1.649267441664e+12, 8.24633720832e+11, 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, @@ -1163,22 +1187,29 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat 12288, 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5, -1.5, -3, -6, -12, -24, -48, -96, -192, -384, -768, -1536, -3072, -6144, -12288, -24576, -49152, -98304, -196608, -393216, -786432, -1.572864e+06, -3.145728e+06, -6.291456e+06, -1.2582912e+07, -2.5165824e+07, -5.0331648e+07, -1.00663296e+08, -2.01326592e+08, -4.02653184e+08, -8.05306368e+08, -1.610612736e+09, -3.221225472e+09, -6.442450944e+09, - -1.2884901888e+10, -2.5769803776e+10, -5.1539607552e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, -8.24633720832e+11}, - Counts: []float64{60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, + -1.2884901888e+10, -2.5769803776e+10, -5.1539607552e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, -8.24633720832e+11, + }, + Counts: []float64{ + 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, - 34, 35, 36, 37, 38, 39, 40}, - Sum: 1000, Count: 2650, Min: -1.099511627776e+12, Max: 9e+17}, + 34, 35, 36, 37, 38, 39, 40, + }, + Sum: 1000, Count: 2650, Min: -1.099511627776e+12, Max: 9e+17, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, { name: "foo", value: &cWMetricHistogram{ - Values: []float64{-1.649267441664e+12, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -5.2776558133248e+13, + Values: []float64{ + -1.649267441664e+12, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -5.2776558133248e+13, -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, -1.688849860263936e+15, -3.377699720527872e+15, -6.755399441055744e+15, -1.3510798882111488e+16, -2.7021597764222976e+16, -5.404319552844595e+16, -1.080863910568919e+17, -2.161727821137838e+17, - -4.323455642275676e+17, -8.646911284551352e+17}, + -4.323455642275676e+17, -8.646911284551352e+17, + }, Counts: []float64{41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60}, - Sum: 0, Count: 1010, Min: -9e+17, Max: -1.099511627776e+12}, + Sum: 0, Count: 1010, Min: -9e+17, Max: -1.099511627776e+12, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, }, @@ -1225,13 +1256,16 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, 1.2884901888e+10, 6.442450944e+09, 3.221225472e+09, 1.610612736e+09, 8.05306368e+08, 4.02653184e+08, 2.01326592e+08, 1.00663296e+08, 5.0331648e+07, 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06, 786432, 393216, 196608, 98304, 49152, 24576, 12288, - 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5}, + 6144, 3072, 1536, 768, 384, 192, 96, 48, 24, 12, 6, 3, 1.5, + }, Counts: []float64{ 100, 99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, - 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - Sum: 100000, Count: 5050, Min: 1, Max: 9e+36}, + 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, + }, + Sum: 100000, Count: 5050, Min: 1, Max: 9e+36, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, { @@ -1253,12 +1287,15 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat -1.8133887294219438e+24, -3.6267774588438875e+24, -7.253554917687775e+24, -1.450710983537555e+25, -2.90142196707511e+25, -5.80284393415022e+25, -1.160568786830044e+26, -2.321137573660088e+26, -4.642275147320176e+26, -9.284550294640352e+26, -1.8569100589280704e+27, -3.713820117856141e+27, -7.427640235712282e+27, -1.4855280471424563e+28, -2.9710560942849127e+28, - -5.942112188569825e+28, -1.188422437713965e+29, -2.37684487542793e+29, -4.75368975085586e+29, -9.50737950171172e+29}, + -5.942112188569825e+28, -1.188422437713965e+29, -2.37684487542793e+29, -4.75368975085586e+29, -9.50737950171172e+29, + }, Counts: []float64{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100}, - Sum: 0, Count: 5050, Min: -9e+36, Max: -1}, + 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, + }, + Sum: 0, Count: 5050, Min: -9e+36, Max: -1, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, }, @@ -1309,13 +1346,16 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 1.649267441664e+12, 8.24633720832e+11, 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 5.1539607552e+10, 2.5769803776e+10, 1.2884901888e+10, 6.442450944e+09, 3.221225472e+09, 1.610612736e+09, 8.05306368e+08, 4.02653184e+08, 2.01326592e+08, 1.00663296e+08, 5.0331648e+07, - 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06}, + 2.5165824e+07, 1.2582912e+07, 6.291456e+06, 3.145728e+06, 1.572864e+06, + }, Counts: []float64{ 120, 119, 118, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105, 104, 103, 102, 101, 100, 99, 98, 97, 96, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 85, 84, 83, 82, 81, 80, 79, 78, 77, 76, 75, 74, 73, 72, 71, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, - 27, 26, 25, 24, 23, 22, 21}, - Sum: 100000, Count: 7050, Min: 1048576, Max: 9e+36}, + 27, 26, 25, 24, 23, 22, 21, + }, + Sum: 100000, Count: 7050, Min: 1048576, Max: 9e+36, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, { @@ -1337,13 +1377,16 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat -1.1068046444225731e+20, -2.2136092888451462e+20, -4.4272185776902924e+20, -8.854437155380585e+20, -1.770887431076117e+21, -3.541774862152234e+21, -7.083549724304468e+21, -1.4167099448608936e+22, -2.833419889721787e+22, -5.666839779443574e+22, -1.1333679558887149e+23, -2.2667359117774297e+23, - -4.5334718235548594e+23}, + -4.5334718235548594e+23, + }, Counts: []float64{ 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79}, - Sum: 0, Count: 3372, Min: -6.044629098073146e+23, Max: 1048576}, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + }, + Sum: 0, Count: 3372, Min: -6.044629098073146e+23, Max: 1048576, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, { @@ -1358,11 +1401,14 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat -3.0423614405477506e+31, -6.084722881095501e+31, -1.2169445762191002e+32, -2.4338891524382005e+32, -4.867778304876401e+32, -9.735556609752802e+32, -1.9471113219505604e+33, -3.894222643901121e+33, -7.788445287802241e+33, -1.5576890575604483e+34, -3.1153781151208966e+34, -6.230756230241793e+34, -1.2461512460483586e+35, -2.4923024920967173e+35, -4.9846049841934345e+35, - -9.969209968386869e+35}, + -9.969209968386869e+35, + }, Counts: []float64{ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120}, - Sum: 0, Count: 4100, Min: -9e+36, Max: -6.044629098073146e+23}, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, + }, + Sum: 0, Count: 4100, Min: -9e+36, Max: -6.044629098073146e+23, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, }, @@ -1394,7 +1440,8 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat { name: "foo", value: &cWMetricHistogram{ - Values: []float64{8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 2.7021597764222976e+16, + Values: []float64{ + 8.646911284551352e+17, 4.323455642275676e+17, 2.161727821137838e+17, 1.080863910568919e+17, 2.7021597764222976e+16, 1.3510798882111488e+16, 6.755399441055744e+15, 3.377699720527872e+15, 8.44424930131968e+14, 4.22212465065984e+14, 2.11106232532992e+14, 1.05553116266496e+14, 2.6388279066624e+13, 1.3194139533312e+13, 6.597069766656e+12, 3.298534883328e+12, 8.24633720832e+11, 4.12316860416e+11, 2.06158430208e+11, 1.03079215104e+11, 2.5769803776e+10, 1.2884901888e+10, 6.442450944e+09, 3.221225472e+09, 8.05306368e+08, 4.02653184e+08, @@ -1404,10 +1451,14 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat -6.442450944e+09, -1.2884901888e+10, -2.5769803776e+10, -1.03079215104e+11, -2.06158430208e+11, -4.12316860416e+11, -8.24633720832e+11, -3.298534883328e+12, -6.597069766656e+12, -1.3194139533312e+13, -2.6388279066624e+13, -1.05553116266496e+14, -2.11106232532992e+14, -4.22212465065984e+14, -8.44424930131968e+14, -3.377699720527872e+15, -6.755399441055744e+15, -1.3510798882111488e+16, -2.7021597764222976e+16, -1.080863910568919e+17, -2.161727821137838e+17, - -4.323455642275676e+17, -8.646911284551352e+17}, - Counts: []float64{4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, - 2, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4}, - Sum: 1000, Count: 242, Min: -9e+17, Max: 9e+17}, + -4.323455642275676e+17, -8.646911284551352e+17, + }, + Counts: []float64{ + 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, 2, 1, 4, 3, + 2, 1, 2, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, + }, + Sum: 1000, Count: 242, Min: -9e+17, Max: 9e+17, + }, labels: map[string]string{oTellibDimensionKey: instrLibName, "label1": "value1"}, }, }, @@ -1429,7 +1480,6 @@ func TestCalculateDeltaDatapoints_ExponentialHistogramDataPointSliceWithSplitDat } }) } - } func TestIsStaleNaNInf_ExponentialHistogramDataPointSlice(t *testing.T) { diff --git a/exporter/awsemfexporter/emf_exporter.go b/exporter/awsemfexporter/emf_exporter.go index f840900b32d2..34b9eaf76856 100644 --- a/exporter/awsemfexporter/emf_exporter.go +++ b/exporter/awsemfexporter/emf_exporter.go @@ -76,7 +76,6 @@ func newEmfExporter(config *Config, set exporter.Settings) (*emfExporter, error) cwlogs.WithUserAgentExtras(userAgentExtras...), ) collectorIdentifier, err := uuid.NewRandom() - if err != nil { return nil, err } diff --git a/exporter/awsemfexporter/metric_translator.go b/exporter/awsemfexporter/metric_translator.go index 35ff1196d8de..8660a227b4c6 100644 --- a/exporter/awsemfexporter/metric_translator.go +++ b/exporter/awsemfexporter/metric_translator.go @@ -451,7 +451,6 @@ func translateCWMetricToEMF(cWMetric *cWMetrics, config *Config) (*cwlogs.Event, func translateGroupedMetricToEmf(groupedMetric *groupedMetric, config *Config, defaultLogStream string) (*cwlogs.Event, error) { cWMetric := translateGroupedMetricToCWMetric(groupedMetric, config) event, err := translateCWMetricToEMF(cWMetric, config) - if err != nil { return nil, err } diff --git a/exporter/awsemfexporter/metric_translator_test.go b/exporter/awsemfexporter/metric_translator_test.go index e86176fc1fa6..15c74767d7d8 100644 --- a/exporter/awsemfexporter/metric_translator_test.go +++ b/exporter/awsemfexporter/metric_translator_test.go @@ -431,7 +431,6 @@ func TestTranslateCWMetricToEMF(t *testing.T) { for name, tc := range testCases { t.Run(name, func(_ *testing.T) { config := &Config{ - // include valid json string, a non-existing key, and keys whose value are not json/string ParseJSONEncodedAttributeValues: []string{"kubernetes", "Sources", "NonExistingAttributeKey", "spanName", "spanCounter"}, Version: tc.emfVersion, @@ -1335,7 +1334,8 @@ func TestGroupedMetricToCWMeasurementsWithFilters(t *testing.T) { Dimensions: [][]string{{}}, MetricNameSelectors: []string{"metric(1|3)"}, }, - }, []cWMeasurement{ + }, + []cWMeasurement{ { Namespace: namespace, Dimensions: [][]string{{}}, @@ -2194,123 +2194,121 @@ type logGroupStreamTest struct { outLogStreamName string } -var ( - logGroupStreamTestCases = []logGroupStreamTest{ - { - name: "log_group_stream_expect_same", - inputMetrics: generateTestMetrics(testMetric{ - metricNames: []string{"metric_1", "metric_2"}, - metricValues: [][]float64{{100}, {4}}, - }), - inLogGroupName: "test-log-group", - inLogStreamName: "test-log-stream", - outLogGroupName: "test-log-group", - outLogStreamName: "test-log-stream", - }, - { - name: "log_group_pattern_from_resource", - inputMetrics: generateTestMetrics(testMetric{ - metricNames: []string{"metric_1", "metric_2"}, - metricValues: [][]float64{{100}, {4}}, - resourceAttributeMap: map[string]any{ - "ClusterName": "test-cluster", - "PodName": "test-pod", - }, - }), - inLogGroupName: "test-log-group-{ClusterName}", - inLogStreamName: "test-log-stream", - outLogGroupName: "test-log-group-test-cluster", - outLogStreamName: "test-log-stream", - }, - { - name: "log_stream_pattern_from_resource", - inputMetrics: generateTestMetrics(testMetric{ - metricNames: []string{"metric_1", "metric_2"}, - metricValues: [][]float64{{100}, {4}}, - resourceAttributeMap: map[string]any{ - "ClusterName": "test-cluster", - "PodName": "test-pod", - }, - }), - inLogGroupName: "test-log-group", - inLogStreamName: "test-log-stream-{PodName}", - outLogGroupName: "test-log-group", - outLogStreamName: "test-log-stream-test-pod", - }, - { - name: "log_group_pattern_from_label", - inputMetrics: generateTestMetrics(testMetric{ - metricNames: []string{"metric_1", "metric_2"}, - metricValues: [][]float64{{100}, {4}}, - attributeMap: map[string]any{ - "ClusterName": "test-cluster", - "PodName": "test-pod", - }, - }), - inLogGroupName: "test-log-group-{ClusterName}", - inLogStreamName: "test-log-stream", - outLogGroupName: "test-log-group-test-cluster", - outLogStreamName: "test-log-stream", - }, - { - name: "log_stream_pattern_from_label", - inputMetrics: generateTestMetrics(testMetric{ - metricNames: []string{"metric_1", "metric_2"}, - metricValues: [][]float64{{100}, {4}}, - attributeMap: map[string]any{ - "ClusterName": "test-cluster", - "PodName": "test-pod", - }, - }), - inLogGroupName: "test-log-group", - inLogStreamName: "test-log-stream-{PodName}", - outLogGroupName: "test-log-group", - outLogStreamName: "test-log-stream-test-pod", - }, - { - name: "config_pattern_from_both_attributes", - inputMetrics: generateTestMetrics(testMetric{ - metricNames: []string{"metric_1", "metric_2"}, - metricValues: [][]float64{{100}, {4}}, - resourceAttributeMap: map[string]any{ - "ClusterName": "test-cluster", - }, - attributeMap: map[string]any{ - "PodName": "test-pod", - }, - }), - inLogGroupName: "test-log-group-{ClusterName}", - inLogStreamName: "test-log-stream-{PodName}", - outLogGroupName: "test-log-group-test-cluster", - outLogStreamName: "test-log-stream-test-pod", - }, - { - name: "config_pattern_missing_from_both_attributes", - inputMetrics: generateTestMetrics(testMetric{ - metricNames: []string{"metric_1", "metric_2"}, - metricValues: [][]float64{{100}, {4}}, - }), - inLogGroupName: "test-log-group-{ClusterName}", - inLogStreamName: "test-log-stream-{PodName}", - outLogGroupName: "test-log-group-undefined", - outLogStreamName: "test-log-stream-undefined", - }, - { - name: "config_pattern_group_missing_stream_present", - inputMetrics: generateTestMetrics(testMetric{ - metricNames: []string{"metric_1", "metric_2"}, - metricValues: [][]float64{{100}, {4}}, - attributeMap: map[string]any{ - "PodName": "test-pod", - }, - }), - inLogGroupName: "test-log-group-{ClusterName}", - inLogStreamName: "test-log-stream-{PodName}", - outLogGroupName: "test-log-group-undefined", - outLogStreamName: "test-log-stream-test-pod", - }, - } -) +var logGroupStreamTestCases = []logGroupStreamTest{ + { + name: "log_group_stream_expect_same", + inputMetrics: generateTestMetrics(testMetric{ + metricNames: []string{"metric_1", "metric_2"}, + metricValues: [][]float64{{100}, {4}}, + }), + inLogGroupName: "test-log-group", + inLogStreamName: "test-log-stream", + outLogGroupName: "test-log-group", + outLogStreamName: "test-log-stream", + }, + { + name: "log_group_pattern_from_resource", + inputMetrics: generateTestMetrics(testMetric{ + metricNames: []string{"metric_1", "metric_2"}, + metricValues: [][]float64{{100}, {4}}, + resourceAttributeMap: map[string]any{ + "ClusterName": "test-cluster", + "PodName": "test-pod", + }, + }), + inLogGroupName: "test-log-group-{ClusterName}", + inLogStreamName: "test-log-stream", + outLogGroupName: "test-log-group-test-cluster", + outLogStreamName: "test-log-stream", + }, + { + name: "log_stream_pattern_from_resource", + inputMetrics: generateTestMetrics(testMetric{ + metricNames: []string{"metric_1", "metric_2"}, + metricValues: [][]float64{{100}, {4}}, + resourceAttributeMap: map[string]any{ + "ClusterName": "test-cluster", + "PodName": "test-pod", + }, + }), + inLogGroupName: "test-log-group", + inLogStreamName: "test-log-stream-{PodName}", + outLogGroupName: "test-log-group", + outLogStreamName: "test-log-stream-test-pod", + }, + { + name: "log_group_pattern_from_label", + inputMetrics: generateTestMetrics(testMetric{ + metricNames: []string{"metric_1", "metric_2"}, + metricValues: [][]float64{{100}, {4}}, + attributeMap: map[string]any{ + "ClusterName": "test-cluster", + "PodName": "test-pod", + }, + }), + inLogGroupName: "test-log-group-{ClusterName}", + inLogStreamName: "test-log-stream", + outLogGroupName: "test-log-group-test-cluster", + outLogStreamName: "test-log-stream", + }, + { + name: "log_stream_pattern_from_label", + inputMetrics: generateTestMetrics(testMetric{ + metricNames: []string{"metric_1", "metric_2"}, + metricValues: [][]float64{{100}, {4}}, + attributeMap: map[string]any{ + "ClusterName": "test-cluster", + "PodName": "test-pod", + }, + }), + inLogGroupName: "test-log-group", + inLogStreamName: "test-log-stream-{PodName}", + outLogGroupName: "test-log-group", + outLogStreamName: "test-log-stream-test-pod", + }, + { + name: "config_pattern_from_both_attributes", + inputMetrics: generateTestMetrics(testMetric{ + metricNames: []string{"metric_1", "metric_2"}, + metricValues: [][]float64{{100}, {4}}, + resourceAttributeMap: map[string]any{ + "ClusterName": "test-cluster", + }, + attributeMap: map[string]any{ + "PodName": "test-pod", + }, + }), + inLogGroupName: "test-log-group-{ClusterName}", + inLogStreamName: "test-log-stream-{PodName}", + outLogGroupName: "test-log-group-test-cluster", + outLogStreamName: "test-log-stream-test-pod", + }, + { + name: "config_pattern_missing_from_both_attributes", + inputMetrics: generateTestMetrics(testMetric{ + metricNames: []string{"metric_1", "metric_2"}, + metricValues: [][]float64{{100}, {4}}, + }), + inLogGroupName: "test-log-group-{ClusterName}", + inLogStreamName: "test-log-stream-{PodName}", + outLogGroupName: "test-log-group-undefined", + outLogStreamName: "test-log-stream-undefined", + }, + { + name: "config_pattern_group_missing_stream_present", + inputMetrics: generateTestMetrics(testMetric{ + metricNames: []string{"metric_1", "metric_2"}, + metricValues: [][]float64{{100}, {4}}, + attributeMap: map[string]any{ + "PodName": "test-pod", + }, + }), + inLogGroupName: "test-log-group-{ClusterName}", + inLogStreamName: "test-log-stream-{PodName}", + outLogGroupName: "test-log-group-undefined", + outLogStreamName: "test-log-stream-test-pod", + }, +} func TestTranslateOtToGroupedMetricForLogGroupAndStream(t *testing.T) { for _, test := range logGroupStreamTestCases {