Merge branch 'main' into expose-physical-disk-collector-from-windows-…

…exporter

.github/workflows/helm-release.yml

@@ -22,7 +22,7 @@ jobs:
           path: source
 
       - name: Install chart-testing
-        uses: helm/[email protected].0
+        uses: helm/[email protected].1
 
       - name: List changed charts
         id: list-changed

.github/workflows/helm-test.yml

@@ -57,7 +57,7 @@ jobs:
           check-latest: true
 
       - name: Install chart-testing
-        uses: helm/[email protected].0
+        uses: helm/[email protected].1
 
       - name: Determine changed charts
         id: list-changed

CHANGELOG.md

@@ -10,6 +10,17 @@ internal API changes are not present.
 Main (unreleased)
 -----------------
 
+### Breaking changes
+
+- `otelcol.receiver.prometheus` will drop all `otel_scope_info` metrics when converting them to OTLP. (@wildum)
+  - If the `otel_scope_info` metric has labels `otel_scope_name` and `otel_scope_version`,
+    their values will be used to set OTLP Instrumentation Scope name and  version respectively. 
+  - Labels of `otel_scope_info` metrics other than `otel_scope_name` and `otel_scope_version` 
+    are added as scope attributes with the matching name and version.
+
+- The `target` block in `prometheus.exporter.blackbox` requires a mandatory `name`
+  argument instead of a block label. (@hainenber)
+
 ### Enhancements
 
 - Flow Windows service: Support environment variables. (@jkroepke)
@@ -32,8 +43,25 @@ Main (unreleased)
 
 - `pyroscope.ebpf` support python on arm64 platforms. (@korniltsev)
 
+- `otelcol.receiver.prometheus` does not drop histograms without buckets anymore. (@wildum)
+
+- Added exemplars support to `otelcol.receiver.prometheus`. (@wildum)
+- `mimir.rules.kubernetes` may now retry its startup on failure. (@hainenber)
+
 - Added links between compatible components in the documentation to make it
   easier to discover them. (@thampiotr)
+
+- Allow defining `HTTPClientConfig` for `discovery.ec2`. (@cmbrad)
+
+- The `remote.http` component can optionally define a request body. (@tpaschalis)
+
+- Added support for `loki.write` to flush WAL on agent shutdown. (@thepalbi)
+
+- Add support for `integrations-next` static to flow config conversion. (@erikbaranowski)
+
+- Add support for passing extra arguments to the static converter such as `-config.expand-env`. (@erikbaranowski)
+
+- Added 'country' mmdb-type to log pipeline-stage geoip. (@superstes)
 
 - Expose `physical_disk` collector from `windows_exporter` v0.24.0 to 
   Flow configuration. (@hainenber)
@@ -44,6 +72,20 @@ Main (unreleased)
 
 - Update `pyroscope.ebpf` to produce more optimal pprof profiles for python processes https://github.com/grafana/pyroscope/pull/2788 (@korniltsev)
 
+- In Static mode's `traces` subsystem, `spanmetrics` used to be generated prior to load balancing.
+  This could lead to inaccurate metrics. This issue only affects Agents using both `spanmetrics` and 
+  `load_balancing`, when running in a load balanced cluster with more than one Agent instance. (@ptodev)
+
+- Fixes `loki.source.docker` a behavior that synced an incomplete list of targets to the tailer manager. (@FerdinandvHagen)
+
+- Fixes `otelcol.connector.servicegraph` store ttl default value from 2ms to 2s. (@rlankfo)
+
+- Add staleness tracking to labelstore to reduce memory usage. (@mattdurham)
+
+### Other changes
+
+- Bump github.com/IBM/sarama from v1.41.2 to v1.42.1
+
 v0.38.1 (2023-11-30)
 --------------------
 
@@ -118,6 +160,8 @@ v0.38.0 (2023-11-21)
 
 - Added support for python profiling to `pyroscope.ebpf` component. (@korniltsev)
 
+- Added support for native Prometheus histograms to `otelcol.exporter.prometheus` (@wildum)
+
 - Windows Flow Installer: Add /CONFIG /DISABLEPROFILING and /DISABLEREPORTING flag (@jkroepke)
 
 - Add queueing logs remote write client for `loki.write` when WAL is enabled. (@thepalbi)

cmd/grafana-agent-operator/Dockerfile

@@ -22,15 +22,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     RELEASE_BUILD=${RELEASE_BUILD} VERSION=${VERSION} \
     make operator
 
-FROM ubuntu:lunar
+FROM ubuntu:mantic
 
 LABEL org.opencontainers.image.source="https://github.com/grafana/agent"
 
 # Install dependencies needed at runtime.
 RUN <<EOF
   apt-get update
   apt-get install -qy libsystemd-dev tzdata ca-certificates
-  apt-get upgrade -y
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 EOF
 

cmd/grafana-agent/Dockerfile

@@ -30,15 +30,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     GOEXPERIMENT=${GOEXPERIMENT} \
     make agent
 
-FROM ubuntu:lunar
+FROM ubuntu:mantic
 
 LABEL org.opencontainers.image.source="https://github.com/grafana/agent"
 
 # Install dependencies needed at runtime.
 RUN <<EOF
   apt-get update
   apt-get install -qy libsystemd-dev tzdata ca-certificates
-  apt-get upgrade -y
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 EOF
 

cmd/grafana-agentctl/Dockerfile

@@ -23,15 +23,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     GO_TAGS="netgo promtail_journal_enabled" \
     make agentctl
 
-FROM ubuntu:lunar
+FROM ubuntu:mantic
 
 LABEL org.opencontainers.image.source="https://github.com/grafana/agent"
 
 # Install dependencies needed at runtime.
 RUN <<EOF
   apt-get update
   apt-get install -qy libsystemd-dev tzdata ca-certificates
-  apt-get upgrade -y
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 EOF
 

cmd/internal/flowmode/cmd_convert.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/grafana/agent/converter"
 	convert_diag "github.com/grafana/agent/converter/diag"
@@ -20,6 +21,7 @@ func convertCommand() *cobra.Command {
 		output:       "",
 		sourceFormat: "",
 		bypassErrors: false,
+		extraArgs:    "",
 	}
 
 	cmd := &cobra.Command{
@@ -41,7 +43,11 @@ The -f flag can be used to specify the format we are converting from.
 
 The -b flag can be used to bypass errors. Errors are defined as 
 non-critical issues identified during the conversion where an
-output can still be generated.`,
+output can still be generated.
+
+The -e flag can be used to pass extra arguments to the converter
+which were used by the original format. Multiple arguments can be passed
+by separating them with a space.`,
 		Args:         cobra.RangeArgs(0, 1),
 		SilenceUsage: true,
 
@@ -71,6 +77,7 @@ output can still be generated.`,
 	cmd.Flags().StringVarP(&f.report, "report", "r", f.report, "The filepath and filename where the report is written.")
 	cmd.Flags().StringVarP(&f.sourceFormat, "source-format", "f", f.sourceFormat, fmt.Sprintf("The format of the source file. Supported formats: %s.", supportedFormatsList()))
 	cmd.Flags().BoolVarP(&f.bypassErrors, "bypass-errors", "b", f.bypassErrors, "Enable bypassing errors when converting")
+	cmd.Flags().StringVarP(&f.extraArgs, "extra-args", "e", f.extraArgs, "Extra arguments from the original format used by the converter. Multiple arguments can be passed by separating them with a space.")
 	return cmd
 }
 
@@ -79,6 +86,7 @@ type flowConvert struct {
 	report       string
 	sourceFormat string
 	bypassErrors bool
+	extraArgs    string
 }
 
 func (fc *flowConvert) Run(configFile string) error {
@@ -112,7 +120,12 @@ func convert(r io.Reader, fc *flowConvert) error {
 		return err
 	}
 
-	riverBytes, diags := converter.Convert(inputBytes, converter.Input(fc.sourceFormat), []string{})
+	ea, err := parseExtraArgs(fc.extraArgs)
+	if err != nil {
+		return err
+	}
+
+	riverBytes, diags := converter.Convert(inputBytes, converter.Input(fc.sourceFormat), ea)
 	err = generateConvertReport(diags, fc)
 	if err != nil {
 		return err
@@ -174,3 +187,65 @@ func supportedFormatsList() string {
 	}
 	return strings.Join(ret, ", ")
 }
+
+func parseExtraArgs(extraArgs string) ([]string, error) {
+	var result []string
+	if extraArgs == "" {
+		return result, nil
+	}
+
+	arguments := strings.Fields(extraArgs)
+	for i, arg := range arguments {
+		fs := pflag.NewFlagSet("extra-args", pflag.ExitOnError)
+		fs.ParseErrorsWhitelist.UnknownFlags = true
+		keyStartIndex := 0
+		doParseFlagValue := false
+
+		// Split the argument into key and value.
+		splitArgs := strings.SplitN(arg, "=", 2)
+
+		// Append the key to the result.
+		result = append(result, splitArgs[0])
+
+		// If the flag has a value, add it to the FlagSet for parsing.
+		if len(splitArgs) == 2 && splitArgs[1] != "" {
+			doParseFlagValue = true
+			if arg[1] == '-' { // longhand flag, ie. --flag=value
+				keyStartIndex = 2
+			} else if arg[0] == '-' { // shorthand flag, ie. -f=value
+				keyStartIndex = 1
+			} else { // invalid flag that was split on '=' but has no dashes in the key
+				return nil, fmt.Errorf("invalid flag found: %s", arg)
+			}
+		}
+
+		if doParseFlagValue {
+			result = append(result, "")
+			lastIndex := len(result) - 1
+			key := splitArgs[0][keyStartIndex:]
+			if keyStartIndex == 2 {
+				fs.StringVar(&result[lastIndex], key, result[lastIndex], "")
+			} else {
+				// static mode uses keys with a single dash. We need to sanitize them here.
+				if len(key) != 1 {
+					arguments[i] = "-" + arguments[i]
+					fs.StringVar(&result[lastIndex], key, result[lastIndex], "")
+				} else {
+					fs.StringVarP(&result[lastIndex], "", key, result[lastIndex], "")
+				}
+			}
+
+			// We must parse the flag here because the pointer to the array element
+			// &result[lastIndex] is overridden by the next iteration of the loop.
+			// This can be improved if we preallocate the array, however we won't
+			// know the final length without analyzing the arguments so there
+			// is some complexity in doing so.
+			err := fs.Parse(arguments)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return result, nil
+}

cmd/internal/flowmode/cmd_convert_test.go

@@ -0,0 +1,86 @@
+package flowmode
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseExtraArgs(t *testing.T) {
+	type testCase struct {
+		name          string
+		extraArgs     string
+		expected      []string
+		expectedError string
+	}
+
+	var testCases = []testCase{
+		{
+			name:      "integrations next with env vars",
+			extraArgs: "-enable-features=integrations-next -config.expand-env",
+			expected:  []string{"-enable-features", "integrations-next", "-config.expand-env"},
+		},
+		{
+			name:      "longhand",
+			extraArgs: "--key=value",
+			expected:  []string{"--key", "value"},
+		},
+		{
+			name:      "shorthand",
+			extraArgs: "-k=value",
+			expected:  []string{"-k", "value"},
+		},
+		{
+			name:      "bool longhand",
+			extraArgs: "--boolVariable",
+			expected:  []string{"--boolVariable"},
+		},
+		{
+			name:      "bool shorthand",
+			extraArgs: "-b",
+			expected:  []string{"-b"},
+		},
+		{
+			name:      "combo",
+			extraArgs: "--key=value -k=value --boolVariable -b",
+			expected:  []string{"--key", "value", "-k", "value", "--boolVariable", "-b"},
+		},
+		{
+			name:      "spaced",
+			extraArgs: "--key value",
+			expected:  []string{"--key", "value"},
+		},
+		{
+			name:      "value with equals",
+			extraArgs: `--key="foo=bar"`,
+			expected:  []string{"--key", `"foo=bar"`},
+		},
+		{
+			name:      "no value",
+			extraArgs: "--key=",
+			expected:  []string{"--key"},
+		},
+		{
+			name:      "no dashes",
+			extraArgs: "key",
+			expected:  []string{"key"},
+		},
+		{
+			name:          "no dashes with value",
+			extraArgs:     "key=value",
+			expectedError: "invalid flag found: key=value",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			res, err := parseExtraArgs(tc.extraArgs)
+			if tc.expectedError != "" {
+				require.EqualError(t, err, tc.expectedError)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, tc.expected, res)
+		})
+	}
+}