diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b27dfa59319e..80540ae4848e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -18,6 +18,6 @@ jobs: with: go-version: "1.21" - name: Set OTEL Exporter Endpoint - run: echo "OTEL_EXPORTER_ENDPOINT=http://172.17.0.1:8080" >> $GITHUB_ENV + run: echo "OTEL_EXPORTER_ENDPOINT=172.17.0.1:4318" >> $GITHUB_ENV - name: Run tests run: make integration-test diff --git a/CHANGELOG.md b/CHANGELOG.md index 377b500196ec..ff28693a0c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,23 +12,24 @@ Main (unreleased) ### Features -- Added support for `loki.write` to flush WAL on agent shutdown. (@thepalbi) +- Agent Management: Introduce support for templated configuration. (@jcreixell) -v0.38.0-rc.1 (2023-11-20) -------------------------- +- Added support for `loki.write` to flush WAL on agent shutdown. (@thepalbi) ### Enhancements - -- Allow agent to start with `module.git` config if cached before. (@hainenber) + +- Flow Windows service: Support environment variables. (@jkroepke) ### Bugfixes -- Fix default configuration file `grafana-agent-flow.river` used in downstream packages. (@bricewge) +- Permit `X-Faro-Session-ID` header in CORS requests for the `faro.receiver` + component (flow mode) and the `app_agent_receiver` integration (static mode). + (@cedricziel) -- Fix converter output for prometheus.exporter.windows to not unnecessarily add empty blocks. (@erikbaranowski) +- Fix issue with windows_exporter defaults not being set correctly. (@mattdurham) -v0.38.0-rc.0 (2023-11-16) -------------------------- +v0.38.0 (2023-11-21) +-------------------- ### Breaking changes @@ -123,6 +124,8 @@ v0.38.0-rc.0 (2023-11-16) - Updated windows exporter to use prometheus-community/windows_exporter commit 1836cd1. (@mattdurham) +- Allow agent to start with `module.git` config if cached before. (@hainenber) + ### Bugfixes - Set exit code 1 on grafana-agentctl non-runnable command. (@fgouteroux) @@ -174,6 +177,12 @@ v0.38.0-rc.0 (2023-11-16) - Added Kubernetes service resolver to static node's loadbalancing exporter and to Flow's `otelcol.exporter.loadbalancing`. (@ptodev) +- Fix default configuration file `grafana-agent-flow.river` used in downstream + packages. (@bricewge) + +- Fix converter output for prometheus.exporter.windows to not unnecessarily add + empty blocks. (@erikbaranowski) + ### Other changes - Bump `mysqld_exporter` version to v0.15.0. (@marctc) diff --git a/cmd/grafana-agent-service/config_windows.go b/cmd/grafana-agent-service/config_windows.go index 961e95ad5e51..e4d712585986 100644 --- a/cmd/grafana-agent-service/config_windows.go +++ b/cmd/grafana-agent-service/config_windows.go @@ -16,6 +16,11 @@ type config struct { // not included. Args []string + // Environment holds environment variables for the Grafana Agent service. + // Each item represents an environment variable in form "key=value". + // All environments variables from the current process with be merged into Environment + Environment []string + // WorkingDirectory points to the working directory to run the Grafana Agent // binary from. WorkingDirectory string @@ -42,9 +47,15 @@ func loadConfig() (*config, error) { return nil, fmt.Errorf("failed to retrieve key Arguments: %w", err) } + env, _, err := agentKey.GetStringsValue("Environment") + if err != nil { + return nil, fmt.Errorf("failed to retrieve key Environment: %w", err) + } + return &config{ ServicePath: servicePath, Args: args, + Environment: env, WorkingDirectory: filepath.Dir(servicePath), }, nil } diff --git a/cmd/grafana-agent-service/main_windows.go b/cmd/grafana-agent-service/main_windows.go index f3660f73e7cc..a94a71228516 100644 --- a/cmd/grafana-agent-service/main_windows.go +++ b/cmd/grafana-agent-service/main_windows.go @@ -29,9 +29,10 @@ func main() { } cfg := serviceManagerConfig{ - Path: managerConfig.ServicePath, - Args: managerConfig.Args, - Dir: managerConfig.WorkingDirectory, + Path: managerConfig.ServicePath, + Args: managerConfig.Args, + Environment: managerConfig.Environment, + Dir: managerConfig.WorkingDirectory, // Send logs directly to the event logger. Stdout: logger, diff --git a/cmd/grafana-agent-service/service.go b/cmd/grafana-agent-service/service.go index 10060e19cc28..c302a98a1a36 100644 --- a/cmd/grafana-agent-service/service.go +++ b/cmd/grafana-agent-service/service.go @@ -24,6 +24,9 @@ type serviceManagerConfig struct { // Args of the binary to run, not including the command itself. Args []string + // Environment of the binary to run, including the command environment itself. + Environment []string + // Dir specifies the working directory to run the binary from. If Dir is // empty, the working directory of the current process is used. Dir string @@ -84,5 +87,7 @@ func (svc *serviceManager) buildCommand(ctx context.Context) *exec.Cmd { cmd.Dir = svc.cfg.Dir cmd.Stdout = svc.cfg.Stdout cmd.Stderr = svc.cfg.Stderr + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, svc.cfg.Environment...) return cmd } diff --git a/cmd/grafana-agent-service/service_test.go b/cmd/grafana-agent-service/service_test.go index d6389132071b..a1d3124b034c 100644 --- a/cmd/grafana-agent-service/service_test.go +++ b/cmd/grafana-agent-service/service_test.go @@ -30,8 +30,9 @@ func Test_serviceManager(t *testing.T) { listenHost := getListenHost(t) mgr := newServiceManager(l, serviceManagerConfig{ - Path: serviceBinary, - Args: []string{"-listen-addr", listenHost}, + Path: serviceBinary, + Args: []string{"-listen-addr", listenHost}, + Environment: []string{"LISTEN=" + listenHost}, }) go mgr.Run(componenttest.TestContext(t)) @@ -40,6 +41,12 @@ func Test_serviceManager(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte("Hello, world!"), resp) }) + + util.Eventually(t, func(t require.TestingT) { + resp, err := makeServiceRequest(listenHost, "/echo/env", nil) + require.NoError(t, err) + require.Contains(t, string(resp), "LISTEN="+listenHost) + }) }) t.Run("terminates service binary", func(t *testing.T) { diff --git a/cmd/grafana-agent-service/testdata/example_service.go b/cmd/grafana-agent-service/testdata/example_service.go index ff11e61dfd85..64808b1b8608 100644 --- a/cmd/grafana-agent-service/testdata/example_service.go +++ b/cmd/grafana-agent-service/testdata/example_service.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "os" + "strings" ) func main() { @@ -46,6 +47,9 @@ func run() error { mux.HandleFunc("/echo/response", func(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(w, r.Body) }) + mux.HandleFunc("/echo/env", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(strings.Join(os.Environ(), "\n"))) + }) srv := &http.Server{Handler: mux} _ = srv.Serve(lis) diff --git a/component/faro/receiver/handler.go b/component/faro/receiver/handler.go index fb8511e0bbde..636f00859e2b 100644 --- a/component/faro/receiver/handler.go +++ b/component/faro/receiver/handler.go @@ -69,7 +69,7 @@ func (h *handler) Update(args ServerArguments) { if len(args.CORSAllowedOrigins) > 0 { h.cors = cors.New(cors.Options{ AllowedOrigins: args.CORSAllowedOrigins, - AllowedHeaders: []string{apiKeyHeader, "content-type"}, + AllowedHeaders: []string{apiKeyHeader, "content-type", "x-faro-session-id"}, }) } else { h.cors = nil // Disable cors. diff --git a/component/prometheus/exporter/windows/config.go b/component/prometheus/exporter/windows/config.go index 674d355011c5..cc4cb20e4b17 100644 --- a/component/prometheus/exporter/windows/config.go +++ b/component/prometheus/exporter/windows/config.go @@ -6,72 +6,6 @@ import ( windows_integration "github.com/grafana/agent/pkg/integrations/windows_exporter" ) -// DefaultArguments holds non-zero default options for Arguments when it is -// unmarshaled from YAML. -// -// Some defaults are populated from init functions in the github.com/grafana/agent/pkg/integrations/windows_exporter package. - -var DefaultArguments = Arguments{ - EnabledCollectors: strings.Split(windows_integration.DefaultConfig.EnabledCollectors, ","), - Dfsr: DfsrConfig{ - SourcesEnabled: strings.Split(windows_integration.DefaultConfig.Dfsr.SourcesEnabled, ","), - }, - Exchange: ExchangeConfig{ - EnabledList: strings.Split(windows_integration.DefaultConfig.Exchange.EnabledList, ","), - }, - IIS: IISConfig{ - AppBlackList: windows_integration.DefaultConfig.IIS.AppBlackList, - AppWhiteList: windows_integration.DefaultConfig.IIS.AppWhiteList, - SiteBlackList: windows_integration.DefaultConfig.IIS.SiteBlackList, - SiteWhiteList: windows_integration.DefaultConfig.IIS.SiteWhiteList, - AppInclude: windows_integration.DefaultConfig.IIS.AppInclude, - AppExclude: windows_integration.DefaultConfig.IIS.AppExclude, - SiteInclude: windows_integration.DefaultConfig.IIS.SiteInclude, - SiteExclude: windows_integration.DefaultConfig.IIS.SiteExclude, - }, - LogicalDisk: LogicalDiskConfig{ - BlackList: windows_integration.DefaultConfig.LogicalDisk.BlackList, - WhiteList: windows_integration.DefaultConfig.LogicalDisk.WhiteList, - Include: windows_integration.DefaultConfig.LogicalDisk.Include, - Exclude: windows_integration.DefaultConfig.LogicalDisk.Exclude, - }, - MSMQ: MSMQConfig{ - Where: windows_integration.DefaultConfig.MSMQ.Where, - }, - MSSQL: MSSQLConfig{ - EnabledClasses: strings.Split(windows_integration.DefaultConfig.MSSQL.EnabledClasses, ","), - }, - Network: NetworkConfig{ - BlackList: windows_integration.DefaultConfig.Network.BlackList, - WhiteList: windows_integration.DefaultConfig.Network.WhiteList, - Include: windows_integration.DefaultConfig.Network.Include, - Exclude: windows_integration.DefaultConfig.Network.Exclude, - }, - Process: ProcessConfig{ - BlackList: windows_integration.DefaultConfig.Process.BlackList, - WhiteList: windows_integration.DefaultConfig.Process.WhiteList, - Include: windows_integration.DefaultConfig.Process.Include, - Exclude: windows_integration.DefaultConfig.Process.Exclude, - }, - ScheduledTask: ScheduledTaskConfig{ - Include: windows_integration.DefaultConfig.ScheduledTask.Include, - Exclude: windows_integration.DefaultConfig.ScheduledTask.Exclude, - }, - Service: ServiceConfig{ - UseApi: windows_integration.DefaultConfig.Service.UseApi, - Where: windows_integration.DefaultConfig.Service.Where, - }, - SMTP: SMTPConfig{ - BlackList: windows_integration.DefaultConfig.SMTP.BlackList, - WhiteList: windows_integration.DefaultConfig.SMTP.WhiteList, - Include: windows_integration.DefaultConfig.SMTP.Include, - Exclude: windows_integration.DefaultConfig.SMTP.Exclude, - }, - TextFile: TextFileConfig{ - TextFileDirectory: windows_integration.DefaultConfig.TextFile.TextFileDirectory, - }, -} - // Arguments is used for controlling for this exporter. type Arguments struct { // Collectors to mark as enabled @@ -92,11 +26,6 @@ type Arguments struct { TextFile TextFileConfig `river:"text_file,block,optional"` } -// SetToDefault implements river.Defaulter. -func (a *Arguments) SetToDefault() { - *a = DefaultArguments -} - // Convert converts the component's Arguments to the integration's Config. func (a *Arguments) Convert() *windows_integration.Config { return &windows_integration.Config{ diff --git a/component/prometheus/exporter/windows/config_default_windows_test.go b/component/prometheus/exporter/windows/config_default_windows_test.go index c17f6e33fa60..9fddd1d635eb 100644 --- a/component/prometheus/exporter/windows/config_default_windows_test.go +++ b/component/prometheus/exporter/windows/config_default_windows_test.go @@ -1,10 +1,8 @@ package windows import ( - "strings" "testing" - windows_integration "github.com/grafana/agent/pkg/integrations/windows_exporter" "github.com/grafana/river" "github.com/stretchr/testify/require" ) @@ -14,26 +12,26 @@ func TestRiverUnmarshalWithDefaultConfig(t *testing.T) { err := river.Unmarshal([]byte(""), &args) require.NoError(t, err) - require.Equal(t, strings.Split(windows_integration.DefaultConfig.EnabledCollectors, ","), args.EnabledCollectors) - require.Equal(t, strings.Split(windows_integration.DefaultConfig.Dfsr.SourcesEnabled, ","), args.Dfsr.SourcesEnabled) - require.Equal(t, strings.Split(windows_integration.DefaultConfig.Exchange.EnabledList, ","), args.Exchange.EnabledList) - require.Equal(t, windows_integration.DefaultConfig.IIS.AppExclude, args.IIS.AppExclude) - require.Equal(t, windows_integration.DefaultConfig.IIS.AppInclude, args.IIS.AppInclude) - require.Equal(t, windows_integration.DefaultConfig.IIS.SiteExclude, args.IIS.SiteExclude) - require.Equal(t, windows_integration.DefaultConfig.IIS.SiteInclude, args.IIS.SiteInclude) - require.Equal(t, windows_integration.DefaultConfig.LogicalDisk.Exclude, args.LogicalDisk.Exclude) - require.Equal(t, windows_integration.DefaultConfig.LogicalDisk.Include, args.LogicalDisk.Include) - require.Equal(t, windows_integration.DefaultConfig.MSMQ.Where, args.MSMQ.Where) - require.Equal(t, strings.Split(windows_integration.DefaultConfig.MSSQL.EnabledClasses, ","), args.MSSQL.EnabledClasses) - require.Equal(t, windows_integration.DefaultConfig.Network.Exclude, args.Network.Exclude) - require.Equal(t, windows_integration.DefaultConfig.Network.Include, args.Network.Include) - require.Equal(t, windows_integration.DefaultConfig.Process.Exclude, args.Process.Exclude) - require.Equal(t, windows_integration.DefaultConfig.Process.Include, args.Process.Include) - require.Equal(t, windows_integration.DefaultConfig.ScheduledTask.Exclude, args.ScheduledTask.Exclude) - require.Equal(t, windows_integration.DefaultConfig.ScheduledTask.Include, args.ScheduledTask.Include) - require.Equal(t, windows_integration.DefaultConfig.Service.UseApi, args.Service.UseApi) - require.Equal(t, windows_integration.DefaultConfig.Service.Where, args.Service.Where) - require.Equal(t, windows_integration.DefaultConfig.SMTP.Exclude, args.SMTP.Exclude) - require.Equal(t, windows_integration.DefaultConfig.SMTP.Include, args.SMTP.Include) - require.Equal(t, windows_integration.DefaultConfig.TextFile.TextFileDirectory, args.TextFile.TextFileDirectory) + require.Equal(t, DefaultArguments.EnabledCollectors, args.EnabledCollectors) + require.Equal(t, DefaultArguments.Dfsr.SourcesEnabled, args.Dfsr.SourcesEnabled) + require.Equal(t, DefaultArguments.Exchange.EnabledList, args.Exchange.EnabledList) + require.Equal(t, DefaultArguments.IIS.AppExclude, args.IIS.AppExclude) + require.Equal(t, DefaultArguments.IIS.AppInclude, args.IIS.AppInclude) + require.Equal(t, DefaultArguments.IIS.SiteExclude, args.IIS.SiteExclude) + require.Equal(t, DefaultArguments.IIS.SiteInclude, args.IIS.SiteInclude) + require.Equal(t, DefaultArguments.LogicalDisk.Exclude, args.LogicalDisk.Exclude) + require.Equal(t, DefaultArguments.LogicalDisk.Include, args.LogicalDisk.Include) + require.Equal(t, DefaultArguments.MSMQ.Where, args.MSMQ.Where) + require.Equal(t, DefaultArguments.MSSQL.EnabledClasses, args.MSSQL.EnabledClasses) + require.Equal(t, DefaultArguments.Network.Exclude, args.Network.Exclude) + require.Equal(t, DefaultArguments.Network.Include, args.Network.Include) + require.Equal(t, DefaultArguments.Process.Exclude, args.Process.Exclude) + require.Equal(t, DefaultArguments.Process.Include, args.Process.Include) + require.Equal(t, DefaultArguments.ScheduledTask.Exclude, args.ScheduledTask.Exclude) + require.Equal(t, DefaultArguments.ScheduledTask.Include, args.ScheduledTask.Include) + require.Equal(t, DefaultArguments.Service.UseApi, args.Service.UseApi) + require.Equal(t, DefaultArguments.Service.Where, args.Service.Where) + require.Equal(t, DefaultArguments.SMTP.Exclude, args.SMTP.Exclude) + require.Equal(t, DefaultArguments.SMTP.Include, args.SMTP.Include) + require.Equal(t, DefaultArguments.TextFile.TextFileDirectory, args.TextFile.TextFileDirectory) } diff --git a/component/prometheus/exporter/windows/config_windows.go b/component/prometheus/exporter/windows/config_windows.go new file mode 100644 index 000000000000..b634788eda8c --- /dev/null +++ b/component/prometheus/exporter/windows/config_windows.go @@ -0,0 +1,75 @@ +package windows + +import ( + windows_integration "github.com/grafana/agent/pkg/integrations/windows_exporter" + col "github.com/prometheus-community/windows_exporter/pkg/collector" + "strings" +) + +// DefaultArguments holds non-zero default options for Arguments when it is +// unmarshaled from YAML. +var DefaultArguments = Arguments{ + EnabledCollectors: strings.Split(windows_integration.DefaultConfig.EnabledCollectors, ","), + Dfsr: DfsrConfig{ + SourcesEnabled: strings.Split(col.ConfigDefaults.Dfsr.DfsrEnabledCollectors, ","), + }, + Exchange: ExchangeConfig{ + EnabledList: strings.Split(col.ConfigDefaults.Exchange.CollectorsEnabled, ","), + }, + IIS: IISConfig{ + AppBlackList: col.ConfigDefaults.Iis.AppExclude, + AppWhiteList: col.ConfigDefaults.Iis.AppInclude, + SiteBlackList: col.ConfigDefaults.Iis.SiteExclude, + SiteWhiteList: col.ConfigDefaults.Iis.SiteInclude, + AppInclude: col.ConfigDefaults.Iis.AppInclude, + AppExclude: col.ConfigDefaults.Iis.AppExclude, + SiteInclude: col.ConfigDefaults.Iis.SiteInclude, + SiteExclude: col.ConfigDefaults.Iis.SiteExclude, + }, + LogicalDisk: LogicalDiskConfig{ + BlackList: col.ConfigDefaults.LogicalDisk.VolumeExclude, + WhiteList: col.ConfigDefaults.LogicalDisk.VolumeInclude, + Include: col.ConfigDefaults.LogicalDisk.VolumeInclude, + Exclude: col.ConfigDefaults.LogicalDisk.VolumeExclude, + }, + MSMQ: MSMQConfig{ + Where: col.ConfigDefaults.Msmq.QueryWhereClause, + }, + MSSQL: MSSQLConfig{ + EnabledClasses: strings.Split(col.ConfigDefaults.Mssql.EnabledCollectors, ","), + }, + Network: NetworkConfig{ + BlackList: col.ConfigDefaults.Net.NicExclude, + WhiteList: col.ConfigDefaults.Net.NicInclude, + Include: col.ConfigDefaults.Net.NicInclude, + Exclude: col.ConfigDefaults.Net.NicExclude, + }, + Process: ProcessConfig{ + BlackList: col.ConfigDefaults.Process.ProcessExclude, + WhiteList: col.ConfigDefaults.Process.ProcessInclude, + Include: col.ConfigDefaults.Process.ProcessInclude, + Exclude: col.ConfigDefaults.Process.ProcessExclude, + }, + ScheduledTask: ScheduledTaskConfig{ + Include: col.ConfigDefaults.ScheduledTask.TaskInclude, + Exclude: col.ConfigDefaults.ScheduledTask.TaskExclude, + }, + Service: ServiceConfig{ + UseApi: "false", + Where: col.ConfigDefaults.Service.ServiceWhereClause, + }, + SMTP: SMTPConfig{ + BlackList: col.ConfigDefaults.Smtp.ServerExclude, + WhiteList: col.ConfigDefaults.Smtp.ServerInclude, + Include: col.ConfigDefaults.Smtp.ServerInclude, + Exclude: col.ConfigDefaults.Smtp.ServerExclude, + }, + TextFile: TextFileConfig{ + TextFileDirectory: col.ConfigDefaults.Textfile.TextFileDirectories, + }, +} + +// SetToDefault implements river.Defaulter. +func (a *Arguments) SetToDefault() { + *a = DefaultArguments +} diff --git a/component/pyroscope/scrape/scrape_loop.go b/component/pyroscope/scrape/scrape_loop.go index 2b74930ed191..a1f7d2a6c1b7 100644 --- a/component/pyroscope/scrape/scrape_loop.go +++ b/component/pyroscope/scrape/scrape_loop.go @@ -228,7 +228,7 @@ func (t *scrapeLoop) scrape() { } } if err := t.fetchProfile(scrapeCtx, profileType, buf); err != nil { - level.Debug(t.logger).Log("msg", "fetch profile failed", "target", t.Labels().String(), "err", err) + level.Error(t.logger).Log("msg", "fetch profile failed", "target", t.Labels().String(), "err", err) t.updateTargetStatus(start, err) return } diff --git a/converter/internal/staticconvert/internal/build/windows_exporter.go b/converter/internal/staticconvert/internal/build/windows_exporter.go index 27e9679887b8..73aa706e8235 100644 --- a/converter/internal/staticconvert/internal/build/windows_exporter.go +++ b/converter/internal/staticconvert/internal/build/windows_exporter.go @@ -47,8 +47,8 @@ func toWindowsExporter(config *windows_exporter.Config) *windows.Arguments { Network: windows.NetworkConfig{ BlackList: config.Network.BlackList, WhiteList: config.Network.WhiteList, - Exclude: config.Network.Include, - Include: config.Network.Exclude, + Exclude: config.Network.Exclude, + Include: config.Network.Include, }, Process: windows.ProcessConfig{ BlackList: config.Process.BlackList, diff --git a/docs/sources/_index.md b/docs/sources/_index.md index 15df6d9c3174..f744311514c5 100644 --- a/docs/sources/_index.md +++ b/docs/sources/_index.md @@ -9,7 +9,7 @@ title: Grafana Agent description: Grafana Agent is a flexible, performant, vendor-neutral, telemetry collector weight: 350 cascade: - AGENT_RELEASE: v0.38.0-rc.1 + AGENT_RELEASE: v0.38.0 OTEL_VERSION: v0.87.0 --- diff --git a/docs/sources/flow/setup/install/windows.md b/docs/sources/flow/setup/install/windows.md index 84b0607e139b..f9106e5f936f 100644 --- a/docs/sources/flow/setup/install/windows.md +++ b/docs/sources/flow/setup/install/windows.md @@ -57,6 +57,14 @@ To do a silent install of Grafana Agent on Windows, perform the following steps. * `/CONFIG=<path>` Path to the configuration file. Default: `$INSTDIR\config.river` * `/DISABLEREPORTING=<yes|no>` Disable [data collection][]. Default: `no` * `/DISABLEPROFILING=<yes|no>` Disable profiling endpoint. Default: `no` +* `/ENVIRONMENT="KEY=VALUE\0KEY2=VALUE2"` Define environment variables for Windows Service. Default: `` + +## Service Configuration + +Grafana Agent uses the Windows Registry `HKLM\Software\Grafana\Grafana Agent Flow` for service configuration. + +* `Arguments` (Type `REG_MULTI_SZ`) Each value represents a binary argument for grafana-agent-flow binary. +* `Environment` (Type `REG_MULTI_SZ`) Each value represents a environment value `KEY=VALUE` for grafana-agent-flow binary. ## Uninstall diff --git a/docs/sources/static/configuration/agent-management.md b/docs/sources/static/configuration/agent-management.md index 0feb5c78def1..af327bb17b6e 100644 --- a/docs/sources/static/configuration/agent-management.md +++ b/docs/sources/static/configuration/agent-management.md @@ -123,7 +123,6 @@ selector: > **Note:** Snippet selection is currently done in the API server. This behaviour is subject to change in the future. - ### Example response body ```yaml @@ -164,3 +163,9 @@ snippets: os: linux app: app1 ``` + +> **Note:** Base configurations and snippets can contain go's [text/template](https://pkg.go.dev/text/template) actions. If you need preserve the literal value of a template action, you can escape it using backticks. For example: + +``` +{{ `{{ .template_var }}` }} +``` diff --git a/integration-tests/configs/otel-collector-contrib/otel-collector-contrib.yaml b/integration-tests/configs/otel-collector-contrib/otel-collector-contrib.yaml deleted file mode 100644 index 7359cfb35215..000000000000 --- a/integration-tests/configs/otel-collector-contrib/otel-collector-contrib.yaml +++ /dev/null @@ -1,27 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - -exporters: - logging: - - otlphttp: - endpoint: ${OTEL_EXPORTER_ENDPOINT} - - -connectors: - spanmetrics: - namespace: span.metrics - exemplars: - enabled: true - metrics_flush_interval: 1s - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [spanmetrics] - metrics: - receivers: [spanmetrics] - exporters: [otlphttp] diff --git a/integration-tests/configs/otel-gen-client/main.go b/integration-tests/configs/otel-gen-client/main.go deleted file mode 100644 index 7066daccf26d..000000000000 --- a/integration-tests/configs/otel-gen-client/main.go +++ /dev/null @@ -1,205 +0,0 @@ -// This file was copied with minor modifications from https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/b0be5c98325ec71f35b82e278a3fc3e6f3fe4954/examples/demo/client/main.go - -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -// Sample contains a simple client that periodically makes a simple http request -// to a server and exports to the OpenTelemetry service. -package main - -import ( - "context" - "log" - "math/rand" - "net/http" - "os" - "time" - - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/baggage" - "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/propagation" - sdkmetric "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.4.0" - "google.golang.org/grpc" -) - -const ( - otelExporterOtlpEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT" - demoServerEndpoint = "DEMO_SERVER_ENDPOINT" -) - -// Initializes an OTLP exporter, and configures the corresponding trace and -// metric providers. -func initProvider() func() { - ctx := context.Background() - - res, err := resource.New(ctx, - resource.WithFromEnv(), - resource.WithProcess(), - resource.WithTelemetrySDK(), - resource.WithHost(), - resource.WithAttributes( - // the service name used to display traces in backends - semconv.ServiceNameKey.String("demo-client"), - ), - ) - handleErr(err, "failed to create resource") - - otelAgentAddr, ok := os.LookupEnv(otelExporterOtlpEndpoint) - if !ok { - otelAgentAddr = "0.0.0.0:4317" - } - - metricExp, err := otlpmetricgrpc.New( - ctx, - otlpmetricgrpc.WithInsecure(), - otlpmetricgrpc.WithEndpoint(otelAgentAddr), - ) - handleErr(err, "Failed to create the collector metric exporter") - - meterProvider := sdkmetric.NewMeterProvider( - sdkmetric.WithResource(res), - sdkmetric.WithReader( - sdkmetric.NewPeriodicReader( - metricExp, - sdkmetric.WithInterval(2*time.Second), - ), - ), - ) - otel.SetMeterProvider(meterProvider) - - traceClient := otlptracegrpc.NewClient( - otlptracegrpc.WithInsecure(), - otlptracegrpc.WithEndpoint(otelAgentAddr), - otlptracegrpc.WithDialOption(grpc.WithBlock())) - sctx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - traceExp, err := otlptrace.New(sctx, traceClient) - handleErr(err, "Failed to create the collector trace exporter") - - bsp := sdktrace.NewBatchSpanProcessor(traceExp) - tracerProvider := sdktrace.NewTracerProvider( - sdktrace.WithSampler(sdktrace.AlwaysSample()), - sdktrace.WithResource(res), - sdktrace.WithSpanProcessor(bsp), - ) - - // set global propagator to tracecontext (the default is no-op). - otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) - otel.SetTracerProvider(tracerProvider) - - return func() { - cxt, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - if err := traceExp.Shutdown(cxt); err != nil { - otel.Handle(err) - } - // pushes any last exports to the receiver - if err := meterProvider.Shutdown(cxt); err != nil { - otel.Handle(err) - } - } -} - -func handleErr(err error, message string) { - if err != nil { - log.Fatalf("%s: %v", message, err) - } -} - -func main() { - shutdown := initProvider() - defer shutdown() - - tracer := otel.Tracer("demo-client-tracer") - meter := otel.Meter("demo-client-meter") - - method, _ := baggage.NewMember("method", "repl") - client, _ := baggage.NewMember("client", "cli") - bag, _ := baggage.New(method, client) - - // labels represent additional key-value descriptors that can be bound to a - // metric observer or recorder. - // TODO: Use baggage when supported to extract labels from baggage. - commonLabels := []attribute.KeyValue{ - attribute.String("method", "repl"), - attribute.String("client", "cli"), - } - - // Recorder metric example - requestLatency, _ := meter.Float64Histogram( - "demo_client/request_latency", - metric.WithDescription("The latency of requests processed"), - ) - - // TODO: Use a view to just count number of measurements for requestLatency when available. - requestCount, _ := meter.Int64Counter( - "demo_client/request_counts", - metric.WithDescription("The number of requests processed"), - ) - - lineLengths, _ := meter.Int64Histogram( - "demo_client/line_lengths", - metric.WithDescription("The lengths of the various lines in"), - ) - - // TODO: Use a view to just count number of measurements for lineLengths when available. - lineCounts, _ := meter.Int64Counter( - "demo_client/line_counts", - metric.WithDescription("The counts of the lines in"), - ) - - defaultCtx := baggage.ContextWithBaggage(context.Background(), bag) - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - for { - startTime := time.Now() - ctx, span := tracer.Start(defaultCtx, "ExecuteRequest") - makeRequest(ctx) - span.End() - latencyMs := float64(time.Since(startTime)) / 1e6 - nr := int(rng.Int31n(7)) - for i := 0; i < nr; i++ { - randLineLength := rng.Int63n(999) - lineCounts.Add(ctx, 1, metric.WithAttributes(commonLabels...)) - lineLengths.Record(ctx, randLineLength, metric.WithAttributes(commonLabels...)) - } - - requestLatency.Record(ctx, latencyMs, metric.WithAttributes(commonLabels...)) - requestCount.Add(ctx, 1, metric.WithAttributes(commonLabels...)) - - time.Sleep(time.Duration(1) * time.Second) - } -} - -func makeRequest(ctx context.Context) { - demoServerAddr, ok := os.LookupEnv(demoServerEndpoint) - if !ok { - demoServerAddr = "http://0.0.0.0:7080/hello" - } - - // Trace an HTTP client by wrapping the transport - client := http.Client{ - Transport: otelhttp.NewTransport(http.DefaultTransport), - } - - // Make sure we pass the context to the request to avoid broken traces. - req, err := http.NewRequestWithContext(ctx, "GET", demoServerAddr, nil) - if err != nil { - handleErr(err, "failed to http request") - } - - // All requests made with this client will create spans. - res, err := client.Do(req) - if err != nil { - panic(err) - } - res.Body.Close() -} diff --git a/integration-tests/configs/otel-gen-server/Dockerfile b/integration-tests/configs/otel-gen-server/Dockerfile deleted file mode 100644 index bc6bc1c6d136..000000000000 --- a/integration-tests/configs/otel-gen-server/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 -FROM golang:1.21 as build -WORKDIR /app/ -COPY go.mod go.sum ./ -RUN go mod download -COPY ./integration-tests/configs/otel-gen-server/ ./ -RUN CGO_ENABLED=0 go build -o main main.go -FROM alpine:3.18 -COPY --from=build /app/main /app/main -CMD ["/app/main"] diff --git a/integration-tests/configs/otel-gen-server/main.go b/integration-tests/configs/otel-gen-server/main.go deleted file mode 100644 index 97bfb1ca2ef7..000000000000 --- a/integration-tests/configs/otel-gen-server/main.go +++ /dev/null @@ -1,155 +0,0 @@ -// This file was copied with minor modifications from https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/b0be5c98325ec71f35b82e278a3fc3e6f3fe4954/examples/demo/server/main.go - -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -// Sample contains a simple http server that exports to the OpenTelemetry agent. - -package main - -import ( - "context" - "log" - "net/http" - "os" - "time" - - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/baggage" - "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/propagation" - sdkmetric "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.4.0" - "go.opentelemetry.io/otel/trace" - "google.golang.org/grpc" -) - -const otelExporterOtlpEndpoint = "OTEL_EXPORTER_OTLP_ENDPOINT" - -// Initializes an OTLP exporter, and configures the corresponding trace and -// metric providers. -func initProvider() func() { - ctx := context.Background() - - res, err := resource.New(ctx, - resource.WithFromEnv(), - resource.WithProcess(), - resource.WithTelemetrySDK(), - resource.WithHost(), - resource.WithAttributes( - // the service name used to display traces in backends - semconv.ServiceNameKey.String("demo-server"), - ), - ) - handleErr(err, "failed to create resource") - - otelAgentAddr, ok := os.LookupEnv(otelExporterOtlpEndpoint) - if !ok { - otelAgentAddr = "0.0.0.0:4317" - } - - metricExp, err := otlpmetricgrpc.New( - ctx, - otlpmetricgrpc.WithInsecure(), - otlpmetricgrpc.WithEndpoint(otelAgentAddr)) - handleErr(err, "Failed to create the collector metric exporter") - - meterProvider := sdkmetric.NewMeterProvider( - sdkmetric.WithResource(res), - sdkmetric.WithReader( - sdkmetric.NewPeriodicReader( - metricExp, - sdkmetric.WithInterval(2*time.Second), - ), - ), - ) - otel.SetMeterProvider(meterProvider) - - traceClient := otlptracegrpc.NewClient( - otlptracegrpc.WithInsecure(), - otlptracegrpc.WithEndpoint(otelAgentAddr), - otlptracegrpc.WithDialOption(grpc.WithBlock())) - traceExp, err := otlptrace.New(ctx, traceClient) - handleErr(err, "Failed to create the collector trace exporter") - - bsp := sdktrace.NewBatchSpanProcessor(traceExp) - tracerProvider := sdktrace.NewTracerProvider( - sdktrace.WithSampler(sdktrace.AlwaysSample()), - sdktrace.WithResource(res), - sdktrace.WithSpanProcessor(bsp), - ) - - // set global propagator to tracecontext (the default is no-op). - otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) - otel.SetTracerProvider(tracerProvider) - - return func() { - cxt, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - if err := traceExp.Shutdown(cxt); err != nil { - otel.Handle(err) - } - // pushes any last exports to the receiver - if err := meterProvider.Shutdown(cxt); err != nil { - otel.Handle(err) - } - } -} - -func handleErr(err error, message string) { - if err != nil { - log.Fatalf("%s: %v", message, err) - } -} - -func main() { - shutdown := initProvider() - defer shutdown() - - meter := otel.Meter("demo-server-meter") - serverAttribute := attribute.String("server-attribute", "foo") - commonLabels := []attribute.KeyValue{serverAttribute} - requestCount, _ := meter.Int64Counter( - "demo_server/request_counts", - metric.WithDescription("The number of requests received"), - ) - - // create a handler wrapped in OpenTelemetry instrumentation - handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - time.Sleep(time.Duration(100) * time.Millisecond) - ctx := req.Context() - requestCount.Add(ctx, 1, metric.WithAttributes(commonLabels...)) - span := trace.SpanFromContext(ctx) - bag := baggage.FromContext(ctx) - - var baggageAttributes []attribute.KeyValue - baggageAttributes = append(baggageAttributes, serverAttribute) - for _, member := range bag.Members() { - baggageAttributes = append(baggageAttributes, attribute.String("baggage key:"+member.Key(), member.Value())) - } - span.SetAttributes(baggageAttributes...) - - if _, err := w.Write([]byte("Hello World")); err != nil { - http.Error(w, "write operation failed.", http.StatusInternalServerError) - return - } - }) - - mux := http.NewServeMux() - mux.Handle("/hello", otelhttp.NewHandler(handler, "/hello")) - server := &http.Server{ - Addr: ":7080", - Handler: mux, - ReadHeaderTimeout: 20 * time.Second, - } - if err := server.ListenAndServe(); err != http.ErrServerClosed { - handleErr(err, "server failed to serve") - } -} diff --git a/integration-tests/configs/otel-gen-client/Dockerfile b/integration-tests/configs/otel-metrics-gen/Dockerfile similarity index 60% rename from integration-tests/configs/otel-gen-client/Dockerfile rename to integration-tests/configs/otel-metrics-gen/Dockerfile index aae9f646067b..8bf00387660e 100644 --- a/integration-tests/configs/otel-gen-client/Dockerfile +++ b/integration-tests/configs/otel-metrics-gen/Dockerfile @@ -1,10 +1,8 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 FROM golang:1.21 as build WORKDIR /app/ COPY go.mod go.sum ./ RUN go mod download -COPY ./integration-tests/configs/otel-gen-client/ ./ +COPY ./integration-tests/configs/otel-metrics-gen/ ./ RUN CGO_ENABLED=0 go build -o main main.go FROM alpine:3.18 COPY --from=build /app/main /app/main diff --git a/integration-tests/configs/otel-metrics-gen/main.go b/integration-tests/configs/otel-metrics-gen/main.go new file mode 100644 index 000000000000..6f88d7725b46 --- /dev/null +++ b/integration-tests/configs/otel-metrics-gen/main.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + "log" + "os" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/sdk/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" +) + +const otelExporterOtlpEndpoint = "OTEL_EXPORTER_ENDPOINT" + +func main() { + ctx := context.Background() + otlpExporterEndpoint, ok := os.LookupEnv(otelExporterOtlpEndpoint) + if !ok { + otlpExporterEndpoint = "localhost:4318" + } + exporter, err := otlpmetrichttp.New(ctx, + otlpmetrichttp.WithInsecure(), + otlpmetrichttp.WithEndpoint(otlpExporterEndpoint), + ) + if err != nil { + log.Fatalf("failed to create exporter: %v", err) + } + + resource, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String("otel-metrics-gen"), + ), + ) + if err != nil { + log.Fatalf("failed to create resource: %v", err) + } + + exponentialHistogramView := metric.NewView( + metric.Instrument{ + Name: "example_exponential_*", + }, + metric.Stream{ + Aggregation: metric.AggregationBase2ExponentialHistogram{ + MaxSize: 160, + MaxScale: 20, + }, + }, + ) + + provider := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exporter, sdkmetric.WithInterval(1*time.Second))), + sdkmetric.WithResource(resource), + metric.WithView(exponentialHistogramView), + ) + otel.SetMeterProvider(provider) + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := provider.Shutdown(ctx); err != nil { + log.Fatalf("Server shutdown error: %v", err) + } + }() + + meter := otel.Meter("example-meter") + counter, _ := meter.Int64Counter("example_counter") + floatCounter, _ := meter.Float64Counter("example_float_counter") + upDownCounter, _ := meter.Int64UpDownCounter("example_updowncounter") + floatUpDownCounter, _ := meter.Float64UpDownCounter("example_float_updowncounter") + histogram, _ := meter.Int64Histogram("example_histogram") + floatHistogram, _ := meter.Float64Histogram("example_float_histogram") + exponentialHistogram, _ := meter.Int64Histogram("example_exponential_histogram") + exponentialFloatHistogram, _ := meter.Float64Histogram("example_exponential_float_histogram") + + for { + counter.Add(ctx, 10) + floatCounter.Add(ctx, 2.5) + upDownCounter.Add(ctx, -5) + floatUpDownCounter.Add(ctx, 3.5) + histogram.Record(ctx, 2) + floatHistogram.Record(ctx, 6.5) + exponentialHistogram.Record(ctx, 5) + exponentialFloatHistogram.Record(ctx, 1.5) + + time.Sleep(200 * time.Millisecond) + } +} diff --git a/integration-tests/docker-compose.yaml b/integration-tests/docker-compose.yaml index b576a425fa68..a94a05db21d9 100644 --- a/integration-tests/docker-compose.yaml +++ b/integration-tests/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3" services: mimir: - image: grafana/mimir:latest + image: grafana/mimir:2.10.4 volumes: - ./configs/mimir:/etc/mimir-config entrypoint: @@ -12,45 +12,17 @@ services: - "9009:9009" loki: - image: grafana/loki:2.8.3 + image: grafana/loki:latest command: -config.file=/etc/loki/local-config.yaml ports: - "3100:3100" - otel-collector: - image: otel/opentelemetry-collector-contrib:0.85.0 - restart: always - command: ["--config=/etc/otel-collector-contrib.yaml", ""] - volumes: - - ./configs/otel-collector-contrib/otel-collector-contrib.yaml:/etc/otel-collector-contrib.yaml - ports: - - "4317:4317" # OTLP gRPC receiver - - "4318:4318" # OTLP HTTP exporter - environment: - - OTEL_EXPORTER_ENDPOINT=${OTEL_EXPORTER_ENDPOINT:-http://host.docker.internal:8080} - - demo-client: + otel-metrics-gen: build: - dockerfile: ./integration-tests/configs/otel-gen-client/Dockerfile + dockerfile: ./integration-tests/configs/otel-metrics-gen/Dockerfile context: .. - restart: always environment: - - OTEL_EXPORTER_OTLP_ENDPOINT=otel-collector:4317 - - DEMO_SERVER_ENDPOINT=http://demo-server:7080/hello - depends_on: - - demo-server - - demo-server: - build: - dockerfile: ./integration-tests/configs/otel-gen-server/Dockerfile - context: .. - restart: always - environment: - - OTEL_EXPORTER_OTLP_ENDPOINT=otel-collector:4317 - ports: - - "7080" - depends_on: - - otel-collector + - OTEL_EXPORTER_ENDPOINT=${OTEL_EXPORTER_ENDPOINT:-host.docker.internal:4318} prom-gen: build: diff --git a/integration-tests/tests/otlp-metrics/config.river b/integration-tests/tests/otlp-metrics/config.river index 8c4f0ebc96e6..26a6212a3c11 100644 --- a/integration-tests/tests/otlp-metrics/config.river +++ b/integration-tests/tests/otlp-metrics/config.river @@ -1,7 +1,5 @@ otelcol.receiver.otlp "otlp_metrics" { - http { - endpoint="0.0.0.0:8080" - } + http {} output { metrics = [otelcol.processor.attributes.otlp_metrics.input] diff --git a/integration-tests/tests/otlp-metrics/otlp_metrics_test.go b/integration-tests/tests/otlp-metrics/otlp_metrics_test.go index a4bdd887c50c..03575e742b2d 100644 --- a/integration-tests/tests/otlp-metrics/otlp_metrics_test.go +++ b/integration-tests/tests/otlp-metrics/otlp_metrics_test.go @@ -1,23 +1,90 @@ package main import ( + "fmt" + "strconv" "testing" "github.com/grafana/agent/integration-tests/common" "github.com/stretchr/testify/assert" ) -const query = "http://localhost:9009/prometheus/api/v1/query?query=span_metrics_duration_bucket{test_name='otlp_metrics'}" +const promURL = "http://localhost:9009/prometheus/api/v1/query?query=" -func TestOtlpMetrics(t *testing.T) { +func metricQuery(metricName string) string { + return fmt.Sprintf("%s%s{test_name='otlp_metrics'}", promURL, metricName) +} + +func TestOTLPMetrics(t *testing.T) { + tests := []struct { + metric string + }{ + // TODO: better differentiate these metric types? + {"example_counter"}, + {"example_float_counter"}, + {"example_updowncounter"}, + {"example_float_updowncounter"}, + {"example_histogram_bucket"}, + {"example_float_histogram_bucket"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.metric, func(t *testing.T) { + t.Parallel() + assertMetricData(t, metricQuery(tt.metric), tt.metric) + }) + } + + histogramTests := []string{ + "example_exponential_histogram", + "example_exponential_float_histogram", + } + + for _, metric := range histogramTests { + metric := metric + t.Run(metric, func(t *testing.T) { + t.Parallel() + assertHistogramData(t, metricQuery(metric), metric) + }) + } +} + +func assertHistogramData(t *testing.T, query string, expectedMetric string) { + var metricResponse common.MetricResponse + assert.EventuallyWithT(t, func(c *assert.CollectT) { + err := common.FetchDataFromURL(query, &metricResponse) + assert.NoError(c, err) + if assert.NotEmpty(c, metricResponse.Data.Result) { + assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) + assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "otlp_metrics") + if assert.NotNil(c, metricResponse.Data.Result[0].Histogram) { + histogram := metricResponse.Data.Result[0].Histogram + if assert.NotEmpty(c, histogram.Data.Count) { + count, _ := strconv.Atoi(histogram.Data.Count) + assert.Greater(c, count, 10, "Count should be at some point greater than 10.") + } + if assert.NotEmpty(c, histogram.Data.Sum) { + sum, _ := strconv.Atoi(histogram.Data.Sum) + assert.Greater(c, sum, 10, "Sum should be at some point greater than 10.") + } + assert.NotEmpty(c, histogram.Data.Buckets) + assert.Nil(c, metricResponse.Data.Result[0].Value) + } + } + }, common.DefaultTimeout, common.DefaultRetryInterval, "Histogram data did not satisfy the conditions within the time limit") +} + +func assertMetricData(t *testing.T, query, expectedMetric string) { var metricResponse common.MetricResponse assert.EventuallyWithT(t, func(c *assert.CollectT) { err := common.FetchDataFromURL(query, &metricResponse) assert.NoError(c, err) if assert.NotEmpty(c, metricResponse.Data.Result) { - assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, "span_metrics_duration_bucket") + assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "otlp_metrics") assert.NotEmpty(c, metricResponse.Data.Result[0].Value.Value) + assert.Nil(c, metricResponse.Data.Result[0].Histogram) } }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") } diff --git a/integration-tests/tests/scrape-prom-metrics/scrape_prom_metrics_test.go b/integration-tests/tests/scrape-prom-metrics/scrape_prom_metrics_test.go index f57e1ca37b3c..467147846e67 100644 --- a/integration-tests/tests/scrape-prom-metrics/scrape_prom_metrics_test.go +++ b/integration-tests/tests/scrape-prom-metrics/scrape_prom_metrics_test.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strconv" "testing" "github.com/grafana/agent/integration-tests/common" @@ -15,28 +16,26 @@ func metricQuery(metricName string) string { } func TestScrapePromMetrics(t *testing.T) { - tests := []struct { - query string - metric string - }{ + metrics := []string{ // TODO: better differentiate these metric types? - {metricQuery("golang_counter"), "golang_counter"}, - {metricQuery("golang_gauge"), "golang_gauge"}, - {metricQuery("golang_histogram_bucket"), "golang_histogram_bucket"}, - {metricQuery("golang_summary"), "golang_summary"}, + "golang_counter", + "golang_gauge", + "golang_histogram_bucket", + "golang_summary", + "golang_native_histogram", } - for _, tt := range tests { - tt := tt - t.Run(tt.metric, func(t *testing.T) { + for _, metric := range metrics { + metric := metric + t.Run(metric, func(t *testing.T) { t.Parallel() - assertMetricData(t, tt.query, tt.metric) + if metric == "golang_native_histogram" { + assertHistogramData(t, metricQuery(metric), metric) + } else { + assertMetricData(t, metricQuery(metric), metric) + } }) } - t.Run("golang_native_histogram", func(t *testing.T) { - t.Parallel() - assertHistogramData(t, metricQuery("golang_native_histogram"), "golang_native_histogram") - }) } func assertHistogramData(t *testing.T, query string, expectedMetric string) { @@ -49,8 +48,14 @@ func assertHistogramData(t *testing.T, query string, expectedMetric string) { assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics") if assert.NotNil(c, metricResponse.Data.Result[0].Histogram) { histogram := metricResponse.Data.Result[0].Histogram - assert.NotEmpty(c, histogram.Data.Count) - assert.NotEmpty(c, histogram.Data.Sum) + if assert.NotEmpty(c, histogram.Data.Count) { + count, _ := strconv.Atoi(histogram.Data.Count) + assert.Greater(c, count, 10, "Count should be at some point greater than 10.") + } + if assert.NotEmpty(c, histogram.Data.Sum) { + sum, _ := strconv.ParseFloat(histogram.Data.Sum, 64) + assert.Greater(c, sum, 10., "Sum should be at some point greater than 10.") + } assert.NotEmpty(c, histogram.Data.Buckets) assert.Nil(c, metricResponse.Data.Result[0].Value) } diff --git a/integration-tests/utils.go b/integration-tests/utils.go index 06aba877203c..d723a235235c 100644 --- a/integration-tests/utils.go +++ b/integration-tests/utils.go @@ -3,6 +3,7 @@ package main import ( "bytes" "fmt" + "log" "os" "os/exec" "path/filepath" @@ -10,9 +11,7 @@ import ( ) const ( - agentBinaryPath = "../../../build/grafana-agent-flow" - dockerComposeCmd = "docker-compose" - makeCmd = "make" + agentBinaryPath = "../../../build/grafana-agent-flow" ) type TestLog struct { @@ -23,20 +22,22 @@ type TestLog struct { var logChan chan TestLog -func buildAgent() { - fmt.Println("Building agent...") - cmd := exec.Command(makeCmd, "-C", "..", "agent-flow") +func executeCommand(command string, args []string, taskDescription string) { + fmt.Printf("%s...\n", taskDescription) + cmd := exec.Command(command, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - panic(err) + log.Fatalf("Error: %s\n", stderr.String()) } } +func buildAgent() { + executeCommand("make", []string{"-C", "..", "agent-flow"}, "Building agent") +} + func setupEnvironment() { - fmt.Println("Setting up environment with Docker Compose...") - cmd := exec.Command(dockerComposeCmd, "up", "-d") - if err := cmd.Run(); err != nil { - panic(err) - } + executeCommand("docker-compose", []string{"up", "-d"}, "Setting up environment with Docker Compose") } func runSingleTest(testDir string) { @@ -105,18 +106,18 @@ func runAllTests() { }(testDir) } wg.Wait() + close(logChan) } func cleanUpEnvironment() { fmt.Println("Cleaning up Docker environment...") - err := exec.Command(dockerComposeCmd, "down", "--volumes", "--rmi", "all").Run() + err := exec.Command("docker-compose", "down", "--volumes", "--rmi", "all").Run() if err != nil { panic(err) } } func reportResults() { - close(logChan) testsFailed := 0 for log := range logChan { fmt.Printf("Failure detected in %s:\n", log.TestDir) diff --git a/operations/helm/charts/grafana-agent/CHANGELOG.md b/operations/helm/charts/grafana-agent/CHANGELOG.md index 6b82c54edf9d..26cc9a4948a6 100644 --- a/operations/helm/charts/grafana-agent/CHANGELOG.md +++ b/operations/helm/charts/grafana-agent/CHANGELOG.md @@ -10,11 +10,20 @@ internal API changes are not present. Unreleased ---------- +### Other changes + +- Update `container.securityContext` Helm value reference to renamed `agent.securityContext`. (@hainenber) + +0.28.0 (2023-11-21) +------------------- + ### Enhancements -- Ensure that `app.kubernetes.io/version` label accounts for any +- Ensure that `app.kubernetes.io/version` label accounts for any image tag overrides supplied to the chart Values. (@tristanburgess) +- Update Grafana Agent version to v0.38.0. (@rfratto) + 0.27.2 (2023-11-07) ---------- diff --git a/operations/helm/charts/grafana-agent/Chart.yaml b/operations/helm/charts/grafana-agent/Chart.yaml index d73883aeae7d..935664274889 100644 --- a/operations/helm/charts/grafana-agent/Chart.yaml +++ b/operations/helm/charts/grafana-agent/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: grafana-agent description: 'Grafana Agent' type: application -version: 0.27.2 -appVersion: 'v0.37.4' +version: 0.28.0 +appVersion: 'v0.38.0' dependencies: - name: crds diff --git a/operations/helm/charts/grafana-agent/README.md b/operations/helm/charts/grafana-agent/README.md index f85c05a0b40b..d9d577e871d4 100644 --- a/operations/helm/charts/grafana-agent/README.md +++ b/operations/helm/charts/grafana-agent/README.md @@ -1,6 +1,6 @@ # Grafana Agent Helm chart -   +   Helm chart for deploying [Grafana Agent][] to Kubernetes. @@ -168,9 +168,9 @@ used. When provided, `agent.configMap.content` must hold a valid River configura [default-config]: ./config/example.river -### controller.securityContext +### agent.securityContext -`controller.securityContext` sets the securityContext passed to the Grafana +`agent.securityContext` sets the securityContext passed to the Grafana Agent container. By default, Grafana Agent containers are not able to collect telemetry from the @@ -241,7 +241,7 @@ This capability is disabled by default. To expose logs from other containers to Grafana Agent: * Set `agent.mounts.dockercontainers` to `true`. -* Set `controller.securityContext` to: +* Set `agent.securityContext` to: ```yaml privileged: true runAsUser: 0 @@ -256,7 +256,7 @@ To expose this information to Grafana Agent for telemetry collection: * Set `agent.mounts.dockercontainers` to `true`. * Mount `/proc` and `/sys` from the host into the container. -* Set `controller.securityContext` to: +* Set `agent.securityContext` to: ```yaml privileged: true runAsUser: 0 diff --git a/operations/helm/charts/grafana-agent/README.md.gotmpl b/operations/helm/charts/grafana-agent/README.md.gotmpl index fd34da2a962a..e9bbe8ece543 100644 --- a/operations/helm/charts/grafana-agent/README.md.gotmpl +++ b/operations/helm/charts/grafana-agent/README.md.gotmpl @@ -80,9 +80,9 @@ used. When provided, `agent.configMap.content` must hold a valid River configura [default-config]: ./config/example.river -### controller.securityContext +### agent.securityContext -`controller.securityContext` sets the securityContext passed to the Grafana +`agent.securityContext` sets the securityContext passed to the Grafana Agent container. By default, Grafana Agent containers are not able to collect telemetry from the @@ -153,7 +153,7 @@ This capability is disabled by default. To expose logs from other containers to Grafana Agent: * Set `agent.mounts.dockercontainers` to `true`. -* Set `controller.securityContext` to: +* Set `agent.securityContext` to: ```yaml privileged: true runAsUser: 0 @@ -168,7 +168,7 @@ To expose this information to Grafana Agent for telemetry collection: * Set `agent.mounts.dockercontainers` to `true`. * Mount `/proc` and `/sys` from the host into the container. -* Set `controller.securityContext` to: +* Set `agent.securityContext` to: ```yaml privileged: true runAsUser: 0 diff --git a/operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml b/operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml index 69354494639f..ff1d21ceba8a 100644 --- a/operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml +++ b/operations/helm/tests/clustering/grafana-agent/templates/controllers/statefulset.yaml @@ -28,7 +28,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml index de0c1e307bc6..e89c55f50d58 100644 --- a/operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/controller-volumes-extra/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml index 0921197e38ef..a35f7494f9d3 100644 --- a/operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/create-daemonset-hostnetwork/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml index 640801c35de4..a116d5a38f58 100644 --- a/operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/create-daemonset/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml b/operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml index c5383a670125..78fc02f1d2fa 100644 --- a/operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml +++ b/operations/helm/tests/create-deployment-autoscaling/grafana-agent/templates/controllers/deployment.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml b/operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml index 37fcf8fc0dc6..03b2aded2fe9 100644 --- a/operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml +++ b/operations/helm/tests/create-deployment/grafana-agent/templates/controllers/deployment.yaml @@ -26,7 +26,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml b/operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml index bc9d31d9cf75..5a207ec50a7c 100644 --- a/operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml +++ b/operations/helm/tests/create-statefulset-autoscaling/grafana-agent/templates/controllers/statefulset.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml b/operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml index 7a641636bc13..b5098dfc5a4c 100644 --- a/operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml +++ b/operations/helm/tests/create-statefulset/grafana-agent/templates/controllers/statefulset.yaml @@ -28,7 +28,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml index 640801c35de4..a116d5a38f58 100644 --- a/operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/custom-config/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml index 640801c35de4..a116d5a38f58 100644 --- a/operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/default-values/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml index 640801c35de4..a116d5a38f58 100644 --- a/operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/enable-servicemonitor/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml index ab153416d599..6b3cb1de8285 100644 --- a/operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/envFrom/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml index 39cf2262d661..77603a03b875 100644 --- a/operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/existing-config/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml index 2b4663c47d86..b10a389bb354 100644 --- a/operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/extra-env/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml index 388eb68a16eb..6c7a61831ea4 100644 --- a/operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/extra-ports/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml index 840b64e244cd..941f0eacc25b 100644 --- a/operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/faro-ingress/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml index 93e08d06f66b..45278301718f 100644 --- a/operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/global-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml @@ -30,7 +30,7 @@ spec: - name: global-cred containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml index 83d83473d371..e5fe82219698 100644 --- a/operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/global-image-registry/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: quay.io/grafana/agent:v0.37.4 + image: quay.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/initcontainers/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/initcontainers/grafana-agent/templates/controllers/daemonset.yaml index dea955f56900..2d4dfe42438b 100644 --- a/operations/helm/tests/initcontainers/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/initcontainers/grafana-agent/templates/controllers/daemonset.yaml @@ -43,7 +43,7 @@ spec: name: geoip containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/local-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/local-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml index 8be0679577ea..0c2770ce63f5 100644 --- a/operations/helm/tests/local-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/local-image-pullsecrets/grafana-agent/templates/controllers/daemonset.yaml @@ -27,7 +27,7 @@ spec: - name: local-cred containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/local-image-registry/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/local-image-registry/grafana-agent/templates/controllers/daemonset.yaml index 83d83473d371..e5fe82219698 100644 --- a/operations/helm/tests/local-image-registry/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/local-image-registry/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: quay.io/grafana/agent:v0.37.4 + image: quay.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/nodeselectors-and-tolerations/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/nodeselectors-and-tolerations/grafana-agent/templates/controllers/daemonset.yaml index a9d14c8e2d29..8002413ce8f4 100644 --- a/operations/helm/tests/nodeselectors-and-tolerations/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/nodeselectors-and-tolerations/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - run diff --git a/operations/helm/tests/static-mode/grafana-agent/templates/controllers/daemonset.yaml b/operations/helm/tests/static-mode/grafana-agent/templates/controllers/daemonset.yaml index 5fc4e67fd107..0b53a3db5e3f 100644 --- a/operations/helm/tests/static-mode/grafana-agent/templates/controllers/daemonset.yaml +++ b/operations/helm/tests/static-mode/grafana-agent/templates/controllers/daemonset.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: grafana-agent containers: - name: grafana-agent - image: docker.io/grafana/agent:v0.37.4 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent args: - -config.file=/etc/agent/config.yaml diff --git a/packaging/grafana-agent-flow/windows/install_script.nsis b/packaging/grafana-agent-flow/windows/install_script.nsis index 19f3fc6cd668..011e6b59c121 100644 --- a/packaging/grafana-agent-flow/windows/install_script.nsis +++ b/packaging/grafana-agent-flow/windows/install_script.nsis @@ -31,6 +31,7 @@ OutFile "${OUT}" Var PassedInParameters Var Config Var ConfigFlag +Var Environment Var DisableReporting Var DisableReportingFlag Var DisableProfiling @@ -51,6 +52,7 @@ Section "install" ${GetParameters} $PassedInParameters ${GetOptions} $PassedInParameters "/DISABLEPROFILING=" $DisableProfiling ${GetOptions} $PassedInParameters "/DISABLEREPORTING=" $DisableReporting + ${GetOptions} $PassedInParameters "/ENVIRONMENT=" $Environment ${GetOptions} $PassedInParameters "/CONFIG=" $Config # Calls to functions like nsExec::ExecToLog below push the exit code to the @@ -146,6 +148,11 @@ Function InitializeRegistry Pop $0 # Ignore return result ${EndIf} + # Define the environment key, which holds environment variables to pass to the + # service. + nsExec::ExecToLog 'Reg.exe add "${REGKEY}" /reg:64 /v Environment /t REG_MULTI_SZ /d "$Environment"' + Pop $0 # Ignore return result + Return FunctionEnd diff --git a/pkg/config/agent_management_remote_config_test.go b/pkg/config/agent_management_remote_config_test.go index af97bd70190a..834375bda3ce 100644 --- a/pkg/config/agent_management_remote_config_test.go +++ b/pkg/config/agent_management_remote_config_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + process_exporter "github.com/grafana/agent/pkg/integrations/process_exporter" "github.com/grafana/agent/pkg/metrics/instance" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" @@ -182,6 +183,82 @@ integration_configs: require.Equal(t, 5*time.Second, c.Integrations.ConfigV1.IntegrationRestartBackoff) }) + t.Run("template variables provided", func(t *testing.T) { + baseConfig := ` +server: + log_level: {{.log_level}} +` + templateInsideTemplate := "`{{ .template_inside_template }}`" + snippet := Snippet{ + Config: ` +integration_configs: + process_exporter: + enabled: true + process_names: + - name: "grafana-agent" + cmdline: + - 'grafana-agent' + - name: "{{.nonexistent.foo.bar.baz.bat}}" + cmdline: + - "{{ ` + templateInsideTemplate + ` }}" + # Custom process monitors + {{- range $key, $value := .process_exporter_processes }} + - name: "{{ $value.name }}" + cmdline: + - "{{ $value.cmdline }}" + {{if $value.exe}} + exe: + - "{{ $value.exe }}" + {{end}} + {{- end }} +`, + } + + rc := RemoteConfig{ + BaseConfig: BaseConfigContent(baseConfig), + Snippets: []Snippet{snippet}, + AgentMetadata: AgentMetadata{ + TemplateVariables: map[string]any{ + "log_level": "debug", + "process_exporter_processes": []map[string]string{ + { + "name": "java_processes", + "cmdline": ".*/java", + }, + { + "name": "{{.ExeFull}}:{{.Matches.Cfgfile}}", + "cmdline": `-config.path\\s+(?P<Cfgfile>\\S+)`, + "exe": "/usr/local/bin/process-exporter", + }, + }, + }, + }, + } + + c, err := rc.BuildAgentConfig() + require.NoError(t, err) + require.Equal(t, 1, len(c.Integrations.ConfigV1.Integrations)) + processExporterConfig := c.Integrations.ConfigV1.Integrations[0].Config.(*process_exporter.Config) + + require.Equal(t, 4, len(processExporterConfig.ProcessExporter)) + + require.Equal(t, "grafana-agent", processExporterConfig.ProcessExporter[0].Name) + require.Equal(t, "grafana-agent", processExporterConfig.ProcessExporter[0].CmdlineRules[0]) + require.Equal(t, 0, len(processExporterConfig.ProcessExporter[0].ExeRules)) + + require.Equal(t, "<no value>", processExporterConfig.ProcessExporter[1].Name) + require.Equal(t, "{{ .template_inside_template }}", processExporterConfig.ProcessExporter[1].CmdlineRules[0]) + require.Equal(t, 0, len(processExporterConfig.ProcessExporter[1].ExeRules)) + + require.Equal(t, "java_processes", processExporterConfig.ProcessExporter[2].Name) + require.Equal(t, ".*/java", processExporterConfig.ProcessExporter[2].CmdlineRules[0]) + require.Equal(t, 0, len(processExporterConfig.ProcessExporter[2].ExeRules)) + + require.Equal(t, "{{.ExeFull}}:{{.Matches.Cfgfile}}", processExporterConfig.ProcessExporter[3].Name) + require.Equal(t, `-config.path\s+(?P<Cfgfile>\S+)`, processExporterConfig.ProcessExporter[3].CmdlineRules[0]) + require.Equal(t, "/usr/local/bin/process-exporter", processExporterConfig.ProcessExporter[3].ExeRules[0]) + }) + t.Run("no external labels provided", func(t *testing.T) { rc := RemoteConfig{ BaseConfig: BaseConfigContent(baseConfig), diff --git a/pkg/config/agentmanagement_remote_config.go b/pkg/config/agentmanagement_remote_config.go index c1f87615930d..8b5093861381 100644 --- a/pkg/config/agentmanagement_remote_config.go +++ b/pkg/config/agentmanagement_remote_config.go @@ -1,6 +1,9 @@ package config import ( + "bytes" + "text/template" + "github.com/grafana/agent/pkg/integrations" "github.com/grafana/agent/pkg/logs" "github.com/grafana/agent/pkg/metrics/instance" @@ -28,7 +31,8 @@ type ( } AgentMetadata struct { - ExternalLabels map[string]string `json:"external_labels,omitempty" yaml:"external_labels,omitempty"` + ExternalLabels map[string]string `json:"external_labels,omitempty" yaml:"external_labels,omitempty"` + TemplateVariables map[string]any `json:"template_variables,omitempty" yaml:"template_variables,omitempty"` } // SnippetContent defines the internal structure of a snippet configuration. @@ -55,8 +59,13 @@ func NewRemoteConfig(buf []byte) (*RemoteConfig, error) { // BuildAgentConfig builds an agent configuration from a base config and a list of snippets func (rc *RemoteConfig) BuildAgentConfig() (*Config, error) { + baseConfig, err := evaluateTemplate(string(rc.BaseConfig), rc.AgentMetadata.TemplateVariables) + if err != nil { + return nil, err + } + c := DefaultConfig() - err := yaml.Unmarshal([]byte(rc.BaseConfig), &c) + err = yaml.Unmarshal([]byte(baseConfig), &c) if err != nil { return nil, err } @@ -66,7 +75,7 @@ func (rc *RemoteConfig) BuildAgentConfig() (*Config, error) { return nil, err } - err = appendSnippets(&c, rc.Snippets) + err = appendSnippets(&c, rc.Snippets, rc.AgentMetadata.TemplateVariables) if err != nil { return nil, err } @@ -74,7 +83,7 @@ func (rc *RemoteConfig) BuildAgentConfig() (*Config, error) { return &c, nil } -func appendSnippets(c *Config, snippets []Snippet) error { +func appendSnippets(c *Config, snippets []Snippet, templateVars map[string]any) error { metricsConfigs := instance.DefaultConfig metricsConfigs.Name = "snippets" logsConfigs := logs.InstanceConfig{ @@ -91,8 +100,13 @@ func appendSnippets(c *Config, snippets []Snippet) error { } for _, snippet := range snippets { + snippetConfig, err := evaluateTemplate(snippet.Config, templateVars) + if err != nil { + return err + } + var snippetContent SnippetContent - err := yaml.Unmarshal([]byte(snippet.Config), &snippetContent) + err = yaml.Unmarshal([]byte(snippetConfig), &snippetContent) if err != nil { return err } @@ -148,3 +162,18 @@ func appendExternalLabels(c *Config, externalLabels map[string]string) { c.Logs.Global.ClientConfigs[i].ExternalLabels.LabelSet = logsExternalLabels.Merge(cc.ExternalLabels.LabelSet) } } + +func evaluateTemplate(config string, templateVariables map[string]any) (string, error) { + tpl, err := template.New("config").Parse(config) + if err != nil { + return "", err + } + + var buf bytes.Buffer + err = tpl.Execute(&buf, templateVariables) + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/pkg/integrations/v2/app_agent_receiver/handler.go b/pkg/integrations/v2/app_agent_receiver/handler.go index 6831885cb51b..c430e9099312 100644 --- a/pkg/integrations/v2/app_agent_receiver/handler.go +++ b/pkg/integrations/v2/app_agent_receiver/handler.go @@ -117,7 +117,7 @@ func (ar *AppAgentReceiverHandler) HTTPHandler(logger log.Logger) http.Handler { if len(ar.config.Server.CORSAllowedOrigins) > 0 { c := cors.New(cors.Options{ AllowedOrigins: ar.config.Server.CORSAllowedOrigins, - AllowedHeaders: []string{apiKeyHeader, "content-type"}, + AllowedHeaders: []string{apiKeyHeader, "content-type", "x-faro-session-id"}, }) handler = c.Handler(handler) } diff --git a/pkg/integrations/windows_exporter/config.go b/pkg/integrations/windows_exporter/config.go index 37f4e249a8fa..006bc5426d72 100644 --- a/pkg/integrations/windows_exporter/config.go +++ b/pkg/integrations/windows_exporter/config.go @@ -7,71 +7,6 @@ import ( "github.com/grafana/agent/pkg/integrations/v2/metricsutils" ) -// DefaultConfig holds the default settings for the windows_exporter integration. -var DefaultConfig = Config{ - // NOTE(rfratto): there is an init function in config_windows.go that - // populates defaults for collectors based on the exporter defaults. - EnabledCollectors: "cpu,cs,logical_disk,net,os,service,system", - - Dfsr: DfsrConfig{ - SourcesEnabled: "", - }, - Exchange: ExchangeConfig{ - EnabledList: "", - }, - IIS: IISConfig{ - AppBlackList: "", - AppWhiteList: "", - SiteBlackList: "", - SiteWhiteList: "", - AppInclude: "", - AppExclude: "", - SiteInclude: "", - SiteExclude: "", - }, - LogicalDisk: LogicalDiskConfig{ - BlackList: "", - WhiteList: "", - Include: "", - Exclude: "", - }, - MSMQ: MSMQConfig{ - Where: "", - }, - MSSQL: MSSQLConfig{ - EnabledClasses: "", - }, - Network: NetworkConfig{ - BlackList: "", - WhiteList: "", - Include: "", - Exclude: "", - }, - Process: ProcessConfig{ - BlackList: "", - WhiteList: "", - Include: "", - Exclude: "", - }, - ScheduledTask: ScheduledTaskConfig{ - Include: "", - Exclude: "", - }, - Service: ServiceConfig{ - UseApi: "", - Where: "", - }, - SMTP: SMTPConfig{ - BlackList: "", - WhiteList: "", - Include: "", - Exclude: "", - }, - TextFile: TextFileConfig{ - TextFileDirectory: "", - }, -} - func init() { integrations.RegisterIntegration(&Config{}) integrations_v2.RegisterLegacy(&Config{}, integrations_v2.TypeSingleton, metricsutils.NewNamedShim("windows")) @@ -96,14 +31,6 @@ type Config struct { ScheduledTask ScheduledTaskConfig `yaml:"scheduled_task,omitempty"` } -// UnmarshalYAML implements yaml.Unmarshaler for Config. -func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = DefaultConfig - - type plain Config - return unmarshal((*plain)(c)) -} - // Name returns the name used, "windows_explorer" func (c *Config) Name() string { return "windows_exporter" diff --git a/pkg/integrations/windows_exporter/config_windows.go b/pkg/integrations/windows_exporter/config_windows.go index cb556363d084..17fd03d4f80c 100644 --- a/pkg/integrations/windows_exporter/config_windows.go +++ b/pkg/integrations/windows_exporter/config_windows.go @@ -1,6 +1,8 @@ package windows_exporter -import "github.com/prometheus-community/windows_exporter/pkg/collector" +import ( + "github.com/prometheus-community/windows_exporter/pkg/collector" +) func (c *Config) ToWindowsExporterConfig() collector.Config { cfg := collector.ConfigDefaults @@ -47,3 +49,73 @@ func coalesceString(v ...string) string { } return "" } + +// DefaultConfig holds the default settings for the windows_exporter integration. +var DefaultConfig = Config{ + EnabledCollectors: "cpu,cs,logical_disk,net,os,service,system", + Dfsr: DfsrConfig{ + SourcesEnabled: collector.ConfigDefaults.Dfsr.DfsrEnabledCollectors, + }, + Exchange: ExchangeConfig{ + EnabledList: collector.ConfigDefaults.Exchange.CollectorsEnabled, + }, + IIS: IISConfig{ + AppBlackList: collector.ConfigDefaults.Iis.AppExclude, + AppWhiteList: collector.ConfigDefaults.Iis.AppInclude, + SiteBlackList: collector.ConfigDefaults.Iis.SiteExclude, + SiteWhiteList: collector.ConfigDefaults.Iis.SiteInclude, + AppInclude: collector.ConfigDefaults.Iis.AppInclude, + AppExclude: collector.ConfigDefaults.Iis.AppExclude, + SiteInclude: collector.ConfigDefaults.Iis.SiteInclude, + SiteExclude: collector.ConfigDefaults.Iis.SiteExclude, + }, + LogicalDisk: LogicalDiskConfig{ + BlackList: collector.ConfigDefaults.LogicalDisk.VolumeExclude, + WhiteList: collector.ConfigDefaults.LogicalDisk.VolumeInclude, + Include: collector.ConfigDefaults.LogicalDisk.VolumeInclude, + Exclude: collector.ConfigDefaults.LogicalDisk.VolumeExclude, + }, + MSMQ: MSMQConfig{ + Where: collector.ConfigDefaults.Msmq.QueryWhereClause, + }, + MSSQL: MSSQLConfig{ + EnabledClasses: collector.ConfigDefaults.Mssql.EnabledCollectors, + }, + Network: NetworkConfig{ + BlackList: collector.ConfigDefaults.Net.NicExclude, + WhiteList: collector.ConfigDefaults.Net.NicInclude, + Include: collector.ConfigDefaults.Net.NicInclude, + Exclude: collector.ConfigDefaults.Net.NicExclude, + }, + Process: ProcessConfig{ + BlackList: collector.ConfigDefaults.Process.ProcessExclude, + WhiteList: collector.ConfigDefaults.Process.ProcessInclude, + Include: collector.ConfigDefaults.Process.ProcessInclude, + Exclude: collector.ConfigDefaults.Process.ProcessExclude, + }, + ScheduledTask: ScheduledTaskConfig{ + Include: collector.ConfigDefaults.ScheduledTask.TaskInclude, + Exclude: collector.ConfigDefaults.ScheduledTask.TaskExclude, + }, + Service: ServiceConfig{ + UseApi: "false", + Where: collector.ConfigDefaults.Service.ServiceWhereClause, + }, + SMTP: SMTPConfig{ + BlackList: collector.ConfigDefaults.Smtp.ServerExclude, + WhiteList: collector.ConfigDefaults.Smtp.ServerInclude, + Include: collector.ConfigDefaults.Smtp.ServerInclude, + Exclude: collector.ConfigDefaults.Smtp.ServerExclude, + }, + TextFile: TextFileConfig{ + TextFileDirectory: collector.ConfigDefaults.Textfile.TextFileDirectories, + }, +} + +// UnmarshalYAML implements yaml.Unmarshaler for Config. +func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultConfig + + type plain Config + return unmarshal((*plain)(c)) +} diff --git a/pkg/operator/defaults.go b/pkg/operator/defaults.go index 27d19feb218d..72aa6b12d6a3 100644 --- a/pkg/operator/defaults.go +++ b/pkg/operator/defaults.go @@ -2,7 +2,7 @@ package operator // Supported versions of the Grafana Agent. var ( - DefaultAgentVersion = "v0.38.0-rc.1" + DefaultAgentVersion = "v0.38.0" DefaultAgentBaseImage = "grafana/agent" DefaultAgentImage = DefaultAgentBaseImage + ":" + DefaultAgentVersion ) diff --git a/tools/gen-versioned-files/agent-version.txt b/tools/gen-versioned-files/agent-version.txt index 840190248a93..765098dc40e9 100644 --- a/tools/gen-versioned-files/agent-version.txt +++ b/tools/gen-versioned-files/agent-version.txt @@ -1 +1 @@ -v0.38.0-rc.1 +v0.38.0