From 9f4d6bc8beca47f823e40492a13632a7cc240ff6 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Fri, 15 Dec 2023 12:43:34 -0500
Subject: [PATCH 01/18] pull out agentseed package for reading and storing
 uuid. Also put it in data dir for flow.

---
 cmd/grafana-agent/entrypoint.go  |  2 +-
 cmd/internal/flowmode/cmd_run.go |  2 +-
 internal/agentseed/uuid.go       | 99 ++++++++++++++++++++++++++++++++
 pkg/usagestats/reporter.go       | 78 +++----------------------
 pkg/usagestats/stats.go          |  3 +-
 5 files changed, 111 insertions(+), 73 deletions(-)
 create mode 100644 internal/agentseed/uuid.go

diff --git a/cmd/grafana-agent/entrypoint.go b/cmd/grafana-agent/entrypoint.go
index c0b36845d330..7fa16efddbe1 100644
--- a/cmd/grafana-agent/entrypoint.go
+++ b/cmd/grafana-agent/entrypoint.go
@@ -98,7 +98,7 @@ func NewEntrypoint(logger *server.Logger, cfg *config.Config, reloader Reloader)
 		return nil, err
 	}
 
-	ep.reporter, err = usagestats.NewReporter(logger)
+	ep.reporter, err = usagestats.NewReporter(logger, "")
 	if err != nil {
 		return nil, err
 	}
diff --git a/cmd/internal/flowmode/cmd_run.go b/cmd/internal/flowmode/cmd_run.go
index 92254ad6e691..88d9ada00f65 100644
--- a/cmd/internal/flowmode/cmd_run.go
+++ b/cmd/internal/flowmode/cmd_run.go
@@ -288,7 +288,7 @@ func (fr *flowRun) Run(configPath string) error {
 
 	// Report usage of enabled components
 	if !fr.disableReporting {
-		reporter, err := usagestats.NewReporter(l)
+		reporter, err := usagestats.NewReporter(l, fr.storagePath)
 		if err != nil {
 			return fmt.Errorf("failed to create reporter: %w", err)
 		}
diff --git a/internal/agentseed/uuid.go b/internal/agentseed/uuid.go
new file mode 100644
index 000000000000..13c380586548
--- /dev/null
+++ b/internal/agentseed/uuid.go
@@ -0,0 +1,99 @@
+package agentseed
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"runtime"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/common/version"
+)
+
+// AgentSeed identifies a unique agent
+type AgentSeed struct {
+	UID       string    `json:"UID"`
+	CreatedAt time.Time `json:"created_at"`
+	Version   string    `json:"version"`
+}
+
+const filename = "agent_seed.json"
+
+var savedSeed *AgentSeed
+
+// Get will return a unique uuid for this agent.
+// Seed will be saved in agent_seed.json
+// If path is not empty, that will be the "preferred" place to read and save it.
+// If it is empty, we will fall back to $APPDATA on windows or /tmp on *nix systems.
+func Get(path string) (seed *AgentSeed, err error) {
+	if savedSeed != nil {
+		return savedSeed, nil
+	}
+	defer func() {
+		if err == nil && seed != nil {
+			savedSeed = seed
+		}
+	}()
+	paths := []string{}
+	if path != "" {
+		paths = append(paths, filepath.Join(path, filename))
+	}
+	paths = append(paths, legacyPath())
+	for i, p := range paths {
+		if fileExists(p) {
+			if seed, err = readSeedFile(p); err == nil {
+				if i == 0 {
+					// we found it at the preferred path. Just return it
+					return seed, err
+				} else {
+					return seed, writeSeedFile(seed, paths[0])
+				}
+			}
+		}
+	}
+	seed = &AgentSeed{
+		UID:       uuid.NewString(),
+		Version:   version.Version,
+		CreatedAt: time.Now(),
+	}
+	return seed, writeSeedFile(seed, paths[0])
+}
+
+// readSeedFile reads the agent seed file
+func readSeedFile(path string) (*AgentSeed, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	seed := &AgentSeed{}
+	err = json.Unmarshal(data, seed)
+	if err != nil {
+		return nil, err
+	}
+	return seed, nil
+}
+
+func legacyPath() string {
+	// windows
+	if runtime.GOOS == "windows" {
+		return filepath.Join(os.Getenv("APPDATA"), filename)
+	}
+	// linux/mac
+	return filepath.Join("/tmp", filename)
+}
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return !errors.Is(err, os.ErrNotExist)
+}
+
+// writeSeedFile writes the agent seed file
+func writeSeedFile(seed *AgentSeed, path string) error {
+	data, err := json.Marshal(*seed)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, data, 0644)
+}
diff --git a/pkg/usagestats/reporter.go b/pkg/usagestats/reporter.go
index 2c1ac8fa43ba..18167328a831 100644
--- a/pkg/usagestats/reporter.go
+++ b/pkg/usagestats/reporter.go
@@ -2,20 +2,14 @@ package usagestats
 
 import (
 	"context"
-	"encoding/json"
-	"errors"
 	"math"
-	"os"
-	"path/filepath"
-	"runtime"
 	"time"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/google/uuid"
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/dskit/backoff"
 	"github.com/grafana/dskit/multierror"
-	"github.com/prometheus/common/version"
 )
 
 var (
@@ -27,85 +21,29 @@ var (
 type Reporter struct {
 	logger log.Logger
 
-	agentSeed  *AgentSeed
+	agentSeed  *agentseed.AgentSeed
+	dataDir    string
 	lastReport time.Time
 }
 
-// AgentSeed identifies a unique agent
-type AgentSeed struct {
-	UID       string    `json:"UID"`
-	CreatedAt time.Time `json:"created_at"`
-	Version   string    `json:"version"`
-}
-
 // NewReporter creates a Reporter that will send periodically reports to grafana.com
-func NewReporter(logger log.Logger) (*Reporter, error) {
+func NewReporter(logger log.Logger, dataDir string) (*Reporter, error) {
 	r := &Reporter{
-		logger: logger,
+		logger:  logger,
+		dataDir: dataDir,
 	}
 	return r, nil
 }
 
-func (rep *Reporter) init(ctx context.Context) error {
-	path := agentSeedFileName()
-
-	if fileExists(path) {
-		seed, err := rep.readSeedFile(path)
-		rep.agentSeed = seed
-		return err
-	}
-	rep.agentSeed = &AgentSeed{
-		UID:       uuid.NewString(),
-		Version:   version.Version,
-		CreatedAt: time.Now(),
-	}
-	return rep.writeSeedFile(*rep.agentSeed, path)
-}
-
-func fileExists(path string) bool {
-	_, err := os.Stat(path)
-	return !errors.Is(err, os.ErrNotExist)
-}
-
-// readSeedFile reads the agent seed file
-func (rep *Reporter) readSeedFile(path string) (*AgentSeed, error) {
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	seed := &AgentSeed{}
-	err = json.Unmarshal(data, seed)
-	if err != nil {
-		return nil, err
-	}
-	return seed, nil
-}
-
-// writeSeedFile writes the agent seed file
-func (rep *Reporter) writeSeedFile(seed AgentSeed, path string) error {
-	data, err := json.Marshal(seed)
-	if err != nil {
-		return err
-	}
-	return os.WriteFile(path, data, 0644)
-}
-
-func agentSeedFileName() string {
-	if runtime.GOOS == "windows" {
-		return filepath.Join(os.Getenv("APPDATA"), "agent_seed.json")
-	}
-	// linux/mac
-	return "/tmp/agent_seed.json"
-}
-
 // Start inits the reporter seed and start sending report for every interval
 func (rep *Reporter) Start(ctx context.Context, metricsFunc func() map[string]interface{}) error {
 	level.Info(rep.logger).Log("msg", "running usage stats reporter")
-	err := rep.init(ctx)
+	seed, err := agentseed.Get(rep.dataDir)
 	if err != nil {
 		level.Info(rep.logger).Log("msg", "failed to init seed", "err", err)
 		return err
 	}
+	rep.agentSeed = seed
 
 	// check every minute if we should report.
 	ticker := time.NewTicker(reportCheckInterval)
diff --git a/pkg/usagestats/stats.go b/pkg/usagestats/stats.go
index d3418a5c4bde..004bc6e6d0e6 100644
--- a/pkg/usagestats/stats.go
+++ b/pkg/usagestats/stats.go
@@ -10,6 +10,7 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/agent/internal/useragent"
 	"github.com/prometheus/common/version"
 )
@@ -31,7 +32,7 @@ type Report struct {
 	DeployMode   string                 `json:"deployMode"`
 }
 
-func sendReport(ctx context.Context, seed *AgentSeed, interval time.Time, metrics map[string]interface{}) error {
+func sendReport(ctx context.Context, seed *agentseed.AgentSeed, interval time.Time, metrics map[string]interface{}) error {
 	report := Report{
 		UsageStatsID: seed.UID,
 		CreatedAt:    seed.CreatedAt,

From 21c1c8394f1761d02ac5e9fec143670827075367 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Fri, 15 Dec 2023 15:32:33 -0500
Subject: [PATCH 02/18] add uid header to prometheus.remote_write and
 loki.write

---
 cmd/grafana-agent/entrypoint.go           |  4 +++-
 cmd/internal/flowmode/cmd_run.go          |  5 ++++-
 component/loki/write/types.go             | 11 +++++++++++
 component/prometheus/remotewrite/types.go | 12 +++++++++++-
 internal/agentseed/uuid.go                | 22 ++++++++++++++++++----
 pkg/usagestats/reporter.go                |  8 +++-----
 6 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/cmd/grafana-agent/entrypoint.go b/cmd/grafana-agent/entrypoint.go
index 7fa16efddbe1..dc40c1db3651 100644
--- a/cmd/grafana-agent/entrypoint.go
+++ b/cmd/grafana-agent/entrypoint.go
@@ -17,6 +17,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/gorilla/mux"
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/agent/pkg/config"
 	"github.com/grafana/agent/pkg/logs"
 	"github.com/grafana/agent/pkg/metrics"
@@ -98,7 +99,8 @@ func NewEntrypoint(logger *server.Logger, cfg *config.Config, reloader Reloader)
 		return nil, err
 	}
 
-	ep.reporter, err = usagestats.NewReporter(logger, "")
+	agentseed.Logger = logger
+	ep.reporter, err = usagestats.NewReporter(logger)
 	if err != nil {
 		return nil, err
 	}
diff --git a/cmd/internal/flowmode/cmd_run.go b/cmd/internal/flowmode/cmd_run.go
index 88d9ada00f65..87aaf8ba542b 100644
--- a/cmd/internal/flowmode/cmd_run.go
+++ b/cmd/internal/flowmode/cmd_run.go
@@ -18,6 +18,7 @@ import (
 	"github.com/grafana/agent/component"
 	"github.com/grafana/agent/converter"
 	convert_diag "github.com/grafana/agent/converter/diag"
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/agent/pkg/boringcrypto"
 	"github.com/grafana/agent/pkg/config/instrumentation"
 	"github.com/grafana/agent/pkg/flow"
@@ -246,6 +247,8 @@ func (fr *flowRun) Run(configPath string) error {
 	}
 
 	labelService := labelstore.New(l, reg)
+	agentseed.DataDir = fr.storagePath
+	agentseed.Logger = l
 
 	f := flow.New(flow.Options{
 		Logger:   l,
@@ -288,7 +291,7 @@ func (fr *flowRun) Run(configPath string) error {
 
 	// Report usage of enabled components
 	if !fr.disableReporting {
-		reporter, err := usagestats.NewReporter(l, fr.storagePath)
+		reporter, err := usagestats.NewReporter(l)
 		if err != nil {
 			return fmt.Errorf("failed to create reporter: %w", err)
 		}
diff --git a/component/loki/write/types.go b/component/loki/write/types.go
index dc240c675e98..9f5d395197c9 100644
--- a/component/loki/write/types.go
+++ b/component/loki/write/types.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/grafana/agent/component/common/loki/client"
 	"github.com/grafana/agent/component/common/loki/utils"
+	"github.com/grafana/agent/internal/agentseed"
 
 	"github.com/alecthomas/units"
 	types "github.com/grafana/agent/component/common/config"
@@ -88,7 +89,17 @@ func (q *QueueConfig) SetToDefault() {
 
 func (args Arguments) convertClientConfigs() []client.Config {
 	var res []client.Config
+	uid := ""
+	if seed, err := agentseed.Get(); err == nil {
+		uid = seed.UID
+	}
 	for _, cfg := range args.Endpoints {
+		if uid != "" {
+			if cfg.Headers == nil {
+				cfg.Headers = map[string]string{}
+			}
+			cfg.Headers["X-Agent-Uid"] = uid
+		}
 		url, _ := url.Parse(cfg.URL)
 		cc := client.Config{
 			Name:      cfg.Name,
diff --git a/component/prometheus/remotewrite/types.go b/component/prometheus/remotewrite/types.go
index 39ef6a55a191..f27e83b47803 100644
--- a/component/prometheus/remotewrite/types.go
+++ b/component/prometheus/remotewrite/types.go
@@ -8,6 +8,7 @@ import (
 
 	types "github.com/grafana/agent/component/common/config"
 	flow_relabel "github.com/grafana/agent/component/common/relabel"
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/river/rivertypes"
 
 	"github.com/google/uuid"
@@ -226,12 +227,21 @@ type Exports struct {
 
 func convertConfigs(cfg Arguments) (*config.Config, error) {
 	var rwConfigs []*config.RemoteWriteConfig
+	uid := ""
+	if seed, err := agentseed.Get(); err == nil {
+		uid = seed.UID
+	}
 	for _, rw := range cfg.Endpoints {
 		parsedURL, err := url.Parse(rw.URL)
 		if err != nil {
 			return nil, fmt.Errorf("cannot parse remote_write url %q: %w", rw.URL, err)
 		}
-
+		if uid != "" {
+			if rw.Headers == nil {
+				rw.Headers = map[string]string{}
+			}
+			rw.Headers["X-Agent-Uid"] = uid
+		}
 		rwConfigs = append(rwConfigs, &config.RemoteWriteConfig{
 			URL:                  &common.URL{URL: parsedURL},
 			RemoteTimeout:        model.Duration(rw.RemoteTimeout),
diff --git a/internal/agentseed/uuid.go b/internal/agentseed/uuid.go
index 13c380586548..f6c9811a6afb 100644
--- a/internal/agentseed/uuid.go
+++ b/internal/agentseed/uuid.go
@@ -8,7 +8,9 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/go-kit/log"
 	"github.com/google/uuid"
+	"github.com/grafana/agent/pkg/flow/logging/level"
 	"github.com/prometheus/common/version"
 )
 
@@ -21,13 +23,17 @@ type AgentSeed struct {
 
 const filename = "agent_seed.json"
 
+// DataDir should be set by an app entrypoint to the data dir to store the agent_seed.json
+var DataDir = ""
+var Logger log.Logger
+
 var savedSeed *AgentSeed
 
 // Get will return a unique uuid for this agent.
 // Seed will be saved in agent_seed.json
 // If path is not empty, that will be the "preferred" place to read and save it.
 // If it is empty, we will fall back to $APPDATA on windows or /tmp on *nix systems.
-func Get(path string) (seed *AgentSeed, err error) {
+func Get() (seed *AgentSeed, err error) {
 	if savedSeed != nil {
 		return savedSeed, nil
 	}
@@ -37,8 +43,8 @@ func Get(path string) (seed *AgentSeed, err error) {
 		}
 	}()
 	paths := []string{}
-	if path != "" {
-		paths = append(paths, filepath.Join(path, filename))
+	if DataDir != "" {
+		paths = append(paths, filepath.Join(DataDir, filename))
 	}
 	paths = append(paths, legacyPath())
 	for i, p := range paths {
@@ -65,11 +71,13 @@ func Get(path string) (seed *AgentSeed, err error) {
 func readSeedFile(path string) (*AgentSeed, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
+		level.Error(Logger).Log("msg", "Reading seed file", "err", err)
 		return nil, err
 	}
 	seed := &AgentSeed{}
 	err = json.Unmarshal(data, seed)
 	if err != nil {
+		level.Error(Logger).Log("msg", "Decoding seed file", "err", err)
 		return nil, err
 	}
 	return seed, nil
@@ -93,7 +101,13 @@ func fileExists(path string) bool {
 func writeSeedFile(seed *AgentSeed, path string) error {
 	data, err := json.Marshal(*seed)
 	if err != nil {
+		level.Error(Logger).Log("msg", "Encoding seed file", "err", err)
+		return err
+	}
+	err = os.WriteFile(path, data, 0644)
+	if err != nil {
+		level.Error(Logger).Log("msg", "Writing seed file", "err", err)
 		return err
 	}
-	return os.WriteFile(path, data, 0644)
+	return err
 }
diff --git a/pkg/usagestats/reporter.go b/pkg/usagestats/reporter.go
index 18167328a831..81ea6dc06ebf 100644
--- a/pkg/usagestats/reporter.go
+++ b/pkg/usagestats/reporter.go
@@ -22,15 +22,13 @@ type Reporter struct {
 	logger log.Logger
 
 	agentSeed  *agentseed.AgentSeed
-	dataDir    string
 	lastReport time.Time
 }
 
 // NewReporter creates a Reporter that will send periodically reports to grafana.com
-func NewReporter(logger log.Logger, dataDir string) (*Reporter, error) {
+func NewReporter(logger log.Logger) (*Reporter, error) {
 	r := &Reporter{
-		logger:  logger,
-		dataDir: dataDir,
+		logger: logger,
 	}
 	return r, nil
 }
@@ -38,7 +36,7 @@ func NewReporter(logger log.Logger, dataDir string) (*Reporter, error) {
 // Start inits the reporter seed and start sending report for every interval
 func (rep *Reporter) Start(ctx context.Context, metricsFunc func() map[string]interface{}) error {
 	level.Info(rep.logger).Log("msg", "running usage stats reporter")
-	seed, err := agentseed.Get(rep.dataDir)
+	seed, err := agentseed.Get()
 	if err != nil {
 		level.Info(rep.logger).Log("msg", "failed to init seed", "err", err)
 		return err

From 7a5713818234c0907c0bf0060378b519761305dd Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 12:51:38 -0500
Subject: [PATCH 03/18] init func

---
 cmd/grafana-agent/entrypoint.go  |  2 +-
 cmd/internal/flowmode/cmd_run.go |  3 +--
 internal/agentseed/uuid.go       | 31 +++++++++++++++++++++----------
 3 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/cmd/grafana-agent/entrypoint.go b/cmd/grafana-agent/entrypoint.go
index dc40c1db3651..172bbf8c58bb 100644
--- a/cmd/grafana-agent/entrypoint.go
+++ b/cmd/grafana-agent/entrypoint.go
@@ -99,7 +99,7 @@ func NewEntrypoint(logger *server.Logger, cfg *config.Config, reloader Reloader)
 		return nil, err
 	}
 
-	agentseed.Logger = logger
+	agentseed.Init("", logger)
 	ep.reporter, err = usagestats.NewReporter(logger)
 	if err != nil {
 		return nil, err
diff --git a/cmd/internal/flowmode/cmd_run.go b/cmd/internal/flowmode/cmd_run.go
index 87aaf8ba542b..24eecb9e5193 100644
--- a/cmd/internal/flowmode/cmd_run.go
+++ b/cmd/internal/flowmode/cmd_run.go
@@ -247,8 +247,7 @@ func (fr *flowRun) Run(configPath string) error {
 	}
 
 	labelService := labelstore.New(l, reg)
-	agentseed.DataDir = fr.storagePath
-	agentseed.Logger = l
+	agentseed.Init(fr.storagePath, l)
 
 	f := flow.New(flow.Options{
 		Logger:   l,
diff --git a/internal/agentseed/uuid.go b/internal/agentseed/uuid.go
index f6c9811a6afb..e36cf1af182d 100644
--- a/internal/agentseed/uuid.go
+++ b/internal/agentseed/uuid.go
@@ -23,17 +23,26 @@ type AgentSeed struct {
 
 const filename = "agent_seed.json"
 
-// DataDir should be set by an app entrypoint to the data dir to store the agent_seed.json
-var DataDir = ""
-var Logger log.Logger
+var dataDir = ""
+var logger log.Logger
 
 var savedSeed *AgentSeed
 
+// Init should be called by an app entrypoint as soon as it can to configure where the unique seed will be stored.
+// dir is the directory where we will read and store agent_seed.json
+// If left empty it will default to $APPDATA or /tmp
+func Init(dir string, log log.Logger) {
+	dataDir = dir
+	logger = log
+}
+
 // Get will return a unique uuid for this agent.
 // Seed will be saved in agent_seed.json
 // If path is not empty, that will be the "preferred" place to read and save it.
-// If it is empty, we will fall back to $APPDATA on windows or /tmp on *nix systems.
+// If it is empty, we will fall back to $APPDATA on windows or /tmp on *nix systems to read the file.
 func Get() (seed *AgentSeed, err error) {
+	// TODO: should this ever return an error?
+	// it could just log issues, and return a generated uid if file reads fail.
 	if savedSeed != nil {
 		return savedSeed, nil
 	}
@@ -42,9 +51,11 @@ func Get() (seed *AgentSeed, err error) {
 			savedSeed = seed
 		}
 	}()
+	// list of paths in preference order.
+	// we will always write to the first path
 	paths := []string{}
-	if DataDir != "" {
-		paths = append(paths, filepath.Join(DataDir, filename))
+	if dataDir != "" {
+		paths = append(paths, filepath.Join(dataDir, filename))
 	}
 	paths = append(paths, legacyPath())
 	for i, p := range paths {
@@ -71,13 +82,13 @@ func Get() (seed *AgentSeed, err error) {
 func readSeedFile(path string) (*AgentSeed, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
-		level.Error(Logger).Log("msg", "Reading seed file", "err", err)
+		level.Error(logger).Log("msg", "Reading seed file", "err", err)
 		return nil, err
 	}
 	seed := &AgentSeed{}
 	err = json.Unmarshal(data, seed)
 	if err != nil {
-		level.Error(Logger).Log("msg", "Decoding seed file", "err", err)
+		level.Error(logger).Log("msg", "Decoding seed file", "err", err)
 		return nil, err
 	}
 	return seed, nil
@@ -101,12 +112,12 @@ func fileExists(path string) bool {
 func writeSeedFile(seed *AgentSeed, path string) error {
 	data, err := json.Marshal(*seed)
 	if err != nil {
-		level.Error(Logger).Log("msg", "Encoding seed file", "err", err)
+		level.Error(logger).Log("msg", "Encoding seed file", "err", err)
 		return err
 	}
 	err = os.WriteFile(path, data, 0644)
 	if err != nil {
-		level.Error(Logger).Log("msg", "Writing seed file", "err", err)
+		level.Error(logger).Log("msg", "Writing seed file", "err", err)
 		return err
 	}
 	return err

From c994c9a8fc5413e2646c4cd8a14a9a459d9a0c86 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 13:01:52 -0500
Subject: [PATCH 04/18] cleaner api with fewer edge cases

---
 component/loki/write/types.go                | 14 +++---
 component/prometheus/remotewrite/types.go    | 14 +++---
 internal/agentseed/{uuid.go => agentseed.go} | 45 ++++++++++++--------
 3 files changed, 38 insertions(+), 35 deletions(-)
 rename internal/agentseed/{uuid.go => agentseed.go} (77%)

diff --git a/component/loki/write/types.go b/component/loki/write/types.go
index 9f5d395197c9..86eeadf515c0 100644
--- a/component/loki/write/types.go
+++ b/component/loki/write/types.go
@@ -89,17 +89,13 @@ func (q *QueueConfig) SetToDefault() {
 
 func (args Arguments) convertClientConfigs() []client.Config {
 	var res []client.Config
-	uid := ""
-	if seed, err := agentseed.Get(); err == nil {
-		uid = seed.UID
-	}
+	uid := agentseed.Get().UID
 	for _, cfg := range args.Endpoints {
-		if uid != "" {
-			if cfg.Headers == nil {
-				cfg.Headers = map[string]string{}
-			}
-			cfg.Headers["X-Agent-Uid"] = uid
+		if cfg.Headers == nil {
+			cfg.Headers = map[string]string{}
 		}
+		cfg.Headers["X-Agent-UID"] = uid
+
 		url, _ := url.Parse(cfg.URL)
 		cc := client.Config{
 			Name:      cfg.Name,
diff --git a/component/prometheus/remotewrite/types.go b/component/prometheus/remotewrite/types.go
index f27e83b47803..d8445084a6ac 100644
--- a/component/prometheus/remotewrite/types.go
+++ b/component/prometheus/remotewrite/types.go
@@ -227,21 +227,17 @@ type Exports struct {
 
 func convertConfigs(cfg Arguments) (*config.Config, error) {
 	var rwConfigs []*config.RemoteWriteConfig
-	uid := ""
-	if seed, err := agentseed.Get(); err == nil {
-		uid = seed.UID
-	}
+	uid := agentseed.Get().UID
 	for _, rw := range cfg.Endpoints {
 		parsedURL, err := url.Parse(rw.URL)
 		if err != nil {
 			return nil, fmt.Errorf("cannot parse remote_write url %q: %w", rw.URL, err)
 		}
-		if uid != "" {
-			if rw.Headers == nil {
-				rw.Headers = map[string]string{}
-			}
-			rw.Headers["X-Agent-Uid"] = uid
+		if rw.Headers == nil {
+			rw.Headers = map[string]string{}
 		}
+		rw.Headers["X-Agent-UID"] = uid
+
 		rwConfigs = append(rwConfigs, &config.RemoteWriteConfig{
 			URL:                  &common.URL{URL: parsedURL},
 			RemoteTimeout:        model.Duration(rw.RemoteTimeout),
diff --git a/internal/agentseed/uuid.go b/internal/agentseed/agentseed.go
similarity index 77%
rename from internal/agentseed/uuid.go
rename to internal/agentseed/agentseed.go
index e36cf1af182d..612578ae783c 100644
--- a/internal/agentseed/uuid.go
+++ b/internal/agentseed/agentseed.go
@@ -40,42 +40,50 @@ func Init(dir string, log log.Logger) {
 // Seed will be saved in agent_seed.json
 // If path is not empty, that will be the "preferred" place to read and save it.
 // If it is empty, we will fall back to $APPDATA on windows or /tmp on *nix systems to read the file.
-func Get() (seed *AgentSeed, err error) {
-	// TODO: should this ever return an error?
-	// it could just log issues, and return a generated uid if file reads fail.
+func Get() (seed *AgentSeed) {
+	// TODO: This will just log errors and always return a valid seed.
+	// If we wanted to have it return an error for some reason, we could change this api
+	// worst case, we generate a new seed if we can't read/write files, and it is only good for the lifetime
+	// of this agent.
 	if savedSeed != nil {
-		return savedSeed, nil
+		return savedSeed
 	}
-	defer func() {
-		if err == nil && seed != nil {
-			savedSeed = seed
-		}
-	}()
+	var err error
 	// list of paths in preference order.
 	// we will always write to the first path
 	paths := []string{}
 	if dataDir != "" {
 		paths = append(paths, filepath.Join(dataDir, filename))
 	}
+	defer func() {
+		// as a fallback, gen and save a new uid
+		if seed == nil || seed.UID == "" {
+			seed = &AgentSeed{
+				UID:       uuid.NewString(),
+				Version:   version.Version,
+				CreatedAt: time.Now(),
+			}
+			writeSeedFile(seed, paths[0])
+		}
+		// cache seed for future calls
+		savedSeed = seed
+	}()
 	paths = append(paths, legacyPath())
 	for i, p := range paths {
 		if fileExists(p) {
 			if seed, err = readSeedFile(p); err == nil {
 				if i == 0 {
 					// we found it at the preferred path. Just return it
-					return seed, err
+					return seed
 				} else {
-					return seed, writeSeedFile(seed, paths[0])
+					writeSeedFile(seed, paths[0])
+					return seed
 				}
 			}
 		}
 	}
-	seed = &AgentSeed{
-		UID:       uuid.NewString(),
-		Version:   version.Version,
-		CreatedAt: time.Now(),
-	}
-	return seed, writeSeedFile(seed, paths[0])
+
+	return seed
 }
 
 // readSeedFile reads the agent seed file
@@ -91,6 +99,9 @@ func readSeedFile(path string) (*AgentSeed, error) {
 		level.Error(logger).Log("msg", "Decoding seed file", "err", err)
 		return nil, err
 	}
+	if seed.UID == "" {
+		level.Error(logger).Log("msg", "Seed file has empty uid")
+	}
 	return seed, nil
 }
 

From d77d1c7ee6be024494f88e606c7d695ac96a7234 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 13:04:52 -0500
Subject: [PATCH 05/18] add to pyroscope

---
 component/loki/write/types.go      | 1 -
 component/pyroscope/write/write.go | 6 ++++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/component/loki/write/types.go b/component/loki/write/types.go
index 86eeadf515c0..d71359dd3f67 100644
--- a/component/loki/write/types.go
+++ b/component/loki/write/types.go
@@ -95,7 +95,6 @@ func (args Arguments) convertClientConfigs() []client.Config {
 			cfg.Headers = map[string]string{}
 		}
 		cfg.Headers["X-Agent-UID"] = uid
-
 		url, _ := url.Parse(cfg.URL)
 		cc := client.Config{
 			Name:      cfg.Name,
diff --git a/component/pyroscope/write/write.go b/component/pyroscope/write/write.go
index 165a7f20a812..916623d76ae8 100644
--- a/component/pyroscope/write/write.go
+++ b/component/pyroscope/write/write.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/bufbuild/connect-go"
 	"github.com/grafana/agent/component/pyroscope"
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/agent/internal/useragent"
 	"github.com/grafana/agent/pkg/flow/logging/level"
 	"github.com/oklog/run"
@@ -156,7 +157,12 @@ type fanOutClient struct {
 // NewFanOut creates a new fan out client that will fan out to all endpoints.
 func NewFanOut(opts component.Options, config Arguments, metrics *metrics) (*fanOutClient, error) {
 	clients := make([]pushv1connect.PusherServiceClient, 0, len(config.Endpoints))
+	uid := agentseed.Get().UID
 	for _, endpoint := range config.Endpoints {
+		if endpoint.Headers == nil {
+			endpoint.Headers = map[string]string{}
+		}
+		endpoint.Headers["X-Agent-UID"] = uid
 		httpClient, err := commonconfig.NewClientFromConfig(*endpoint.HTTPClientConfig.Convert(), endpoint.Name)
 		if err != nil {
 			return nil, err

From fa578931d7846ee85a808f3f26034a5f05e1c066 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 13:06:00 -0500
Subject: [PATCH 06/18] compile

---
 pkg/usagestats/reporter.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/pkg/usagestats/reporter.go b/pkg/usagestats/reporter.go
index 81ea6dc06ebf..a175e154b92a 100644
--- a/pkg/usagestats/reporter.go
+++ b/pkg/usagestats/reporter.go
@@ -36,12 +36,7 @@ func NewReporter(logger log.Logger) (*Reporter, error) {
 // Start inits the reporter seed and start sending report for every interval
 func (rep *Reporter) Start(ctx context.Context, metricsFunc func() map[string]interface{}) error {
 	level.Info(rep.logger).Log("msg", "running usage stats reporter")
-	seed, err := agentseed.Get()
-	if err != nil {
-		level.Info(rep.logger).Log("msg", "failed to init seed", "err", err)
-		return err
-	}
-	rep.agentSeed = seed
+	rep.agentSeed = agentseed.Get()
 
 	// check every minute if we should report.
 	ticker := time.NewTicker(reportCheckInterval)

From beb872c2e68c7df2d8a1a8916729ff98aa02edaf Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 13:10:53 -0500
Subject: [PATCH 07/18] add to static remote write

---
 pkg/metrics/instance/instance.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pkg/metrics/instance/instance.go b/pkg/metrics/instance/instance.go
index 3d0e9fd47ca7..1f6f5098df2d 100644
--- a/pkg/metrics/instance/instance.go
+++ b/pkg/metrics/instance/instance.go
@@ -17,6 +17,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/agent/internal/useragent"
 	"github.com/grafana/agent/pkg/metrics/wal"
 	"github.com/grafana/agent/pkg/util"
@@ -157,7 +158,7 @@ func (c *Config) ApplyDefaults(global GlobalConfig) error {
 	}
 
 	rwNames := map[string]struct{}{}
-
+	uid := agentseed.Get().UID
 	// If the instance remote write is not filled in, then apply the prometheus write config
 	if len(c.RemoteWrite) == 0 {
 		c.RemoteWrite = c.global.RemoteWrite
@@ -166,6 +167,10 @@ func (c *Config) ApplyDefaults(global GlobalConfig) error {
 		if cfg == nil {
 			return fmt.Errorf("empty or null remote write config section")
 		}
+		if cfg.Headers == nil {
+			cfg.Headers = map[string]string{}
+		}
+		cfg.Headers["X-Agent-UID"] = uid
 
 		// Typically Prometheus ignores empty names here, but we need to assign a
 		// unique name to the config so we can pull metrics from it when running
@@ -183,7 +188,6 @@ func (c *Config) ApplyDefaults(global GlobalConfig) error {
 			cfg.Name = c.Name + "-" + hash[:6]
 			generatedName = true
 		}
-
 		if _, exists := rwNames[cfg.Name]; exists {
 			if generatedName {
 				return fmt.Errorf("found two identical remote_write configs")

From 1b25aeacd33be825c92a5a4ab92cf26581f8c2d4 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 13:13:08 -0500
Subject: [PATCH 08/18] add to static mode loki write

---
 pkg/logs/config.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/pkg/logs/config.go b/pkg/logs/config.go
index dc47ef907ac0..90cc990e8ced 100644
--- a/pkg/logs/config.go
+++ b/pkg/logs/config.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/loki/clients/pkg/promtail/client"
 	"github.com/grafana/loki/clients/pkg/promtail/limit"
 	"github.com/grafana/loki/clients/pkg/promtail/positions"
@@ -75,6 +76,13 @@ func (c *Config) ApplyDefaults() error {
 		if len(ic.ClientConfigs) == 0 {
 			ic.ClientConfigs = c.Global.ClientConfigs
 		}
+		uid := agentseed.Get().UID
+		for _, cfg := range ic.ClientConfigs {
+			if cfg.Headers == nil {
+				cfg.Headers = map[string]string{}
+			}
+			cfg.Headers["X-Agent-UID"] = uid
+		}
 	}
 
 	return nil

From 086482e7056cd167e85d9b1c77ccc10ca289cf30 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 16:22:05 -0500
Subject: [PATCH 09/18] remove return from write. we never need it.

---
 internal/agentseed/agentseed.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/internal/agentseed/agentseed.go b/internal/agentseed/agentseed.go
index 612578ae783c..a4ac12d887b9 100644
--- a/internal/agentseed/agentseed.go
+++ b/internal/agentseed/agentseed.go
@@ -76,6 +76,7 @@ func Get() (seed *AgentSeed) {
 					// we found it at the preferred path. Just return it
 					return seed
 				} else {
+					// it was at a backup path. write it to the preferred path.
 					writeSeedFile(seed, paths[0])
 					return seed
 				}
@@ -120,16 +121,15 @@ func fileExists(path string) bool {
 }
 
 // writeSeedFile writes the agent seed file
-func writeSeedFile(seed *AgentSeed, path string) error {
+func writeSeedFile(seed *AgentSeed, path string) {
 	data, err := json.Marshal(*seed)
 	if err != nil {
 		level.Error(logger).Log("msg", "Encoding seed file", "err", err)
-		return err
+		return
 	}
 	err = os.WriteFile(path, data, 0644)
 	if err != nil {
 		level.Error(logger).Log("msg", "Writing seed file", "err", err)
-		return err
+		return
 	}
-	return err
 }

From 723ad5c2ec7a5541d771edf5496d952965be937f Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 16:44:43 -0500
Subject: [PATCH 10/18] move loki write out of convert function

---
 component/loki/write/types.go | 6 ------
 component/loki/write/write.go | 8 ++++++++
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/component/loki/write/types.go b/component/loki/write/types.go
index d71359dd3f67..dc240c675e98 100644
--- a/component/loki/write/types.go
+++ b/component/loki/write/types.go
@@ -7,7 +7,6 @@ import (
 
 	"github.com/grafana/agent/component/common/loki/client"
 	"github.com/grafana/agent/component/common/loki/utils"
-	"github.com/grafana/agent/internal/agentseed"
 
 	"github.com/alecthomas/units"
 	types "github.com/grafana/agent/component/common/config"
@@ -89,12 +88,7 @@ func (q *QueueConfig) SetToDefault() {
 
 func (args Arguments) convertClientConfigs() []client.Config {
 	var res []client.Config
-	uid := agentseed.Get().UID
 	for _, cfg := range args.Endpoints {
-		if cfg.Headers == nil {
-			cfg.Headers = map[string]string{}
-		}
-		cfg.Headers["X-Agent-UID"] = uid
 		url, _ := url.Parse(cfg.URL)
 		cc := client.Config{
 			Name:      cfg.Name,
diff --git a/component/loki/write/write.go b/component/loki/write/write.go
index a31cb0745976..8524963dca14 100644
--- a/component/loki/write/write.go
+++ b/component/loki/write/write.go
@@ -12,6 +12,7 @@ import (
 	"github.com/grafana/agent/component/common/loki/client"
 	"github.com/grafana/agent/component/common/loki/limit"
 	"github.com/grafana/agent/component/common/loki/wal"
+	"github.com/grafana/agent/internal/agentseed"
 )
 
 func init() {
@@ -159,6 +160,13 @@ func (c *Component) Update(args component.Arguments) error {
 	}
 
 	cfgs := newArgs.convertClientConfigs()
+	uid := agentseed.Get().UID
+	for _, cfg := range cfgs {
+		if cfg.Headers == nil {
+			cfg.Headers = map[string]string{}
+		}
+		cfg.Headers["X-Agent-UID"] = uid
+	}
 	walCfg := wal.Config{
 		Enabled:       newArgs.WAL.Enabled,
 		MaxSegmentAge: newArgs.WAL.MaxSegmentAge,

From 643c8fc49a347f3cca811e86b935ec68fdda5fc4 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 16:46:17 -0500
Subject: [PATCH 11/18] move out of prometheus convert function

---
 component/prometheus/remotewrite/remote_write.go | 8 ++++++++
 component/prometheus/remotewrite/types.go        | 7 -------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/component/prometheus/remotewrite/remote_write.go b/component/prometheus/remotewrite/remote_write.go
index e337ff8f8cca..fadc8a683d20 100644
--- a/component/prometheus/remotewrite/remote_write.go
+++ b/component/prometheus/remotewrite/remote_write.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/component"
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/agent/internal/useragent"
 	"github.com/grafana/agent/pkg/flow/logging/level"
 	"github.com/grafana/agent/pkg/metrics/wal"
@@ -257,6 +258,13 @@ func (c *Component) Update(newConfig component.Arguments) error {
 	if err != nil {
 		return err
 	}
+	uid := agentseed.Get().UID
+	for _, cfg := range convertedConfig.RemoteWriteConfigs {
+		if cfg.Headers == nil {
+			cfg.Headers = map[string]string{}
+		}
+		cfg.Headers["X-Agent-UID"] = uid
+	}
 	err = c.remoteStore.ApplyConfig(convertedConfig)
 	if err != nil {
 		return err
diff --git a/component/prometheus/remotewrite/types.go b/component/prometheus/remotewrite/types.go
index d8445084a6ac..473a2928e246 100644
--- a/component/prometheus/remotewrite/types.go
+++ b/component/prometheus/remotewrite/types.go
@@ -8,7 +8,6 @@ import (
 
 	types "github.com/grafana/agent/component/common/config"
 	flow_relabel "github.com/grafana/agent/component/common/relabel"
-	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/river/rivertypes"
 
 	"github.com/google/uuid"
@@ -227,17 +226,11 @@ type Exports struct {
 
 func convertConfigs(cfg Arguments) (*config.Config, error) {
 	var rwConfigs []*config.RemoteWriteConfig
-	uid := agentseed.Get().UID
 	for _, rw := range cfg.Endpoints {
 		parsedURL, err := url.Parse(rw.URL)
 		if err != nil {
 			return nil, fmt.Errorf("cannot parse remote_write url %q: %w", rw.URL, err)
 		}
-		if rw.Headers == nil {
-			rw.Headers = map[string]string{}
-		}
-		rw.Headers["X-Agent-UID"] = uid
-
 		rwConfigs = append(rwConfigs, &config.RemoteWriteConfig{
 			URL:                  &common.URL{URL: parsedURL},
 			RemoteTimeout:        model.Duration(rw.RemoteTimeout),

From be737c0cd2326cd941f102eb1e8a29986e5ab398 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 16:54:11 -0500
Subject: [PATCH 12/18] static prom, get out of defaults function

---
 pkg/metrics/instance/instance.go | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/pkg/metrics/instance/instance.go b/pkg/metrics/instance/instance.go
index 1f6f5098df2d..b8ae40a4aec2 100644
--- a/pkg/metrics/instance/instance.go
+++ b/pkg/metrics/instance/instance.go
@@ -158,7 +158,6 @@ func (c *Config) ApplyDefaults(global GlobalConfig) error {
 	}
 
 	rwNames := map[string]struct{}{}
-	uid := agentseed.Get().UID
 	// If the instance remote write is not filled in, then apply the prometheus write config
 	if len(c.RemoteWrite) == 0 {
 		c.RemoteWrite = c.global.RemoteWrite
@@ -167,11 +166,6 @@ func (c *Config) ApplyDefaults(global GlobalConfig) error {
 		if cfg == nil {
 			return fmt.Errorf("empty or null remote write config section")
 		}
-		if cfg.Headers == nil {
-			cfg.Headers = map[string]string{}
-		}
-		cfg.Headers["X-Agent-UID"] = uid
-
 		// Typically Prometheus ignores empty names here, but we need to assign a
 		// unique name to the config so we can pull metrics from it when running
 		// an instance.
@@ -423,6 +417,13 @@ func (i *Instance) initialize(ctx context.Context, reg prometheus.Registerer, cf
 	// Set up the remote storage
 	remoteLogger := log.With(i.logger, "component", "remote")
 	i.remoteStore = remote.NewStorage(remoteLogger, reg, i.wal.StartTime, i.wal.Directory(), cfg.RemoteFlushDeadline, i.readyScrapeManager)
+	uid := agentseed.Get().UID
+	for _, rw := range cfg.RemoteWrite {
+		if rw.Headers == nil {
+			rw.Headers = map[string]string{}
+		}
+		rw.Headers["X-Agent-UID"] = uid
+	}
 	err = i.remoteStore.ApplyConfig(&config.Config{
 		GlobalConfig:       cfg.global.Prometheus,
 		RemoteWriteConfigs: cfg.RemoteWrite,

From f611604331857c1049e41d71a46e0c1562dd820c Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Mon, 18 Dec 2023 16:55:42 -0500
Subject: [PATCH 13/18] static logs: take out of defaults function

---
 pkg/logs/config.go | 8 --------
 pkg/logs/logs.go   | 9 +++++++++
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/pkg/logs/config.go b/pkg/logs/config.go
index 90cc990e8ced..dc47ef907ac0 100644
--- a/pkg/logs/config.go
+++ b/pkg/logs/config.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/loki/clients/pkg/promtail/client"
 	"github.com/grafana/loki/clients/pkg/promtail/limit"
 	"github.com/grafana/loki/clients/pkg/promtail/positions"
@@ -76,13 +75,6 @@ func (c *Config) ApplyDefaults() error {
 		if len(ic.ClientConfigs) == 0 {
 			ic.ClientConfigs = c.Global.ClientConfigs
 		}
-		uid := agentseed.Get().UID
-		for _, cfg := range ic.ClientConfigs {
-			if cfg.Headers == nil {
-				cfg.Headers = map[string]string{}
-			}
-			cfg.Headers["X-Agent-UID"] = uid
-		}
 	}
 
 	return nil
diff --git a/pkg/logs/logs.go b/pkg/logs/logs.go
index a821298be9fd..4d92afebec0e 100644
--- a/pkg/logs/logs.go
+++ b/pkg/logs/logs.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/grafana/agent/internal/agentseed"
 	"github.com/grafana/agent/internal/useragent"
 	"github.com/grafana/agent/pkg/util"
 	"github.com/grafana/loki/clients/pkg/promtail"
@@ -183,6 +184,14 @@ func (i *Instance) ApplyConfig(c *InstanceConfig, g GlobalConfig, dryRun bool) e
 		return nil
 	}
 
+	uid := agentseed.Get().UID
+	for _, cfg := range c.ClientConfigs {
+		if cfg.Headers == nil {
+			cfg.Headers = map[string]string{}
+		}
+		cfg.Headers["X-Agent-UID"] = uid
+	}
+
 	clientMetrics := client.NewMetrics(i.reg)
 	cfg := DefaultConfig()
 	cfg.Global = config.GlobalConfig{

From 14796c8844db3f91f978af86d781c57afc48969f Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Tue, 19 Dec 2023 11:31:54 -0500
Subject: [PATCH 14/18] constant for header. Work done in init with sync.once.
 Hardening

---
 component/loki/write/write.go                 |  2 +-
 .../prometheus/remotewrite/remote_write.go    |  2 +-
 component/pyroscope/write/write.go            |  2 +-
 internal/agentseed/agentseed.go               | 86 +++++++++++--------
 pkg/logs/logs.go                              |  2 +-
 pkg/metrics/instance/instance.go              |  2 +-
 6 files changed, 56 insertions(+), 40 deletions(-)

diff --git a/component/loki/write/write.go b/component/loki/write/write.go
index 8524963dca14..65fd04c6f692 100644
--- a/component/loki/write/write.go
+++ b/component/loki/write/write.go
@@ -165,7 +165,7 @@ func (c *Component) Update(args component.Arguments) error {
 		if cfg.Headers == nil {
 			cfg.Headers = map[string]string{}
 		}
-		cfg.Headers["X-Agent-UID"] = uid
+		cfg.Headers[agentseed.HeaderName] = uid
 	}
 	walCfg := wal.Config{
 		Enabled:       newArgs.WAL.Enabled,
diff --git a/component/prometheus/remotewrite/remote_write.go b/component/prometheus/remotewrite/remote_write.go
index fadc8a683d20..354e3248450b 100644
--- a/component/prometheus/remotewrite/remote_write.go
+++ b/component/prometheus/remotewrite/remote_write.go
@@ -263,7 +263,7 @@ func (c *Component) Update(newConfig component.Arguments) error {
 		if cfg.Headers == nil {
 			cfg.Headers = map[string]string{}
 		}
-		cfg.Headers["X-Agent-UID"] = uid
+		cfg.Headers[agentseed.HeaderName] = uid
 	}
 	err = c.remoteStore.ApplyConfig(convertedConfig)
 	if err != nil {
diff --git a/component/pyroscope/write/write.go b/component/pyroscope/write/write.go
index 916623d76ae8..4c20797a611c 100644
--- a/component/pyroscope/write/write.go
+++ b/component/pyroscope/write/write.go
@@ -162,7 +162,7 @@ func NewFanOut(opts component.Options, config Arguments, metrics *metrics) (*fan
 		if endpoint.Headers == nil {
 			endpoint.Headers = map[string]string{}
 		}
-		endpoint.Headers["X-Agent-UID"] = uid
+		endpoint.Headers[agentseed.HeaderName] = uid
 		httpClient, err := commonconfig.NewClientFromConfig(*endpoint.HTTPClientConfig.Convert(), endpoint.Name)
 		if err != nil {
 			return nil, err
diff --git a/internal/agentseed/agentseed.go b/internal/agentseed/agentseed.go
index a4ac12d887b9..3591780ac6f8 100644
--- a/internal/agentseed/agentseed.go
+++ b/internal/agentseed/agentseed.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sync"
 	"time"
 
 	"github.com/go-kit/log"
@@ -21,74 +22,88 @@ type AgentSeed struct {
 	Version   string    `json:"version"`
 }
 
-const filename = "agent_seed.json"
+const HeaderName = "X-Agent-Id"
 
-var dataDir = ""
-var logger log.Logger
+const filename = "agent_seed.json"
 
 var savedSeed *AgentSeed
 
+var once sync.Once
+
 // Init should be called by an app entrypoint as soon as it can to configure where the unique seed will be stored.
 // dir is the directory where we will read and store agent_seed.json
 // If left empty it will default to $APPDATA or /tmp
-func Init(dir string, log log.Logger) {
-	dataDir = dir
-	logger = log
+// A unique agent seed will be generated when this method is first called, and reused for the lifetime of this agent.
+func Init(dir string, l log.Logger) {
+	if l == nil {
+		l = log.NewNopLogger()
+	}
+	once.Do(func() {
+		loadOrGenerate(dir, l)
+	})
 }
 
-// Get will return a unique uuid for this agent.
-// Seed will be saved in agent_seed.json
-// If path is not empty, that will be the "preferred" place to read and save it.
-// If it is empty, we will fall back to $APPDATA on windows or /tmp on *nix systems to read the file.
-func Get() (seed *AgentSeed) {
-	// TODO: This will just log errors and always return a valid seed.
-	// If we wanted to have it return an error for some reason, we could change this api
-	// worst case, we generate a new seed if we can't read/write files, and it is only good for the lifetime
-	// of this agent.
-	if savedSeed != nil {
-		return savedSeed
-	}
+func loadOrGenerate(dir string, l log.Logger) {
 	var err error
+	var seed *AgentSeed
 	// list of paths in preference order.
 	// we will always write to the first path
 	paths := []string{}
-	if dataDir != "" {
-		paths = append(paths, filepath.Join(dataDir, filename))
+	if dir != "" {
+		paths = append(paths, filepath.Join(dir, filename))
 	}
+	paths = append(paths, legacyPath())
 	defer func() {
 		// as a fallback, gen and save a new uid
 		if seed == nil || seed.UID == "" {
-			seed = &AgentSeed{
-				UID:       uuid.NewString(),
-				Version:   version.Version,
-				CreatedAt: time.Now(),
-			}
-			writeSeedFile(seed, paths[0])
+			seed = generateNew()
+			writeSeedFile(seed, paths[0], l)
 		}
-		// cache seed for future calls
+		// Finally save seed
 		savedSeed = seed
 	}()
-	paths = append(paths, legacyPath())
 	for i, p := range paths {
 		if fileExists(p) {
-			if seed, err = readSeedFile(p); err == nil {
+			if seed, err = readSeedFile(p, l); err == nil {
 				if i == 0 {
 					// we found it at the preferred path. Just return it
-					return seed
+					return
 				} else {
 					// it was at a backup path. write it to the preferred path.
-					writeSeedFile(seed, paths[0])
-					return seed
+					writeSeedFile(seed, paths[0], l)
+					return
 				}
 			}
 		}
 	}
+}
 
-	return seed
+func generateNew() *AgentSeed {
+	return &AgentSeed{
+		UID:       uuid.NewString(),
+		Version:   version.Version,
+		CreatedAt: time.Now(),
+	}
+}
+
+// Get will return a unique agent seed for this agent.
+// It will always return a valid seed, even if previous attempts to
+// load or save the seed file have failed
+func Get() *AgentSeed {
+	// Init should have been called before this. If not, call it now with defaults.
+	once.Do(func() {
+		loadOrGenerate("", log.NewNopLogger())
+	})
+	if savedSeed != nil {
+		return savedSeed
+	}
+	// we should never get here. But if somehow we do,
+	// still return a valid seed for this request only
+	return generateNew()
 }
 
 // readSeedFile reads the agent seed file
-func readSeedFile(path string) (*AgentSeed, error) {
+func readSeedFile(path string, logger log.Logger) (*AgentSeed, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
 		level.Error(logger).Log("msg", "Reading seed file", "err", err)
@@ -100,6 +115,7 @@ func readSeedFile(path string) (*AgentSeed, error) {
 		level.Error(logger).Log("msg", "Decoding seed file", "err", err)
 		return nil, err
 	}
+
 	if seed.UID == "" {
 		level.Error(logger).Log("msg", "Seed file has empty uid")
 	}
@@ -121,7 +137,7 @@ func fileExists(path string) bool {
 }
 
 // writeSeedFile writes the agent seed file
-func writeSeedFile(seed *AgentSeed, path string) {
+func writeSeedFile(seed *AgentSeed, path string, logger log.Logger) {
 	data, err := json.Marshal(*seed)
 	if err != nil {
 		level.Error(logger).Log("msg", "Encoding seed file", "err", err)
diff --git a/pkg/logs/logs.go b/pkg/logs/logs.go
index 4d92afebec0e..118c1a75bf58 100644
--- a/pkg/logs/logs.go
+++ b/pkg/logs/logs.go
@@ -189,7 +189,7 @@ func (i *Instance) ApplyConfig(c *InstanceConfig, g GlobalConfig, dryRun bool) e
 		if cfg.Headers == nil {
 			cfg.Headers = map[string]string{}
 		}
-		cfg.Headers["X-Agent-UID"] = uid
+		cfg.Headers[agentseed.HeaderName] = uid
 	}
 
 	clientMetrics := client.NewMetrics(i.reg)
diff --git a/pkg/metrics/instance/instance.go b/pkg/metrics/instance/instance.go
index b8ae40a4aec2..eae93936be2f 100644
--- a/pkg/metrics/instance/instance.go
+++ b/pkg/metrics/instance/instance.go
@@ -422,7 +422,7 @@ func (i *Instance) initialize(ctx context.Context, reg prometheus.Registerer, cf
 		if rw.Headers == nil {
 			rw.Headers = map[string]string{}
 		}
-		rw.Headers["X-Agent-UID"] = uid
+		rw.Headers[agentseed.HeaderName] = uid
 	}
 	err = i.remoteStore.ApplyConfig(&config.Config{
 		GlobalConfig:       cfg.global.Prometheus,

From 97426eee7e4124e0a21eb3cbb02c51437b6de79e Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Tue, 19 Dec 2023 11:46:13 -0500
Subject: [PATCH 15/18] added some tests

---
 internal/agentseed/agentseed.go      |  2 -
 internal/agentseed/agentseed_test.go | 74 ++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 2 deletions(-)
 create mode 100644 internal/agentseed/agentseed_test.go

diff --git a/internal/agentseed/agentseed.go b/internal/agentseed/agentseed.go
index 3591780ac6f8..2405c732d9c5 100644
--- a/internal/agentseed/agentseed.go
+++ b/internal/agentseed/agentseed.go
@@ -23,11 +23,9 @@ type AgentSeed struct {
 }
 
 const HeaderName = "X-Agent-Id"
-
 const filename = "agent_seed.json"
 
 var savedSeed *AgentSeed
-
 var once sync.Once
 
 // Init should be called by an app entrypoint as soon as it can to configure where the unique seed will be stored.
diff --git a/internal/agentseed/agentseed_test.go b/internal/agentseed/agentseed_test.go
new file mode 100644
index 000000000000..dcec01b8e250
--- /dev/null
+++ b/internal/agentseed/agentseed_test.go
@@ -0,0 +1,74 @@
+package agentseed
+
+import (
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/go-kit/log"
+	"github.com/stretchr/testify/require"
+)
+
+func setupFile(dir string) {
+
+}
+
+func reset() {
+	savedSeed = nil
+	once = sync.Once{}
+}
+
+func TestNoExistingFile(t *testing.T) {
+	t.Cleanup(reset)
+	dir := t.TempDir()
+	l := log.NewNopLogger()
+	Init(dir, l)
+	f := filepath.Join(dir, filename)
+	require.FileExists(t, f)
+	loaded, err := readSeedFile(f, l)
+	require.NoError(t, err)
+	seed := Get()
+	require.Equal(t, seed.UID, loaded.UID)
+}
+
+func TestExistingFile(t *testing.T) {
+	t.Cleanup(reset)
+	dir := t.TempDir()
+	l := log.NewNopLogger()
+	f := filepath.Join(dir, filename)
+	seed := generateNew()
+	writeSeedFile(seed, f, l)
+	Init(dir, l)
+	require.NotNil(t, savedSeed)
+	require.Equal(t, seed.UID, savedSeed.UID)
+	require.Equal(t, seed.UID, Get().UID)
+}
+
+func TestNoInitCalled(t *testing.T) {
+	t.Cleanup(reset)
+	l := log.NewNopLogger()
+	seed := Get()
+	require.NotNil(t, seed)
+	f := legacyPath()
+	require.FileExists(t, f)
+	loaded, err := readSeedFile(f, l)
+	require.NoError(t, err)
+	require.Equal(t, seed.UID, loaded.UID)
+}
+
+func TestLegacyExists(t *testing.T) {
+	t.Cleanup(reset)
+	dir := t.TempDir()
+	l := log.NewNopLogger()
+	f := filepath.Join(dir, filename)
+	seed := generateNew()
+	writeSeedFile(seed, legacyPath(), l)
+	Init(dir, l)
+	require.FileExists(t, f)
+	require.NotNil(t, savedSeed)
+	require.Equal(t, seed.UID, savedSeed.UID)
+	require.Equal(t, seed.UID, Get().UID)
+	loaded, err := readSeedFile(f, l)
+	require.NoError(t, err)
+	require.Equal(t, seed.UID, loaded.UID)
+}

From ed4c24ce804c498d38bdd5b3119823b1014c4e47 Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Tue, 19 Dec 2023 11:50:38 -0500
Subject: [PATCH 16/18] maybe fix tests

---
 internal/agentseed/agentseed_test.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/internal/agentseed/agentseed_test.go b/internal/agentseed/agentseed_test.go
index dcec01b8e250..e334a648ec37 100644
--- a/internal/agentseed/agentseed_test.go
+++ b/internal/agentseed/agentseed_test.go
@@ -1,6 +1,7 @@
 package agentseed
 
 import (
+	"os"
 	"path/filepath"
 	"sync"
 	"testing"
@@ -9,11 +10,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func setupFile(dir string) {
-
-}
-
 func reset() {
+	os.Remove(legacyPath())
 	savedSeed = nil
 	once = sync.Once{}
 }
@@ -22,8 +20,9 @@ func TestNoExistingFile(t *testing.T) {
 	t.Cleanup(reset)
 	dir := t.TempDir()
 	l := log.NewNopLogger()
-	Init(dir, l)
 	f := filepath.Join(dir, filename)
+	require.NoFileExists(t, f)
+	Init(dir, l)
 	require.FileExists(t, f)
 	loaded, err := readSeedFile(f, l)
 	require.NoError(t, err)

From 42fe213b9add6b4a2301f15859fedce07cf9d08c Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Tue, 19 Dec 2023 11:54:30 -0500
Subject: [PATCH 17/18] testmain?

---
 internal/agentseed/agentseed_test.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/internal/agentseed/agentseed_test.go b/internal/agentseed/agentseed_test.go
index e334a648ec37..91650c59e7d1 100644
--- a/internal/agentseed/agentseed_test.go
+++ b/internal/agentseed/agentseed_test.go
@@ -10,6 +10,12 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestMain(m *testing.M) {
+	os.Remove(legacyPath())
+	exitVal := m.Run()
+	os.Exit(exitVal)
+}
+
 func reset() {
 	os.Remove(legacyPath())
 	savedSeed = nil

From 8dc77839bf7898f8992ac95caeb0ce07ef0a373d Mon Sep 17 00:00:00 2001
From: Craig Peterson <192540+captncraig@users.noreply.github.com>
Date: Tue, 2 Jan 2024 13:32:41 -0500
Subject: [PATCH 18/18] changelog

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index afb563681613..0cafe1d8e448 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -79,6 +79,8 @@ Main (unreleased)
 
 - Bump github.com/IBM/sarama from v1.41.2 to v1.42.1
 
+- Attatch unique Agent ID header to remote-write requests. (@captncraig)
+
 v0.38.1 (2023-11-30)
 --------------------