From a73e2e3137633f88b9b8dccbe136cb0b944fdd57 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 30 Nov 2023 13:57:52 +0100 Subject: [PATCH 01/36] Implement new modules with import and declare keywords. --- .github/workflows/integration-tests.yml | 2 + cmd/internal/flowmode/cmd_run.go | 4 +- component/component_provider.go | 6 +- component/module/file/file.go | 6 +- component/module/git/git.go | 6 +- component/module/http/http.go | 6 +- component/module/module.go | 41 +- component/module/string/string.go | 4 +- component/registry.go | 2 +- converter/internal/test_common/testing.go | 2 +- .../configs/http-module/module.river | 21 + integration-tests/docker-compose.yaml | 9 +- .../tests/module-file/config.river | 38 + .../tests/module-file/logfile.river | 5 + integration-tests/tests/module-file/logs.txt | 13 + .../tests/module-file/loki.river | 17 + .../tests/module-file/loki_write.river | 13 + .../tests/module-file/module_file_test.go | 95 +++ .../tests/module-file/prom-scrape.river | 21 + .../config.river | 27 + .../scrape_prom_metrics_module_git_test.go | 78 ++ .../config.river | 24 + .../scrape_prom_metrics_module_http_test.go | 78 ++ pkg/flow/componenttest/testfailmodule.go | 6 +- pkg/flow/flow.go | 22 +- pkg/flow/flow_components.go | 36 +- pkg/flow/flow_services_test.go | 14 +- pkg/flow/flow_test.go | 4 +- pkg/flow/flow_updates_test.go | 10 +- .../internal/controller/component_node.go | 34 + pkg/flow/internal/controller/declare.go | 9 + pkg/flow/internal/controller/loader.go | 229 ++++-- pkg/flow/internal/controller/loader_test.go | 7 +- pkg/flow/internal/controller/module_info.go | 111 +++ .../internal/controller/module_references.go | 77 ++ pkg/flow/internal/controller/node_config.go | 7 + .../internal/controller/node_config_import.go | 396 ++++++++++ pkg/flow/internal/controller/node_declare.go | 55 ++ .../controller/node_declare_component.go | 361 +++++++++ ..._component.go => node_native_component.go} | 70 +- ..._test.go => node_native_component_test.go} | 4 +- .../controller/node_with_dependants.go | 18 + pkg/flow/internal/controller/queue.go | 16 +- pkg/flow/internal/controller/queue_test.go | 8 +- .../internal/import-source/import_file.go | 86 ++ pkg/flow/internal/import-source/import_git.go | 281 +++++++ .../internal/import-source/import_http.go | 87 ++ .../internal/import-source/import_source.go | 58 ++ pkg/flow/module.go | 4 +- pkg/flow/module_caching_test.go | 4 +- pkg/flow/module_declare_test.go | 227 ++++++ pkg/flow/module_fail_test.go | 2 +- pkg/flow/module_import_test.go | 741 ++++++++++++++++++ pkg/flow/module_test.go | 8 +- pkg/flow/source.go | 9 +- pkg/flow/source_test.go | 4 +- .../git/internal/vcs => pkg/util/git}/auth.go | 0 .../internal/vcs => pkg/util/git}/errors.go | 0 .../git/internal/vcs => pkg/util/git}/git.go | 0 .../internal/vcs => pkg/util/git}/git_test.go | 2 +- 60 files changed, 3357 insertions(+), 168 deletions(-) create mode 100644 integration-tests/configs/http-module/module.river create mode 100644 integration-tests/tests/module-file/config.river create mode 100644 integration-tests/tests/module-file/logfile.river create mode 100644 integration-tests/tests/module-file/logs.txt create mode 100644 integration-tests/tests/module-file/loki.river create mode 100644 integration-tests/tests/module-file/loki_write.river create mode 100644 integration-tests/tests/module-file/module_file_test.go create mode 100644 integration-tests/tests/module-file/prom-scrape.river create mode 100644 integration-tests/tests/scrape-prom-metrics-module-git/config.river create mode 100644 integration-tests/tests/scrape-prom-metrics-module-git/scrape_prom_metrics_module_git_test.go create mode 100644 integration-tests/tests/scrape-prom-metrics-module-http/config.river create mode 100644 integration-tests/tests/scrape-prom-metrics-module-http/scrape_prom_metrics_module_http_test.go create mode 100644 pkg/flow/internal/controller/component_node.go create mode 100644 pkg/flow/internal/controller/declare.go create mode 100644 pkg/flow/internal/controller/module_info.go create mode 100644 pkg/flow/internal/controller/module_references.go create mode 100644 pkg/flow/internal/controller/node_config_import.go create mode 100644 pkg/flow/internal/controller/node_declare.go create mode 100644 pkg/flow/internal/controller/node_declare_component.go rename pkg/flow/internal/controller/{node_component.go => node_native_component.go} (85%) rename pkg/flow/internal/controller/{node_component_test.go => node_native_component_test.go} (93%) create mode 100644 pkg/flow/internal/controller/node_with_dependants.go create mode 100644 pkg/flow/internal/import-source/import_file.go create mode 100644 pkg/flow/internal/import-source/import_git.go create mode 100644 pkg/flow/internal/import-source/import_http.go create mode 100644 pkg/flow/internal/import-source/import_source.go create mode 100644 pkg/flow/module_declare_test.go create mode 100644 pkg/flow/module_import_test.go rename {component/module/git/internal/vcs => pkg/util/git}/auth.go (100%) rename {component/module/git/internal/vcs => pkg/util/git}/errors.go (100%) rename {component/module/git/internal/vcs => pkg/util/git}/git.go (100%) rename {component/module/git/internal/vcs => pkg/util/git}/git_test.go (97%) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4b9f7077ed57..11e6f260ca05 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -19,5 +19,7 @@ jobs: go-version: "1.21" - name: Set OTEL Exporter Endpoint run: echo "OTEL_EXPORTER_ENDPOINT=172.17.0.1:4318" >> $GITHUB_ENV + - name: Set Module HTTP Endpoint + run: echo "MODULE_HTTP_ENDPOINT=http://172.17.0.1:8090/module.river" >> $GITHUB_ENV - name: Run tests run: make integration-test diff --git a/cmd/internal/flowmode/cmd_run.go b/cmd/internal/flowmode/cmd_run.go index c8618b928b85..c70905dfcb1f 100644 --- a/cmd/internal/flowmode/cmd_run.go +++ b/cmd/internal/flowmode/cmd_run.go @@ -274,7 +274,7 @@ func (fr *flowRun) Run(configPath string) error { if err != nil { return nil, fmt.Errorf("reading config path %q: %w", configPath, err) } - if err := f.LoadSource(flowSource, nil); err != nil { + if err := f.LoadSource(flowSource, nil, nil); err != nil { return flowSource, fmt.Errorf("error during the initial grafana/agent load: %w", err) } @@ -360,7 +360,7 @@ func getEnabledComponentsFunc(f *flow.Flow) func() map[string]interface{} { components := component.GetAllComponents(f, component.InfoOptions{}) componentNames := map[string]struct{}{} for _, c := range components { - componentNames[c.Registration.Name] = struct{}{} + componentNames[c.BlockName] = struct{}{} } return map[string]interface{}{"enabled-components": maps.Keys(componentNames)} } diff --git a/component/component_provider.go b/component/component_provider.go index 90454b5b04c3..b299f7479bf9 100644 --- a/component/component_provider.go +++ b/component/component_provider.go @@ -93,8 +93,8 @@ type Info struct { // this component depends on, or is depended on by, respectively. References, ReferencedBy []string - Registration Registration // Component registration. - Health Health // Current component health. + BlockName string // Component block name. + Health Health // Current component health. Arguments Arguments // Current arguments value of the component. Exports Exports // Current exports value of the component. @@ -157,7 +157,7 @@ func (info *Info) MarshalJSON() ([]byte, error) { } return json.Marshal(&componentDetailJSON{ - Name: info.Registration.Name, + Name: info.BlockName, Type: "block", ModuleID: info.ID.ModuleID, LocalID: info.ID.LocalID, diff --git a/component/module/file/file.go b/component/module/file/file.go index e40c5dc9ca48..fdc19804eb69 100644 --- a/component/module/file/file.go +++ b/component/module/file/file.go @@ -58,7 +58,7 @@ var ( // New creates a new module.file component. func New(o component.Options, args Arguments) (*Component, error) { - m, err := module.NewModuleComponent(o) + m, err := module.NewModuleComponentDeprecated(o) if err != nil { return nil, err } @@ -88,7 +88,7 @@ func (c *Component) newManagedLocalComponent(o component.Options) (*file.Compone if !c.inUpdate.Load() && c.isCreated.Load() { // Any errors found here are reported via component health - _ = c.mod.LoadFlowSource(c.getArgs().Arguments, c.getContent().Value) + _ = c.mod.LoadFlowSource(c.getArgs().Arguments, c.getContent().Value, nil) } } @@ -135,7 +135,7 @@ func (c *Component) Update(args component.Arguments) error { // Force a content load here and bubble up any error. This will catch problems // on initial load. - return c.mod.LoadFlowSource(newArgs.Arguments, c.getContent().Value) + return c.mod.LoadFlowSource(newArgs.Arguments, c.getContent().Value, nil) } // CurrentHealth implements component.HealthComponent. diff --git a/component/module/git/git.go b/component/module/git/git.go index dfe17ef2cb4a..a7d7c5f27dd9 100644 --- a/component/module/git/git.go +++ b/component/module/git/git.go @@ -12,8 +12,8 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/component" "github.com/grafana/agent/component/module" - "github.com/grafana/agent/component/module/git/internal/vcs" "github.com/grafana/agent/pkg/flow/logging/level" + vcs "github.com/grafana/agent/pkg/util/git" ) func init() { @@ -74,7 +74,7 @@ var ( // New creates a new module.git component. func New(o component.Options, args Arguments) (*Component, error) { - m, err := module.NewModuleComponent(o) + m, err := module.NewModuleComponentDeprecated(o) if err != nil { return nil, err } @@ -239,7 +239,7 @@ func (c *Component) pollFile(ctx context.Context, args Arguments) error { return err } - return c.mod.LoadFlowSource(args.Arguments, string(bb)) + return c.mod.LoadFlowSource(args.Arguments, string(bb), nil) } // CurrentHealth implements component.HealthComponent. diff --git a/component/module/http/http.go b/component/module/http/http.go index bc1be2158fdb..39855b9b8595 100644 --- a/component/module/http/http.go +++ b/component/module/http/http.go @@ -57,7 +57,7 @@ var ( // New creates a new module.http component. func New(o component.Options, args Arguments) (*Component, error) { - m, err := module.NewModuleComponent(o) + m, err := module.NewModuleComponentDeprecated(o) if err != nil { return nil, err } @@ -87,7 +87,7 @@ func (c *Component) newManagedLocalComponent(o component.Options) (*remote_http. if !c.inUpdate.Load() && c.isCreated.Load() { // Any errors found here are reported via component health - _ = c.mod.LoadFlowSource(c.getArgs().Arguments, c.getContent().Value) + _ = c.mod.LoadFlowSource(c.getArgs().Arguments, c.getContent().Value, nil) } } @@ -134,7 +134,7 @@ func (c *Component) Update(args component.Arguments) error { // Force a content load here and bubble up any error. This will catch problems // on initial load. - return c.mod.LoadFlowSource(newArgs.Arguments, c.getContent().Value) + return c.mod.LoadFlowSource(newArgs.Arguments, c.getContent().Value, nil) } // CurrentHealth implements component.HealthComponent. diff --git a/component/module/module.go b/component/module/module.go index 7995fdbca5c9..40e2d19d138c 100644 --- a/component/module/module.go +++ b/component/module/module.go @@ -16,13 +16,15 @@ type ModuleComponent struct { opts component.Options mod component.Module - mut sync.RWMutex - health component.Health - latestContent string - latestArgs map[string]any + mut sync.RWMutex + health component.Health + latestContent string + latestArgs map[string]any + latestParentModuleDefinitions map[string]string } // Exports holds values which are exported from the run module. +// This export type is deprecated. type Exports struct { // Exports exported from the running module. Exports map[string]any `river:"exports,block"` @@ -30,6 +32,18 @@ type Exports struct { // NewModuleComponent initializes a new ModuleComponent. func NewModuleComponent(o component.Options) (*ModuleComponent, error) { + c := &ModuleComponent{ + opts: o, + } + var err error + c.mod, err = o.ModuleController.NewModule("", func(exports map[string]any) { + c.opts.OnStateChange(exports) + }) + return c, err +} + +// TODO: Remove when getting rid of old modules +func NewModuleComponentDeprecated(o component.Options) (*ModuleComponent, error) { c := &ModuleComponent{ opts: o, } @@ -43,12 +57,12 @@ func NewModuleComponent(o component.Options) (*ModuleComponent, error) { // LoadFlowSource loads the flow controller with the current component source. // It will set the component health in addition to return the error so that the consumer can rely on either or both. // If the content is the same as the last time it was successfully loaded, it will not be reloaded. -func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue string) error { - if reflect.DeepEqual(args, c.getLatestArgs()) && contentValue == c.getLatestContent() { +func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue string, parentModuleDefinitions map[string]string) error { + if reflect.DeepEqual(args, c.getLatestArgs()) && contentValue == c.getLatestContent() && reflect.DeepEqual(args, c.getLatestParentModuleDefinitions()) { return nil } - err := c.mod.LoadConfig([]byte(contentValue), args) + err := c.mod.LoadConfig([]byte(contentValue), args, parentModuleDefinitions) if err != nil { c.setHealth(component.Health{ Health: component.HealthTypeUnhealthy, @@ -61,6 +75,7 @@ func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue strin c.setLatestArgs(args) c.setLatestContent(contentValue) + c.setLatestParentModuleDefinitions(parentModuleDefinitions) c.setHealth(component.Health{ Health: component.HealthTypeHealthy, Message: "module content loaded", @@ -104,6 +119,18 @@ func (c *ModuleComponent) getLatestContent() string { return c.latestContent } +func (c *ModuleComponent) setLatestParentModuleDefinitions(parentModuleDefinitions map[string]string) { + c.mut.Lock() + defer c.mut.Unlock() + c.latestParentModuleDefinitions = parentModuleDefinitions +} + +func (c *ModuleComponent) getLatestParentModuleDefinitions() map[string]string { + c.mut.RLock() + defer c.mut.RUnlock() + return c.latestParentModuleDefinitions +} + func (c *ModuleComponent) setLatestArgs(args map[string]any) { c.mut.Lock() defer c.mut.Unlock() diff --git a/component/module/string/string.go b/component/module/string/string.go index bd3e6193f441..65e2cdf6190d 100644 --- a/component/module/string/string.go +++ b/component/module/string/string.go @@ -42,7 +42,7 @@ var ( // New creates a new module.string component. func New(o component.Options, args Arguments) (*Component, error) { - m, err := module.NewModuleComponent(o) + m, err := module.NewModuleComponentDeprecated(o) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func (c *Component) Run(ctx context.Context) error { func (c *Component) Update(args component.Arguments) error { newArgs := args.(Arguments) - return c.mod.LoadFlowSource(newArgs.Arguments, newArgs.Content.Value) + return c.mod.LoadFlowSource(newArgs.Arguments, newArgs.Content.Value, nil) } // CurrentHealth implements component.HealthComponent. diff --git a/component/registry.go b/component/registry.go index 11cc593b0ddd..81ead2f9e0d7 100644 --- a/component/registry.go +++ b/component/registry.go @@ -44,7 +44,7 @@ type Module interface { // LoadConfig parses River config and loads it into the Module. // LoadConfig can be called multiple times, and called prior to // [Module.Run]. - LoadConfig(config []byte, args map[string]any) error + LoadConfig(config []byte, args map[string]any, moduleDefinitions map[string]string) error // Run starts the Module. No components within the Module // will be run until Run is called. diff --git a/converter/internal/test_common/testing.go b/converter/internal/test_common/testing.go index 03855fc2ca31..3d6a04c03880 100644 --- a/converter/internal/test_common/testing.go +++ b/converter/internal/test_common/testing.go @@ -198,7 +198,7 @@ func attemptLoadingFlowConfig(t *testing.T, river []byte) { labelstore.New(nil, prometheus.DefaultRegisterer), }, }) - err = f.LoadSource(cfg, nil) + err = f.LoadSource(cfg, nil, nil) // Many components will fail to build as e.g. the cert files are missing, so we ignore these errors. // This is not ideal, but we still validate for other potential issues. diff --git a/integration-tests/configs/http-module/module.river b/integration-tests/configs/http-module/module.river new file mode 100644 index 000000000000..808b79a86b4b --- /dev/null +++ b/integration-tests/configs/http-module/module.river @@ -0,0 +1,21 @@ +declare "myModule" { + argument "scrape_endpoint" {} + + argument "forward_to" {} + + argument "scrape_interval" { + optional = true + default = "1s" + } + + prometheus.scrape "scrape_prom_metrics_module_file" { + targets = [ + {"__address__" = argument.scrape_endpoint.value}, + ] + forward_to = argument.forward_to.value + scrape_classic_histograms = true + enable_protobuf_negotiation = true + scrape_interval = argument.scrape_interval.value + scrape_timeout = "500ms" + } +} \ No newline at end of file diff --git a/integration-tests/docker-compose.yaml b/integration-tests/docker-compose.yaml index a94a05db21d9..d5503de64b62 100644 --- a/integration-tests/docker-compose.yaml +++ b/integration-tests/docker-compose.yaml @@ -29,4 +29,11 @@ services: dockerfile: ./integration-tests/configs/prom-gen/Dockerfile context: .. ports: - - "9001:9001" \ No newline at end of file + - "9001:9001" + + http-module: + image: nginx:alpine + ports: + - "8090:80" + volumes: + - ./configs/http-module/module.river:/usr/share/nginx/html/module.river:ro \ No newline at end of file diff --git a/integration-tests/tests/module-file/config.river b/integration-tests/tests/module-file/config.river new file mode 100644 index 000000000000..a8e776d5cd24 --- /dev/null +++ b/integration-tests/tests/module-file/config.river @@ -0,0 +1,38 @@ +import.file "import_loki" { + filename = "loki.river" +} + +import_loki.loki "loki" {} + +import.file "import_prom_scrape" { + filename = "prom-scrape.river" +} + +declare "target" { + export "output" { + value = "localhost:9001" + } +} + +target "promTarget" {} + +import_prom_scrape.scrape "promScraper" { + scrape_endpoint = target.promTarget.output + forward_to = [prometheus.remote_write.module_file.receiver] +} + +prometheus.remote_write "module_file" { + endpoint { + url = "http://localhost:9009/api/v1/push" + send_native_histograms = true + metadata_config { + send_interval = "1s" + } + queue_config { + max_samples_per_send = 100 + } + } + external_labels = { + test_name = "module_file", + } +} \ No newline at end of file diff --git a/integration-tests/tests/module-file/logfile.river b/integration-tests/tests/module-file/logfile.river new file mode 100644 index 000000000000..a9b2b9762b3a --- /dev/null +++ b/integration-tests/tests/module-file/logfile.river @@ -0,0 +1,5 @@ +declare "getLogFile" { + export "output" { + value = [{__path__ = "logs.txt"}] + } +} \ No newline at end of file diff --git a/integration-tests/tests/module-file/logs.txt b/integration-tests/tests/module-file/logs.txt new file mode 100644 index 000000000000..ed6c24c81170 --- /dev/null +++ b/integration-tests/tests/module-file/logs.txt @@ -0,0 +1,13 @@ +[2023-10-02 14:25:43] INFO: Starting the web application... +[2023-10-02 14:25:45] DEBUG: Database connection established. +[2023-10-02 14:26:01] INFO: User 'john_doe' logged in. +[2023-10-02 14:26:05] WARNING: User 'john_doe' attempted to access restricted area. +[2023-10-02 14:26:10] ERROR: Failed to retrieve data for item ID: 1234. +[2023-10-02 14:26:15] INFO: User 'john_doe' logged out. +[2023-10-02 14:27:00] INFO: User 'admin' logged in. +[2023-10-02 14:27:05] INFO: Data backup started. +[2023-10-02 14:30:00] INFO: Data backup completed successfully. +[2023-10-02 14:31:23] ERROR: Database connection lost. Retrying in 5 seconds... +[2023-10-02 14:31:28] INFO: Database reconnected. +[2023-10-02 14:32:00] INFO: User 'admin' logged out. +[2023-10-02 14:32:05] INFO: Shutting down the web application... diff --git a/integration-tests/tests/module-file/loki.river b/integration-tests/tests/module-file/loki.river new file mode 100644 index 000000000000..24f5af09e874 --- /dev/null +++ b/integration-tests/tests/module-file/loki.river @@ -0,0 +1,17 @@ +import.file "logTarget" { + filename = "logfile.river" +} + +import.file "write" { + filename = "loki_write.river" +} + +declare "loki" { + logTarget.getLogFile "logFile" {} + loki.source.file "test" { + targets = logTarget.getLogFile.logFile.output + forward_to = [write.loki_write.default.receiver] + } + write.loki_write "default" {} +} + diff --git a/integration-tests/tests/module-file/loki_write.river b/integration-tests/tests/module-file/loki_write.river new file mode 100644 index 000000000000..03c9ed44b66b --- /dev/null +++ b/integration-tests/tests/module-file/loki_write.river @@ -0,0 +1,13 @@ +declare "loki_write" { + loki.write "test" { + endpoint { + url = "http://localhost:3100/loki/api/v1/push" + } + external_labels = { + test_name = "module_file", + } + } + export "receiver" { + value = loki.write.test.receiver + } +} \ No newline at end of file diff --git a/integration-tests/tests/module-file/module_file_test.go b/integration-tests/tests/module-file/module_file_test.go new file mode 100644 index 000000000000..bb7db88feb44 --- /dev/null +++ b/integration-tests/tests/module-file/module_file_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "fmt" + "strconv" + "testing" + + "github.com/grafana/agent/integration-tests/common" + "github.com/stretchr/testify/assert" +) + +const promURL = "http://localhost:9009/prometheus/api/v1/query?query=" +const lokiUrl = "http://localhost:3100/loki/api/v1/query?query={test_name=%22module_file%22}" + +func metricQuery(metricName string) string { + return fmt.Sprintf("%s%s{test_name='module_file'}", promURL, metricName) +} + +func TestScrapePromMetricsModuleFile(t *testing.T) { + metrics := []string{ + // TODO: better differentiate these metric types? + "golang_counter", + "golang_gauge", + "golang_histogram_bucket", + "golang_summary", + "golang_native_histogram", + } + + for _, metric := range metrics { + metric := metric + t.Run(metric, func(t *testing.T) { + t.Parallel() + if metric == "golang_native_histogram" { + assertHistogramData(t, metricQuery(metric), metric) + } else { + assertMetricData(t, metricQuery(metric), metric) + } + }) + } +} + +func assertHistogramData(t *testing.T, query string, expectedMetric string) { + var metricResponse common.MetricResponse + assert.EventuallyWithT(t, func(c *assert.CollectT) { + err := common.FetchDataFromURL(query, &metricResponse) + assert.NoError(c, err) + if assert.NotEmpty(c, metricResponse.Data.Result) { + assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) + assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "module_file") + if assert.NotNil(c, metricResponse.Data.Result[0].Histogram) { + histogram := metricResponse.Data.Result[0].Histogram + if assert.NotEmpty(c, histogram.Data.Count) { + count, _ := strconv.Atoi(histogram.Data.Count) + assert.Greater(c, count, 10, "Count should be at some point greater than 10.") + } + if assert.NotEmpty(c, histogram.Data.Sum) { + sum, _ := strconv.ParseFloat(histogram.Data.Sum, 64) + assert.Greater(c, sum, 10., "Sum should be at some point greater than 10.") + } + assert.NotEmpty(c, histogram.Data.Buckets) + assert.Nil(c, metricResponse.Data.Result[0].Value) + } + } + }, common.DefaultTimeout, common.DefaultRetryInterval, "Histogram data did not satisfy the conditions within the time limit") +} + +func TestReadLogFile(t *testing.T) { + var logResponse common.LogResponse + assert.EventuallyWithT(t, func(c *assert.CollectT) { + err := common.FetchDataFromURL(lokiUrl, &logResponse) + assert.NoError(c, err) + if assert.NotEmpty(c, logResponse.Data.Result) { + assert.Equal(c, logResponse.Data.Result[0].Stream["filename"], "logs.txt") + logs := make([]string, len(logResponse.Data.Result[0].Values)) + for i, valuePair := range logResponse.Data.Result[0].Values { + logs[i] = valuePair[1] + } + assert.Contains(c, logs, "[2023-10-02 14:25:43] INFO: Starting the web application...") + } + }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") +} + +func assertMetricData(t *testing.T, query, expectedMetric string) { + var metricResponse common.MetricResponse + assert.EventuallyWithT(t, func(c *assert.CollectT) { + err := common.FetchDataFromURL(query, &metricResponse) + assert.NoError(c, err) + if assert.NotEmpty(c, metricResponse.Data.Result) { + assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) + assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "module_file") + assert.NotEmpty(c, metricResponse.Data.Result[0].Value.Value) + assert.Nil(c, metricResponse.Data.Result[0].Histogram) + } + }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") +} diff --git a/integration-tests/tests/module-file/prom-scrape.river b/integration-tests/tests/module-file/prom-scrape.river new file mode 100644 index 000000000000..f6e0b31aadcd --- /dev/null +++ b/integration-tests/tests/module-file/prom-scrape.river @@ -0,0 +1,21 @@ +declare "scrape" { + argument "scrape_endpoint" {} + + argument "forward_to" {} + + argument "scrape_interval" { + optional = true + default = "1s" + } + + prometheus.scrape "module_file" { + targets = [ + {"__address__" = argument.scrape_endpoint.value}, + ] + forward_to = argument.forward_to.value + scrape_classic_histograms = true + enable_protobuf_negotiation = true + scrape_interval = argument.scrape_interval.value + scrape_timeout = "500ms" + } +} diff --git a/integration-tests/tests/scrape-prom-metrics-module-git/config.river b/integration-tests/tests/scrape-prom-metrics-module-git/config.river new file mode 100644 index 000000000000..68dffaef6ffd --- /dev/null +++ b/integration-tests/tests/scrape-prom-metrics-module-git/config.river @@ -0,0 +1,27 @@ +import.git "scrape_module" { + // TODO: change this to use either a docker container hosting a git repo just for the integration tests (not easy) + // or put the module.river file in the agent-modules repo (easy) + repository = "https://github.com/wildum/module.git" + path = "module.river" +} + +scrape_module.myModule "scrape_prom_metrics_module_git" { + scrape_endpoint = "localhost:9001" + forward_to = [prometheus.remote_write.scrape_prom_metrics_module_git.receiver] +} + +prometheus.remote_write "scrape_prom_metrics_module_git" { + endpoint { + url = "http://localhost:9009/api/v1/push" + send_native_histograms = true + metadata_config { + send_interval = "1s" + } + queue_config { + max_samples_per_send = 100 + } + } + external_labels = { + test_name = "scrape_prom_metrics_module_git", + } +} diff --git a/integration-tests/tests/scrape-prom-metrics-module-git/scrape_prom_metrics_module_git_test.go b/integration-tests/tests/scrape-prom-metrics-module-git/scrape_prom_metrics_module_git_test.go new file mode 100644 index 000000000000..1c095f2dd6bf --- /dev/null +++ b/integration-tests/tests/scrape-prom-metrics-module-git/scrape_prom_metrics_module_git_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "strconv" + "testing" + + "github.com/grafana/agent/integration-tests/common" + "github.com/stretchr/testify/assert" +) + +const promURL = "http://localhost:9009/prometheus/api/v1/query?query=" + +func metricQuery(metricName string) string { + return fmt.Sprintf("%s%s{test_name='scrape_prom_metrics_module_git'}", promURL, metricName) +} + +func TestScrapePromMetricsModuleGit(t *testing.T) { + metrics := []string{ + // TODO: better differentiate these metric types? + "golang_counter", + "golang_gauge", + "golang_histogram_bucket", + "golang_summary", + "golang_native_histogram", + } + + for _, metric := range metrics { + metric := metric + t.Run(metric, func(t *testing.T) { + t.Parallel() + if metric == "golang_native_histogram" { + assertHistogramData(t, metricQuery(metric), metric) + } else { + assertMetricData(t, metricQuery(metric), metric) + } + }) + } +} + +func assertHistogramData(t *testing.T, query string, expectedMetric string) { + var metricResponse common.MetricResponse + assert.EventuallyWithT(t, func(c *assert.CollectT) { + err := common.FetchDataFromURL(query, &metricResponse) + assert.NoError(c, err) + if assert.NotEmpty(c, metricResponse.Data.Result) { + assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) + assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics_module_git") + if assert.NotNil(c, metricResponse.Data.Result[0].Histogram) { + histogram := metricResponse.Data.Result[0].Histogram + if assert.NotEmpty(c, histogram.Data.Count) { + count, _ := strconv.Atoi(histogram.Data.Count) + assert.Greater(c, count, 10, "Count should be at some point greater than 10.") + } + if assert.NotEmpty(c, histogram.Data.Sum) { + sum, _ := strconv.ParseFloat(histogram.Data.Sum, 64) + assert.Greater(c, sum, 10., "Sum should be at some point greater than 10.") + } + assert.NotEmpty(c, histogram.Data.Buckets) + assert.Nil(c, metricResponse.Data.Result[0].Value) + } + } + }, common.DefaultTimeout, common.DefaultRetryInterval, "Histogram data did not satisfy the conditions within the time limit") +} + +func assertMetricData(t *testing.T, query, expectedMetric string) { + var metricResponse common.MetricResponse + assert.EventuallyWithT(t, func(c *assert.CollectT) { + err := common.FetchDataFromURL(query, &metricResponse) + assert.NoError(c, err) + if assert.NotEmpty(c, metricResponse.Data.Result) { + assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) + assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics_module_git") + assert.NotEmpty(c, metricResponse.Data.Result[0].Value.Value) + assert.Nil(c, metricResponse.Data.Result[0].Histogram) + } + }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") +} diff --git a/integration-tests/tests/scrape-prom-metrics-module-http/config.river b/integration-tests/tests/scrape-prom-metrics-module-http/config.river new file mode 100644 index 000000000000..dcb93d4d86a3 --- /dev/null +++ b/integration-tests/tests/scrape-prom-metrics-module-http/config.river @@ -0,0 +1,24 @@ +import.http "scrape_module" { + url = coalesce(env("MODULE_HTTP_ENDPOINT"), "http://localhost:8090/module.river") +} + +scrape_module.myModule "scrape_prom_metrics_module_http" { + scrape_endpoint = "localhost:9001" + forward_to = [prometheus.remote_write.scrape_prom_metrics_module_http.receiver] +} + +prometheus.remote_write "scrape_prom_metrics_module_http" { + endpoint { + url = "http://localhost:9009/api/v1/push" + send_native_histograms = true + metadata_config { + send_interval = "1s" + } + queue_config { + max_samples_per_send = 100 + } + } + external_labels = { + test_name = "scrape_prom_metrics_module_http", + } +} diff --git a/integration-tests/tests/scrape-prom-metrics-module-http/scrape_prom_metrics_module_http_test.go b/integration-tests/tests/scrape-prom-metrics-module-http/scrape_prom_metrics_module_http_test.go new file mode 100644 index 000000000000..b847de78d444 --- /dev/null +++ b/integration-tests/tests/scrape-prom-metrics-module-http/scrape_prom_metrics_module_http_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "strconv" + "testing" + + "github.com/grafana/agent/integration-tests/common" + "github.com/stretchr/testify/assert" +) + +const promURL = "http://localhost:9009/prometheus/api/v1/query?query=" + +func metricQuery(metricName string) string { + return fmt.Sprintf("%s%s{test_name='scrape_prom_metrics_module_http'}", promURL, metricName) +} + +func TestScrapePromMetricsModuleHTTP(t *testing.T) { + metrics := []string{ + // TODO: better differentiate these metric types? + "golang_counter", + "golang_gauge", + "golang_histogram_bucket", + "golang_summary", + "golang_native_histogram", + } + + for _, metric := range metrics { + metric := metric + t.Run(metric, func(t *testing.T) { + t.Parallel() + if metric == "golang_native_histogram" { + assertHistogramData(t, metricQuery(metric), metric) + } else { + assertMetricData(t, metricQuery(metric), metric) + } + }) + } +} + +func assertHistogramData(t *testing.T, query string, expectedMetric string) { + var metricResponse common.MetricResponse + assert.EventuallyWithT(t, func(c *assert.CollectT) { + err := common.FetchDataFromURL(query, &metricResponse) + assert.NoError(c, err) + if assert.NotEmpty(c, metricResponse.Data.Result) { + assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) + assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics_module_http") + if assert.NotNil(c, metricResponse.Data.Result[0].Histogram) { + histogram := metricResponse.Data.Result[0].Histogram + if assert.NotEmpty(c, histogram.Data.Count) { + count, _ := strconv.Atoi(histogram.Data.Count) + assert.Greater(c, count, 10, "Count should be at some point greater than 10.") + } + if assert.NotEmpty(c, histogram.Data.Sum) { + sum, _ := strconv.ParseFloat(histogram.Data.Sum, 64) + assert.Greater(c, sum, 10., "Sum should be at some point greater than 10.") + } + assert.NotEmpty(c, histogram.Data.Buckets) + assert.Nil(c, metricResponse.Data.Result[0].Value) + } + } + }, common.DefaultTimeout, common.DefaultRetryInterval, "Histogram data did not satisfy the conditions within the time limit") +} + +func assertMetricData(t *testing.T, query, expectedMetric string) { + var metricResponse common.MetricResponse + assert.EventuallyWithT(t, func(c *assert.CollectT) { + err := common.FetchDataFromURL(query, &metricResponse) + assert.NoError(c, err) + if assert.NotEmpty(c, metricResponse.Data.Result) { + assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) + assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics_module_http") + assert.NotEmpty(c, metricResponse.Data.Result[0].Value.Value) + assert.Nil(c, metricResponse.Data.Result[0].Histogram) + } + }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") +} diff --git a/pkg/flow/componenttest/testfailmodule.go b/pkg/flow/componenttest/testfailmodule.go index 011659f95564..3481965ef9f5 100644 --- a/pkg/flow/componenttest/testfailmodule.go +++ b/pkg/flow/componenttest/testfailmodule.go @@ -15,14 +15,14 @@ func init() { Exports: mod.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { - m, err := mod.NewModuleComponent(opts) + m, err := mod.NewModuleComponentDeprecated(opts) if err != nil { return nil, err } if args.(TestFailArguments).Fail { return nil, fmt.Errorf("module told to fail") } - err = m.LoadFlowSource(nil, args.(TestFailArguments).Content) + err = m.LoadFlowSource(nil, args.(TestFailArguments).Content, nil) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func (t *TestFailModule) Run(ctx context.Context) error { func (t *TestFailModule) UpdateContent(content string) error { t.content = content - err := t.mc.LoadFlowSource(nil, t.content) + err := t.mc.LoadFlowSource(nil, t.content, nil) return err } diff --git a/pkg/flow/flow.go b/pkg/flow/flow.go index 76d62bdb9cb2..8d0d776fcd6f 100644 --- a/pkg/flow/flow.go +++ b/pkg/flow/flow.go @@ -185,7 +185,7 @@ func newController(o controllerOptions) *Flow { Logger: log, TraceProvider: tracer, DataPath: o.DataPath, - OnComponentUpdate: func(cn *controller.ComponentNode) { + OnComponentUpdate: func(cn controller.NodeWithDependants) { // Changed components should be queued for reevaluation. f.updateQueue.Enqueue(cn) }, @@ -245,15 +245,25 @@ func (f *Flow) Run(ctx context.Context) { level.Info(f.log).Log("msg", "scheduling loaded components and services") var ( - components = f.loader.Components() - services = f.loader.Services() + components = f.loader.Components() + services = f.loader.Services() + imports = f.loader.Imports() + declareComponents = f.loader.DeclareComponent() - runnables = make([]controller.RunnableNode, 0, len(components)+len(services)) + runnables = make([]controller.RunnableNode, 0, len(components)+len(services)+len(imports)+len(declareComponents)) ) for _, c := range components { runnables = append(runnables, c) } + for _, i := range imports { + runnables = append(runnables, i) + } + + for _, d := range declareComponents { + runnables = append(runnables, d) + } + // Only the root controller should run services, since modules share the // same service instance as the root. if !f.opts.IsModule { @@ -276,11 +286,11 @@ func (f *Flow) Run(ctx context.Context) { // // The controller will only start running components after Load is called once // without any configuration errors. -func (f *Flow) LoadSource(source *Source, args map[string]any) error { +func (f *Flow) LoadSource(source *Source, args map[string]any, parentModuleDefinitions map[string]string) error { f.loadMut.Lock() defer f.loadMut.Unlock() - diags := f.loader.Apply(args, source.components, source.configBlocks) + diags := f.loader.Apply(args, source.components, source.configBlocks, source.declares, parentModuleDefinitions) if !f.loadedOnce.Load() && diags.HasErrors() { // The first call to Load should not run any components if there were // errors in the configuration file. diff --git a/pkg/flow/flow_components.go b/pkg/flow/flow_components.go index 0899971339b7..e4f4c7734b0a 100644 --- a/pkg/flow/flow_components.go +++ b/pkg/flow/flow_components.go @@ -29,9 +29,9 @@ func (f *Flow) GetComponent(id component.ID, opts component.InfoOptions) (*compo return nil, component.ErrComponentNotFound } - cn, ok := node.(*controller.ComponentNode) + cn, ok := node.(controller.ComponentNode) if !ok { - return nil, fmt.Errorf("%q is not a component", id) + return nil, fmt.Errorf("%q is not a ComponentNode", id) } return f.getComponentDetail(cn, graph, opts), nil @@ -52,18 +52,30 @@ func (f *Flow) ListComponents(moduleID string, opts component.InfoOptions) ([]*c } var ( - components = f.loader.Components() - graph = f.loader.OriginalGraph() + components = f.loader.Components() + imports = f.loader.Imports() + declareComponents = f.loader.DeclareComponent() + graph = f.loader.OriginalGraph() ) - detail := make([]*component.Info, len(components)) - for i, component := range components { - detail[i] = f.getComponentDetail(component, graph, opts) + detail := make([]*component.Info, len(components)+len(imports)+len(declareComponents)) + idx := 0 + for _, component := range components { + detail[idx] = f.getComponentDetail(component, graph, opts) + idx++ + } + for _, importNode := range imports { + detail[idx] = f.getComponentDetail(importNode, graph, opts) + idx++ + } + for _, declareComponent := range declareComponents { + detail[idx] = f.getComponentDetail(declareComponent, graph, opts) + idx++ } return detail, nil } -func (f *Flow) getComponentDetail(cn *controller.ComponentNode, graph *dag.Graph, opts component.InfoOptions) *component.Info { +func (f *Flow) getComponentDetail(cn controller.ComponentNode, graph *dag.Graph, opts component.InfoOptions) *component.Info { var references, referencedBy []string // Skip over any edge which isn't between two component nodes. This is a @@ -75,12 +87,12 @@ func (f *Flow) getComponentDetail(cn *controller.ComponentNode, graph *dag.Graph // // TODO(rfratto): add support for config block nodes in the API and UI. for _, dep := range graph.Dependencies(cn) { - if _, ok := dep.(*controller.ComponentNode); ok { + if _, ok := dep.(controller.ComponentNode); ok { references = append(references, dep.NodeID()) } } for _, dep := range graph.Dependants(cn) { - if _, ok := dep.(*controller.ComponentNode); ok { + if _, ok := dep.(controller.ComponentNode); ok { referencedBy = append(referencedBy, dep.NodeID()) } } @@ -119,8 +131,8 @@ func (f *Flow) getComponentDetail(cn *controller.ComponentNode, graph *dag.Graph References: references, ReferencedBy: referencedBy, - Registration: cn.Registration(), - Health: health, + BlockName: cn.BlockName(), + Health: health, Arguments: arguments, Exports: exports, diff --git a/pkg/flow/flow_services_test.go b/pkg/flow/flow_services_test.go index a4bf2b4cb848..077a5c2cd4e7 100644 --- a/pkg/flow/flow_services_test.go +++ b/pkg/flow/flow_services_test.go @@ -37,7 +37,7 @@ func TestServices(t *testing.T) { opts.Services = append(opts.Services, svc) ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, nil)) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -88,7 +88,7 @@ func TestServices_Configurable(t *testing.T) { ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, nil)) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -134,7 +134,7 @@ func TestServices_Configurable_Optional(t *testing.T) { ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, nil)) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -168,7 +168,7 @@ func TestFlow_GetServiceConsumers(t *testing.T) { ctrl := New(opts) defer cleanUpController(ctrl) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, nil)) expectConsumers := []service.Consumer{{ Type: service.ConsumerTypeService, @@ -246,7 +246,7 @@ func TestComponents_Using_Services(t *testing.T) { ComponentRegistry: registry, ModuleRegistry: newModuleRegistry(), }) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, nil)) go ctrl.Run(ctx) require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built") @@ -276,7 +276,7 @@ func TestComponents_Using_Services_In_Modules(t *testing.T) { mod, err := opts.ModuleController.NewModule("", nil) require.NoError(t, err, "Failed to create module") - err = mod.LoadConfig([]byte(`service_consumer "example" {}`), nil) + err = mod.LoadConfig([]byte(`service_consumer "example" {}`), nil, nil) require.NoError(t, err, "Failed to load module config") return &testcomponents.Fake{ @@ -321,7 +321,7 @@ func TestComponents_Using_Services_In_Modules(t *testing.T) { ComponentRegistry: registry, ModuleRegistry: newModuleRegistry(), }) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, nil)) go ctrl.Run(ctx) require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built") diff --git a/pkg/flow/flow_test.go b/pkg/flow/flow_test.go index 590f97a424f1..17a2b60d41d9 100644 --- a/pkg/flow/flow_test.go +++ b/pkg/flow/flow_test.go @@ -42,7 +42,7 @@ func TestController_LoadSource_Evaluation(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, nil) require.NoError(t, err) require.Len(t, ctrl.loader.Components(), 4) @@ -59,7 +59,7 @@ func getFields(t *testing.T, g *dag.Graph, nodeID string) (component.Arguments, n := g.GetByID(nodeID) require.NotNil(t, n, "couldn't find node %q in graph", nodeID) - uc := n.(*controller.ComponentNode) + uc := n.(*controller.NativeComponentNode) return uc.Arguments(), uc.Exports() } diff --git a/pkg/flow/flow_updates_test.go b/pkg/flow/flow_updates_test.go index c2349928f06a..cf77237b7c76 100644 --- a/pkg/flow/flow_updates_test.go +++ b/pkg/flow/flow_updates_test.go @@ -42,7 +42,7 @@ func TestController_Updates(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -122,7 +122,7 @@ func TestController_Updates_WithQueueFull(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -195,7 +195,7 @@ func TestController_Updates_WithLag(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -269,7 +269,7 @@ func TestController_Updates_WithOtherLaggingPipeline(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -338,7 +338,7 @@ func TestController_Updates_WithLaggingComponent(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/flow/internal/controller/component_node.go b/pkg/flow/internal/controller/component_node.go new file mode 100644 index 000000000000..269e313b99fa --- /dev/null +++ b/pkg/flow/internal/controller/component_node.go @@ -0,0 +1,34 @@ +package controller + +import ( + "github.com/grafana/agent/component" + "github.com/grafana/agent/pkg/flow/internal/dag" +) + +type ComponentNode interface { + dag.Node + + // CurrentHealth returns the current health of the node. + CurrentHealth() component.Health + + // DebugInfo returns debugging information from the managed component (if any). + DebugInfo() interface{} + + // Arguments returns the current arguments of the managed component. + Arguments() component.Arguments + + // Exports returns the current set of exports from the managed component. + Exports() component.Exports + + // Component returns the instance of the managed component. + Component() component.Component + + // ModuleIDs returns the current list of modules that this component is managing. + ModuleIDs() []string + + // Label returns the label for the block or "" if none was specified. + Label() string + + // BlockName returns the name of the block. + BlockName() string +} diff --git a/pkg/flow/internal/controller/declare.go b/pkg/flow/internal/controller/declare.go new file mode 100644 index 000000000000..e97554396c32 --- /dev/null +++ b/pkg/flow/internal/controller/declare.go @@ -0,0 +1,9 @@ +package controller + +import "github.com/grafana/river/ast" + +// Should this be defined somewhere else? +type Declare struct { + Block *ast.BlockStmt + Content string +} diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 10a6f37965ab..2c4ddb098b34 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "sync" "time" @@ -37,11 +38,17 @@ type Loader struct { // also prevents log spamming with errors. backoffConfig backoff.Config - mut sync.RWMutex - graph *dag.Graph - originalGraph *dag.Graph - componentNodes []*ComponentNode - serviceNodes []*ServiceNode + mut sync.RWMutex + graph *dag.Graph + originalGraph *dag.Graph + componentNodes []*NativeComponentNode + serviceNodes []*ServiceNode + declareComponentNodes []*DeclareComponentNode + importNodes map[string]*ImportConfigNode + declareNodes map[string]*DeclareNode + parentModuleDefinitions map[string]string + moduleReferences map[string][]ModuleReference + cache *valueCache blocks []*ast.BlockStmt // Most recently loaded blocks, used for writing cm *controllerMetrics @@ -75,13 +82,16 @@ func NewLoader(opts LoaderOptions) *Loader { } l := &Loader{ - log: log.With(globals.Logger, "controller_id", globals.ControllerID), - tracer: tracing.WrapTracerForLoader(globals.TraceProvider, globals.ControllerID), - globals: globals, - services: services, - host: host, - componentReg: reg, - workerPool: opts.WorkerPool, + log: log.With(globals.Logger, "controller_id", globals.ControllerID), + tracer: tracing.WrapTracerForLoader(globals.TraceProvider, globals.ControllerID), + globals: globals, + services: services, + host: host, + componentReg: reg, + workerPool: opts.WorkerPool, + importNodes: map[string]*ImportConfigNode{}, + declareNodes: map[string]*DeclareNode{}, + moduleReferences: map[string][]ModuleReference{}, // This is a reasonable default which should work for most cases. If a component is completely stuck, we would // retry and log an error every 10 seconds, at most. @@ -117,7 +127,7 @@ func NewLoader(opts LoaderOptions) *Loader { // The provided parentContext can be used to provide global variables and // functions to components. A child context will be constructed from the parent // to expose values of other components. -func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt) diag.Diagnostics { +func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []Declare, parentModuleDefinitions map[string]string) diag.Diagnostics { start := time.Now() l.mut.Lock() defer l.mut.Unlock() @@ -129,15 +139,17 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co } l.cache.SyncModuleArgs(args) - newGraph, diags := l.loadNewGraph(args, componentBlocks, configBlocks) + l.parentModuleDefinitions = parentModuleDefinitions + newGraph, diags := l.loadNewGraph(args, componentBlocks, configBlocks, declares) if diags.HasErrors() { return diags } var ( - components = make([]*ComponentNode, 0, len(componentBlocks)) - componentIDs = make([]ComponentID, 0, len(componentBlocks)) - services = make([]*ServiceNode, 0, len(l.services)) + components = make([]*NativeComponentNode, 0, len(componentBlocks)) + componentIDs = make([]ComponentID, 0, len(componentBlocks)) + services = make([]*ServiceNode, 0, len(l.services)) + declareComponentNodes = make([]*DeclareComponentNode, 0, len(l.declareComponentNodes)) ) tracer := l.tracer.Tracer("") @@ -168,7 +180,7 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co var err error switch n := n.(type) { - case *ComponentNode: + case *NativeComponentNode: components = append(components, n) componentIDs = append(componentIDs, n.ID()) @@ -185,7 +197,6 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co }) } } - case *ServiceNode: services = append(services, n) @@ -202,7 +213,22 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co }) } } - + case *DeclareComponentNode: + declareComponentNodes = append(declareComponentNodes, n) + componentIDs = append(componentIDs, n.ID()) + if err = l.evaluate(logger, n); err != nil { + var evalDiags diag.Diagnostics + if errors.As(err, &evalDiags) { + diags = append(diags, evalDiags...) + } else { + diags.Add(diag.Diagnostic{ + Severity: diag.SeverityLevelError, + Message: fmt.Sprintf("Failed to build declared component: %s", err), + StartPos: ast.StartPos(n.Block()).Position(), + EndPos: ast.EndPos(n.Block()).Position(), + }) + } + } case BlockNode: if err = l.evaluate(logger, n); err != nil { diags.Add(diag.Diagnostic{ @@ -227,6 +253,7 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co return nil }) + l.declareComponentNodes = declareComponentNodes l.componentNodes = components l.serviceNodes = services l.graph = &newGraph @@ -252,9 +279,11 @@ func (l *Loader) Cleanup(stopWorkerPool bool) { } // loadNewGraph creates a new graph from the provided blocks and validates it. -func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt) (dag.Graph, diag.Diagnostics) { +func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []Declare) (dag.Graph, diag.Diagnostics) { var g dag.Graph + l.moduleReferences = make(map[string][]ModuleReference) + // Split component blocks into blocks for components and services. componentBlocks, serviceBlocks := l.splitComponentBlocks(componentBlocks) @@ -266,6 +295,10 @@ func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockS configBlockDiags := l.populateConfigBlockNodes(args, &g, configBlocks) diags = append(diags, configBlockDiags...) + // Fill our graph with declare nodes + declareDiags := l.populateDeclareNodes(&g, declares) + diags = append(diags, declareDiags...) + // Fill our graph with components. componentNodeDiags := l.populateComponentNodes(&g, componentBlocks) diags = append(diags, componentNodeDiags...) @@ -310,6 +343,24 @@ func (l *Loader) splitComponentBlocks(blocks []*ast.BlockStmt) (componentBlocks, return componentBlocks, serviceBlocks } +func (l *Loader) populateDeclareNodes(g *dag.Graph, declares []Declare) diag.Diagnostics { + var diags diag.Diagnostics + l.declareNodes = map[string]*DeclareNode{} + for _, declare := range declares { + node := NewDeclareNode(declare.Block, declare.Content) + if g.GetByID(node.NodeID()) != nil { + diags.Add(diag.Diagnostic{ + Severity: diag.SeverityLevelError, + Message: fmt.Sprintf("cannot add declare node %q; node with same ID already exists", node.NodeID()), + }) + continue + } + g.Add(node) + l.declareNodes[node.label] = node + } + return diags +} + // populateServiceNodes adds service nodes to the graph. func (l *Loader) populateServiceNodes(g *dag.Graph, serviceBlocks []*ast.BlockStmt) diag.Diagnostics { var diags diag.Diagnostics @@ -413,6 +464,8 @@ func (l *Loader) populateConfigBlockNodes(args map[string]any, g *dag.Graph, con g.Add(c) } + l.importNodes = nodeMap.importMap + return diags } @@ -423,7 +476,6 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo blockMap = make(map[string]*ast.BlockStmt, len(componentBlocks)) ) for _, block := range componentBlocks { - var c *ComponentNode id := BlockComponentID(block).String() if orig, redefined := blockMap[id]; redefined { @@ -438,23 +490,18 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo blockMap[id] = block // Check the graph from the previous call to Load to see we can copy an - // existing instance of ComponentNode. + // existing instance of NativeComponentNode and DeclareComponentNode. if exist := l.graph.GetByID(id); exist != nil { - c = exist.(*ComponentNode) - c.UpdateBlock(block) + switch v := exist.(type) { + case *NativeComponentNode: + v.UpdateBlock(block) + g.Add(v) + case *DeclareComponentNode: + v.UpdateBlock(block) + g.Add(v) + } } else { componentName := block.GetBlockName() - registration, exists := l.componentReg.Get(componentName) - if !exists { - diags.Add(diag.Diagnostic{ - Severity: diag.SeverityLevelError, - Message: fmt.Sprintf("Unrecognized component name %q", componentName), - StartPos: block.NamePos.Position(), - EndPos: block.NamePos.Add(len(componentName) - 1).Position(), - }) - continue - } - if block.Label == "" { diags.Add(diag.Diagnostic{ Severity: diag.SeverityLevelError, @@ -464,17 +511,57 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo }) continue } - - // Create a new component - c = NewComponentNode(l.globals, registration, block) + firstPart := strings.Split(componentName, ".")[0] + if l.shouldAddDeclareComponentNode(firstPart, componentName) { + g.Add(NewDeclareComponentNode(l.globals, block, l.getModuleInfo)) + } else { + registration, exists := l.componentReg.Get(componentName) + if !exists { + diags.Add(diag.Diagnostic{ + Severity: diag.SeverityLevelError, + Message: fmt.Sprintf("Unrecognized component name %q", componentName), + StartPos: block.NamePos.Position(), + EndPos: block.NamePos.Add(len(componentName) - 1).Position(), + }) + continue + } + g.Add(NewComponentNode(l.globals, registration, block)) + } } - - g.Add(c) } return diags } +func (l *Loader) shouldAddDeclareComponentNode(firstPart, componentName string) bool { + _, declareExists := l.declareNodes[firstPart] + _, importExists := l.importNodes[firstPart] + _, moduleDepExists := l.parentModuleDefinitions[componentName] + + return declareExists || importExists || moduleDepExists +} + +func (l *Loader) wireModuleReferences(g *dag.Graph, dc *DeclareComponentNode, declareNode *DeclareNode) error { + var references []ModuleReference + if deps, ok := l.moduleReferences[declareNode.label]; ok { + references = deps + } else { + var err error + references, err = GetModuleReferences(declareNode.content, l.importNodes, l.declareNodes, l.parentModuleDefinitions) + if err != nil { + return err + } + l.moduleReferences[declareNode.label] = references + } + // Add edges between the DeclareComponentNode and all import nodes that it needs. + for _, ref := range references { + if ref.importNode != nil { + g.AddEdge(dag.Edge{From: dc, To: ref.importNode}) + } + } + return nil +} + // Wire up all the related nodes func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { var diags diag.Diagnostics @@ -495,6 +582,20 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { g.AddEdge(dag.Edge{From: n, To: dep}) } + case *DeclareNode: + // A DeclareNode has no edge, it only holds a static content. + continue + case *DeclareComponentNode: + err := l.wireDeclareComponentNode(g, n) + if err != nil { + diags.Add(diag.Diagnostic{ + Severity: diag.SeverityLevelError, + Message: fmt.Sprintf("Error while parsing the declare component %s: %v", n.label, err), + StartPos: n.block.NamePos.Position(), + EndPos: n.block.NamePos.Add(len(n.componentName) - 1).Position(), + }) + continue + } } // Finally, wire component references. @@ -508,6 +609,18 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { return diags } +func (l *Loader) wireDeclareComponentNode(g *dag.Graph, dc *DeclareComponentNode) error { + if declareNode, exists := l.declareNodes[dc.declareLabel]; exists { + err := l.wireModuleReferences(g, dc, declareNode) + if err != nil { + return err + } + } else if importNode, exists := l.importNodes[dc.importLabel]; exists { + g.AddEdge(dag.Edge{From: dc, To: importNode}) + } + return nil +} + // Variables returns the Variables the Loader exposes for other Flow components // to reference. func (l *Loader) Variables() map[string]interface{} { @@ -515,12 +628,18 @@ func (l *Loader) Variables() map[string]interface{} { } // Components returns the current set of loaded components. -func (l *Loader) Components() []*ComponentNode { +func (l *Loader) Components() []*NativeComponentNode { l.mut.RLock() defer l.mut.RUnlock() return l.componentNodes } +func (l *Loader) DeclareComponent() []*DeclareComponentNode { + l.mut.RLock() + defer l.mut.RUnlock() + return l.declareComponentNodes +} + // Services returns the current set of service nodes. func (l *Loader) Services() []*ServiceNode { l.mut.RLock() @@ -528,6 +647,12 @@ func (l *Loader) Services() []*ServiceNode { return l.serviceNodes } +func (l *Loader) Imports() map[string]*ImportConfigNode { + l.mut.RLock() + defer l.mut.RUnlock() + return l.importNodes +} + // Graph returns a copy of the DAG managed by the Loader. func (l *Loader) Graph() *dag.Graph { l.mut.RLock() @@ -549,7 +674,7 @@ func (l *Loader) OriginalGraph() *dag.Graph { // the worker pool starts to evaluate them, resulting in smaller number of total evaluations when // node updates are frequent. If the worker pool's queue is full, EvaluateDependants will retry with a backoff until // it succeeds or until the ctx is cancelled. -func (l *Loader) EvaluateDependants(ctx context.Context, updatedNodes []*ComponentNode) { +func (l *Loader) EvaluateDependants(ctx context.Context, updatedNodes []NodeWithDependants) { if len(updatedNodes) == 0 { return } @@ -565,7 +690,7 @@ func (l *Loader) EvaluateDependants(ctx context.Context, updatedNodes []*Compone l.mut.RLock() defer l.mut.RUnlock() - dependenciesToParentsMap := make(map[dag.Node]*ComponentNode) + dependenciesToParentsMap := make(map[dag.Node]NodeWithDependants) for _, parent := range updatedNodes { // Make sure we're in-sync with the current exports of parent. l.cache.CacheExports(parent.ID(), parent.Exports()) @@ -624,9 +749,9 @@ func (l *Loader) EvaluateDependants(ctx context.Context, updatedNodes []*Compone // concurrentEvalFn returns a function that evaluates a node and updates the cache. This function can be submitted to // a worker pool for asynchronous evaluation. -func (l *Loader) concurrentEvalFn(n dag.Node, spanCtx context.Context, tracer trace.Tracer, parent *ComponentNode) { +func (l *Loader) concurrentEvalFn(n dag.Node, spanCtx context.Context, tracer trace.Tracer, parent NodeWithDependants) { start := time.Now() - l.cm.dependenciesWaitTime.Observe(time.Since(parent.lastUpdateTime.Load()).Seconds()) + l.cm.dependenciesWaitTime.Observe(time.Since(parent.LastUpdateTime()).Seconds()) _, span := tracer.Start(spanCtx, "EvaluateNode", trace.WithSpanKind(trace.SpanKindInternal)) span.SetAttributes(attribute.String("node_id", n.NodeID())) defer span.End() @@ -687,11 +812,14 @@ func (l *Loader) evaluate(logger log.Logger, bn BlockNode) error { // mut must be held when calling postEvaluate. func (l *Loader) postEvaluate(logger log.Logger, bn BlockNode, err error) error { switch c := bn.(type) { - case *ComponentNode: + case *NativeComponentNode: // Always update the cache both the arguments and exports, since both might // change when a component gets re-evaluated. We also want to cache the arguments and exports in case of an error l.cache.CacheArguments(c.ID(), c.Arguments()) l.cache.CacheExports(c.ID(), c.Exports()) + case *DeclareComponentNode: + l.cache.CacheArguments(c.ID(), c.Arguments()) + l.cache.CacheExports(c.ID(), c.Exports()) case *ArgumentConfigNode: if _, found := l.cache.moduleArguments[c.Label()]; !found { if c.Optional() { @@ -711,6 +839,13 @@ func (l *Loader) postEvaluate(logger log.Logger, bn BlockNode, err error) error return nil } +func (l *Loader) getModuleInfo(componentName string, importLabel string, declareLabel string) (ModuleInfo, error) { + if importLabel == "" { + return getLocalModuleInfo(l.declareNodes, l.moduleReferences, l.parentModuleDefinitions, componentName, declareLabel) + } + return getImportedModuleInfo(l.importNodes, l.parentModuleDefinitions, componentName, declareLabel, importLabel) +} + func multierrToDiags(errors error) diag.Diagnostics { var diags diag.Diagnostics for _, err := range errors.(*multierror.Error).Errors { diff --git a/pkg/flow/internal/controller/loader_test.go b/pkg/flow/internal/controller/loader_test.go index e93f757b1a2f..9351dda73657 100644 --- a/pkg/flow/internal/controller/loader_test.go +++ b/pkg/flow/internal/controller/loader_test.go @@ -73,7 +73,7 @@ func TestLoader(t *testing.T) { Logger: l, TraceProvider: noop.NewTracerProvider(), DataPath: t.TempDir(), - OnComponentUpdate: func(cn *controller.ComponentNode) { /* no-op */ }, + OnComponentUpdate: func(cn controller.NodeWithDependants) { /* no-op */ }, Registerer: prometheus.NewRegistry(), NewModuleController: func(id string) controller.ModuleController { return nil @@ -207,7 +207,7 @@ func TestScopeWithFailingComponent(t *testing.T) { Logger: l, TraceProvider: noop.NewTracerProvider(), DataPath: t.TempDir(), - OnComponentUpdate: func(cn *controller.ComponentNode) { /* no-op */ }, + OnComponentUpdate: func(cn controller.NodeWithDependants) { /* no-op */ }, Registerer: prometheus.NewRegistry(), NewModuleController: func(id string) controller.ModuleController { return fakeModuleController{} @@ -230,6 +230,7 @@ func applyFromContent(t *testing.T, l *controller.Loader, componentBytes []byte, diags diag.Diagnostics componentBlocks []*ast.BlockStmt configBlocks []*ast.BlockStmt = nil + declares []controller.Declare ) componentBlocks, diags = fileToBlock(t, componentBytes) @@ -244,7 +245,7 @@ func applyFromContent(t *testing.T, l *controller.Loader, componentBytes []byte, } } - applyDiags := l.Apply(nil, componentBlocks, configBlocks) + applyDiags := l.Apply(nil, componentBlocks, configBlocks, declares, nil) diags = append(diags, applyDiags...) return diags diff --git a/pkg/flow/internal/controller/module_info.go b/pkg/flow/internal/controller/module_info.go new file mode 100644 index 000000000000..7273a3cce685 --- /dev/null +++ b/pkg/flow/internal/controller/module_info.go @@ -0,0 +1,111 @@ +package controller + +import ( + "fmt" + "strings" +) + +type ModuleInfo struct { + content string + moduleDefinitions map[string]string +} + +func getLocalModuleInfo( + declareNodes map[string]*DeclareNode, + moduleReferences map[string][]ModuleReference, + parentModuleDefinitions map[string]string, + componentName string, + declareLabel string, +) (ModuleInfo, error) { + + var moduleInfo ModuleInfo + var content string + var err error + + if node, exists := declareNodes[declareLabel]; exists { + moduleInfo.moduleDefinitions, err = getLocalModuleDefinitions(componentName, moduleReferences, parentModuleDefinitions) + if err != nil { + return moduleInfo, err + } + + content, err = node.ModuleContent() + if err != nil { + return moduleInfo, err + } + } else if c, ok := parentModuleDefinitions[componentName]; ok { + content = c + moduleInfo.moduleDefinitions = parentModuleDefinitions + } else { + return moduleInfo, fmt.Errorf("could not find a definition for the declared module %s", componentName) + } + moduleInfo.content = content + return moduleInfo, nil +} + +func getLocalModuleDefinitions(componentName string, + localModuleReferences map[string][]ModuleReference, + parentModuleDefinitions map[string]string, +) (map[string]string, error) { + + moduleReferences := make(map[string]string) + for _, moduleDependency := range localModuleReferences[componentName] { + if moduleDependency.importNode != nil { + for importModulePath, importModuleContent := range moduleDependency.importNode.importedDeclares { + moduleReferences[moduleDependency.importNode.label+"."+importModulePath] = importModuleContent + } + } else if moduleDependency.declareNode != nil { + ref, err := moduleDependency.declareNode.ModuleContent() + if err != nil { + return moduleReferences, nil + } + moduleReferences[moduleDependency.declareLabel] = ref + } else { + // Nested declares have access to their parents module definitions. + if c, ok := parentModuleDefinitions[moduleDependency.componentName]; ok { + moduleReferences[moduleDependency.componentName] = c + } else { + return moduleReferences, fmt.Errorf("could not find the required module dependency %s for the module %s", moduleDependency.componentName, componentName) + } + } + } + return moduleReferences, nil +} + +func getImportedModuleInfo( + importNodes map[string]*ImportConfigNode, + parentModuleDefinitions map[string]string, + componentName string, + declareLabel string, + importLabel string, +) (ModuleInfo, error) { + + var moduleInfo ModuleInfo + var content string + var err error + if node, exists := importNodes[importLabel]; exists { + moduleInfo.moduleDefinitions = node.importedDeclares + content, err = node.ModuleContent(declareLabel) + if err != nil { + return moduleInfo, err + } + } else if c, ok := parentModuleDefinitions[componentName]; ok { + content = c + moduleInfo.moduleDefinitions = filterParentModuleDefinitions(importLabel, parentModuleDefinitions) + } else { + return moduleInfo, fmt.Errorf("could not find a definition for the imported module %s", componentName) + } + moduleInfo.content = content + return moduleInfo, nil +} + +// filterParentModuleDefinitions prevents modules from accessing other module definitions which are not in their scope. +func filterParentModuleDefinitions(importLabel string, parentModuleDefinitions map[string]string) map[string]string { + filteredParentModuleDefinitions := make(map[string]string) + for importPath, content := range parentModuleDefinitions { + // The scope is defined by the importLabel prefix in the importPath of the modules. + if strings.HasPrefix(importPath, importLabel) { + filteredParentModuleDefinitions[strings.TrimPrefix(importPath, importLabel+".")] = content + } + } + return filteredParentModuleDefinitions +} diff --git a/pkg/flow/internal/controller/module_references.go b/pkg/flow/internal/controller/module_references.go new file mode 100644 index 000000000000..2d975be9d169 --- /dev/null +++ b/pkg/flow/internal/controller/module_references.go @@ -0,0 +1,77 @@ +package controller + +import ( + "strings" + + "github.com/grafana/river/ast" + "github.com/grafana/river/parser" +) + +type ModuleReference struct { + componentName string + importLabel string + declareLabel string + importNode *ImportConfigNode + declareNode *DeclareNode +} + +// This function will parse the provided river content and collect references to known modules. +func GetModuleReferences( + content string, + importNodes map[string]*ImportConfigNode, + declareNodes map[string]*DeclareNode, + parentModuleDefinitions map[string]string, +) ([]ModuleReference, error) { + + uniqueReferences := make(map[string]ModuleReference) + err := getModuleReferences(content, importNodes, declareNodes, uniqueReferences, parentModuleDefinitions) + if err != nil { + return nil, err + } + + references := make([]ModuleReference, 0, len(uniqueReferences)) + for _, ref := range uniqueReferences { + references = append(references, ref) + } + + return references, nil +} + +func getModuleReferences( + content string, + importNodes map[string]*ImportConfigNode, + declareNodes map[string]*DeclareNode, + uniqueReferences map[string]ModuleReference, + parentModuleDefinitions map[string]string, +) error { + + node, err := parser.ParseFile("", []byte(content)) + if err != nil { + return err + } + + for _, stmt := range node.Body { + switch stmt := stmt.(type) { + case *ast.BlockStmt: + componentName := strings.Join(stmt.Name, ".") + switch componentName { + case "declare": + declareContent := content[stmt.LCurlyPos.Position().Offset+1 : stmt.RCurlyPos.Position().Offset-1] + err = getModuleReferences(declareContent, importNodes, declareNodes, uniqueReferences, parentModuleDefinitions) + if err != nil { + return err + } + default: + potentialImportLabel, potentialDeclareLabel := ExtractImportAndDeclareLabels(componentName) + if declareNode, ok := declareNodes[potentialDeclareLabel]; ok { + uniqueReferences[componentName] = ModuleReference{componentName: componentName, importLabel: "", declareLabel: potentialDeclareLabel, declareNode: declareNode} + } else if importNode, ok := importNodes[potentialImportLabel]; ok { + uniqueReferences[componentName] = ModuleReference{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel, importNode: importNode} + } else if _, ok := parentModuleDefinitions[componentName]; ok { + uniqueReferences[componentName] = ModuleReference{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel} + } + } + } + } + return nil +} diff --git a/pkg/flow/internal/controller/node_config.go b/pkg/flow/internal/controller/node_config.go index d583dc1e1061..7b84067cea4c 100644 --- a/pkg/flow/internal/controller/node_config.go +++ b/pkg/flow/internal/controller/node_config.go @@ -3,6 +3,7 @@ package controller import ( "fmt" + importsource "github.com/grafana/agent/pkg/flow/internal/import-source" "github.com/grafana/river/ast" "github.com/grafana/river/diag" ) @@ -26,6 +27,8 @@ func NewConfigNode(block *ast.BlockStmt, globals ComponentGlobals) (BlockNode, d return NewLoggingConfigNode(block, globals), nil case tracingBlockID: return NewTracingConfigNode(block, globals), nil + case importsource.BlockImportFile, importsource.BlockImportGit, importsource.BlockImportHTTP: + return NewImportConfigNode(block, globals, importsource.GetSourceType(block.GetBlockName())), nil default: var diags diag.Diagnostics diags.Add(diag.Diagnostic{ @@ -46,6 +49,7 @@ type ConfigNodeMap struct { tracing *TracingConfigNode argumentMap map[string]*ArgumentConfigNode exportMap map[string]*ExportConfigNode + importMap map[string]*ImportConfigNode } // NewConfigNodeMap will create an initial ConfigNodeMap. Append must be called @@ -56,6 +60,7 @@ func NewConfigNodeMap() *ConfigNodeMap { tracing: nil, argumentMap: map[string]*ArgumentConfigNode{}, exportMap: map[string]*ExportConfigNode{}, + importMap: map[string]*ImportConfigNode{}, } } @@ -73,6 +78,8 @@ func (nodeMap *ConfigNodeMap) Append(configNode BlockNode) diag.Diagnostics { nodeMap.logging = n case *TracingConfigNode: nodeMap.tracing = n + case *ImportConfigNode: + nodeMap.importMap[n.Label()] = n default: diags.Add(diag.Diagnostic{ Severity: diag.SeverityLevelError, diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go new file mode 100644 index 000000000000..856e1cb1bdf8 --- /dev/null +++ b/pkg/flow/internal/controller/node_config_import.go @@ -0,0 +1,396 @@ +package controller + +import ( + "context" + "fmt" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/go-kit/log" + "github.com/grafana/agent/component" + importsource "github.com/grafana/agent/pkg/flow/internal/import-source" + "github.com/grafana/agent/pkg/flow/logging/level" + "github.com/grafana/agent/pkg/flow/tracing" + "github.com/grafana/river/ast" + "github.com/grafana/river/parser" + "github.com/grafana/river/vm" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/atomic" +) + +type ImportConfigNode struct { + id ComponentID + label string + nodeID string + componentName string + globalID string + globals ComponentGlobals // Need a copy of the globals to create other import nodes. + source importsource.ImportSource + registry *prometheus.Registry + importedDeclares map[string]string + importConfigNodesChildren map[string]*ImportConfigNode + OnComponentUpdate func(cn NodeWithDependants) + logger log.Logger + inContentUpdate bool + + mut sync.RWMutex + importedContentMut sync.RWMutex + block *ast.BlockStmt // Current River blocks to derive config from + lastUpdateTime atomic.Time + + healthMut sync.RWMutex + evalHealth component.Health // Health of the last evaluate + runHealth component.Health // Health of running the component +} + +var _ NodeWithDependants = (*ImportConfigNode)(nil) +var _ RunnableNode = (*ImportConfigNode)(nil) +var _ ComponentNode = (*ImportConfigNode)(nil) + +// NewImportConfigNode creates a new ImportConfigNode from an initial ast.BlockStmt. +// The underlying config isn't applied until Evaluate is called. +func NewImportConfigNode(block *ast.BlockStmt, globals ComponentGlobals, sourceType importsource.SourceType) *ImportConfigNode { + var ( + id = BlockComponentID(block) + nodeID = id.String() + ) + + initHealth := component.Health{ + Health: component.HealthTypeUnknown, + Message: "component created", + UpdateTime: time.Now(), + } + globalID := nodeID + if globals.ControllerID != "" { + globalID = path.Join(globals.ControllerID, nodeID) + } + cn := &ImportConfigNode{ + id: id, + globalID: globalID, + label: block.Label, + globals: globals, + nodeID: BlockComponentID(block).String(), + componentName: block.GetBlockName(), + importedDeclares: make(map[string]string), + OnComponentUpdate: globals.OnComponentUpdate, + block: block, + evalHealth: initHealth, + runHealth: initHealth, + } + managedOpts := getImportManagedOptions(globals, cn) + cn.logger = managedOpts.Logger + cn.source = importsource.NewImportSource(sourceType, managedOpts, vm.New(block.Body), cn.onContentUpdate) + return cn +} + +func getImportManagedOptions(globals ComponentGlobals, cn *ImportConfigNode) component.Options { + cn.registry = prometheus.NewRegistry() + return component.Options{ + ID: cn.globalID, + Logger: log.With(globals.Logger, "component", cn.globalID), + Registerer: prometheus.WrapRegistererWith(prometheus.Labels{ + "component_id": cn.globalID, + }, cn.registry), + Tracer: tracing.WrapTracer(globals.TraceProvider, cn.globalID), + + DataPath: filepath.Join(globals.DataPath, cn.globalID), + + GetServiceData: func(name string) (interface{}, error) { + return globals.GetServiceData(name) + }, + } +} + +// Evaluate implements BlockNode and updates the arguments for the managed config block +// by re-evaluating its River block with the provided scope. The managed config block +// will be built the first time Evaluate is called. +// +// Evaluate will return an error if the River block cannot be evaluated or if +// decoding to arguments fails. +func (cn *ImportConfigNode) Evaluate(scope *vm.Scope) error { + err := cn.evaluate(scope) + + switch err { + case nil: + cn.setEvalHealth(component.HealthTypeHealthy, "component evaluated") + default: + msg := fmt.Sprintf("component evaluation failed: %s", err) + cn.setEvalHealth(component.HealthTypeUnhealthy, msg) + } + return err +} + +func (cn *ImportConfigNode) setEvalHealth(t component.HealthType, msg string) { + cn.healthMut.Lock() + defer cn.healthMut.Unlock() + + cn.evalHealth = component.Health{ + Health: t, + Message: msg, + UpdateTime: time.Now(), + } +} + +func (cn *ImportConfigNode) evaluate(scope *vm.Scope) error { + cn.mut.Lock() + defer cn.mut.Unlock() + return cn.source.Evaluate(scope) +} + +// processNodeBody processes the body of a node. +func (cn *ImportConfigNode) processNodeBody(node *ast.File, content string) { + for _, stmt := range node.Body { + switch stmt := stmt.(type) { + case *ast.BlockStmt: + fullName := strings.Join(stmt.Name, ".") + switch fullName { + case "declare": + cn.processDeclareBlock(stmt, content) + case importsource.BlockImportFile, importsource.BlockImportGit, importsource.BlockImportHTTP: + cn.processImportBlock(stmt, fullName) + default: + level.Error(cn.logger).Log("msg", "only declare and import blocks are allowed in a module", "forbidden", fullName) + } + default: + level.Error(cn.logger).Log("msg", "only declare and import blocks are allowed in a module") + } + } +} + +// processDeclareBlock processes a declare block. +func (cn *ImportConfigNode) processDeclareBlock(stmt *ast.BlockStmt, content string) { + if _, ok := cn.importedDeclares[stmt.Label]; ok { + level.Error(cn.logger).Log("msg", "declare block redefined", "name", stmt.Label) + return + } + cn.importedDeclares[stmt.Label] = content[stmt.LCurlyPos.Position().Offset+1 : stmt.RCurlyPos.Position().Offset-1] +} + +// processDeclareBlock processes an import block. +func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName string) { + sourceType := importsource.GetSourceType(fullName) + if _, ok := cn.importConfigNodesChildren[stmt.Label]; ok { + level.Error(cn.logger).Log("msg", "import block redefined", "name", stmt.Label) + return + } + childGlobals := cn.globals + // Children have a special OnComponentUpdate function which will surface all the imported declares to the root import config node. + childGlobals.OnComponentUpdate = cn.OnChildrenContentUpdate + cn.importConfigNodesChildren[stmt.Label] = NewImportConfigNode(stmt, childGlobals, sourceType) +} + +// onContentUpdate is triggered every time the managed import component has new content. +func (cn *ImportConfigNode) onContentUpdate(content string) { + cn.importedContentMut.Lock() + defer cn.importedContentMut.Unlock() + cn.inContentUpdate = true + defer func() { + cn.inContentUpdate = false + }() + cn.importedDeclares = make(map[string]string) + // We recreate the nodes when the content changes. Can we copy instead for optimization? + cn.importConfigNodesChildren = make(map[string]*ImportConfigNode) + node, err := parser.ParseFile(cn.label, []byte(content)) + if err != nil { + level.Error(cn.logger).Log("msg", "failed to parse file on update", "err", err) + return + } + cn.processNodeBody(node, content) + err = cn.evaluateChildren() + if err != nil { + level.Error(cn.logger).Log("msg", "failed to update content", "err", err) + return + } + cn.lastUpdateTime.Store(time.Now()) + cn.OnComponentUpdate(cn) +} + +// evaluateChildren evaluates the import nodes managed by this import node. +func (cn *ImportConfigNode) evaluateChildren() error { + for _, child := range cn.importConfigNodesChildren { + err := child.Evaluate(&vm.Scope{ + Parent: nil, + Variables: make(map[string]interface{}), + }) + if err != nil { + return fmt.Errorf("imported node %s failed to evaluate, %v", child.label, err) + } + } + return nil +} + +// runChildren run the import nodes managed by this import node. +func (cn *ImportConfigNode) runChildren(ctx context.Context) error { + var wg sync.WaitGroup + errChildrenChan := make(chan error, len(cn.importConfigNodesChildren)) + + for _, child := range cn.importConfigNodesChildren { + wg.Add(1) + go func(child *ImportConfigNode) { + defer wg.Done() + if err := child.Run(ctx); err != nil { + errChildrenChan <- err + } + }(child) + } + + go func() { + wg.Wait() + close(errChildrenChan) + }() + + return <-errChildrenChan +} + +// OnChildrenContentUpdate passes their imported content to their parents. +// To avoid collisions, the content is scoped via namespaces. +func (cn *ImportConfigNode) OnChildrenContentUpdate(child NodeWithDependants) { + switch child := child.(type) { + case *ImportConfigNode: + for importedDeclareLabel, content := range child.importedDeclares { + label := child.label + "." + importedDeclareLabel + cn.importedDeclares[label] = content + } + } + // This avoids OnComponentUpdate to be called multiple times in a row when the content changes. + if !cn.inContentUpdate { + cn.OnComponentUpdate(cn) + } +} + +// ModuleContent returns the content of a declare block imported by the node. +func (cn *ImportConfigNode) ModuleContent(declareLabel string) (string, error) { + cn.importedContentMut.Lock() + defer cn.importedContentMut.Unlock() + if content, ok := cn.importedDeclares[declareLabel]; ok { + return content, nil + } + return "", fmt.Errorf("declareLabel %s not found in imported node %s", declareLabel, cn.label) +} + +// Run runs the managed component in the calling goroutine until ctx is +// canceled. Evaluate must have been called at least once without returning an +// error before calling Run. +// +// Run will immediately return ErrUnevaluated if Evaluate has never been called +// successfully. Otherwise, Run will return nil. +func (cn *ImportConfigNode) Run(ctx context.Context) error { + cn.mut.RLock() + managed := cn.source + cn.mut.RUnlock() + + if managed == nil { + return ErrUnevaluated + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errChan := make(chan error, 1) + + if len(cn.importConfigNodesChildren) > 0 { + go func() { + errChan <- cn.runChildren(ctx) + }() + } + + cn.setRunHealth(component.HealthTypeHealthy, "started component") + + go func() { + errChan <- managed.Run(ctx) + }() + + err := <-errChan + + var exitMsg string + if err != nil { + level.Error(cn.logger).Log("msg", "component exited with error", "err", err) + exitMsg = fmt.Sprintf("component shut down with error: %s", err) + } else { + level.Info(cn.logger).Log("msg", "component exited") + exitMsg = "component shut down normally" + } + + cn.setRunHealth(component.HealthTypeExited, exitMsg) + return err +} + +func (cn *ImportConfigNode) setRunHealth(t component.HealthType, msg string) { + cn.healthMut.Lock() + defer cn.healthMut.Unlock() + + cn.runHealth = component.Health{ + Health: t, + Message: msg, + UpdateTime: time.Now(), + } +} + +func (cn *ImportConfigNode) Label() string { return cn.label } + +// Block implements BlockNode and returns the current block of the managed config node. +func (cn *ImportConfigNode) Block() *ast.BlockStmt { + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.block +} + +// NodeID implements dag.Node and returns the unique ID for the config node. +func (cn *ImportConfigNode) NodeID() string { return cn.nodeID } + +// This node has no exports. +func (cn *ImportConfigNode) Exports() component.Exports { + return nil +} + +func (cn *ImportConfigNode) ID() ComponentID { return cn.id } + +func (cn *ImportConfigNode) LastUpdateTime() time.Time { + return cn.lastUpdateTime.Load() +} + +// Arguments returns the current arguments of the managed component. +func (cn *ImportConfigNode) Arguments() component.Arguments { + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.source.Arguments() +} + +// Component returns the instance of the managed component. Component may be +// nil if the ComponentNode has not been successfully evaluated yet. +func (cn *ImportConfigNode) Component() component.Component { + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.source.Component() +} + +// CurrentHealth returns the current health of the ComponentNode. +// +// The health of a ComponentNode is determined by combining: +// +// 1. Health from the call to Run(). +// 2. Health from the last call to Evaluate(). +// 3. Health reported from the component. +func (cn *ImportConfigNode) CurrentHealth() component.Health { + cn.healthMut.RLock() + defer cn.healthMut.RUnlock() + return component.LeastHealthy(cn.runHealth, cn.evalHealth, cn.source.CurrentHealth()) +} + +// FileComponent does not have DebugInfo +func (cn *ImportConfigNode) DebugInfo() interface{} { + return nil +} + +// This component does not manage modules. +func (cn *ImportConfigNode) ModuleIDs() []string { + return nil +} + +// BlockName returns the name of the block. +func (cn *ImportConfigNode) BlockName() string { + return cn.componentName +} diff --git a/pkg/flow/internal/controller/node_declare.go b/pkg/flow/internal/controller/node_declare.go new file mode 100644 index 000000000000..ec9ff4596f83 --- /dev/null +++ b/pkg/flow/internal/controller/node_declare.go @@ -0,0 +1,55 @@ +package controller + +import ( + "sync" + + "github.com/grafana/river/ast" + "github.com/grafana/river/vm" +) + +type DeclareNode struct { + label string + nodeID string + componentName string + content string + + mut sync.RWMutex + block *ast.BlockStmt +} + +var _ BlockNode = (*DeclareNode)(nil) + +// NewDeclareNode creates a new declare node with a content which will be loaded by declare component nodes. +func NewDeclareNode(block *ast.BlockStmt, content string) *DeclareNode { + return &DeclareNode{ + label: block.Label, + nodeID: BlockComponentID(block).String(), + componentName: block.GetBlockName(), + content: content, + + block: block, + } +} + +func (cn *DeclareNode) ModuleContent() (string, error) { + cn.mut.Lock() + defer cn.mut.Unlock() + return cn.content, nil +} + +// Evaluate does nothing for this node. +func (cn *DeclareNode) Evaluate(scope *vm.Scope) error { + return nil +} + +func (cn *DeclareNode) Label() string { return cn.label } + +// Block implements BlockNode and returns the current block of the managed config node. +func (cn *DeclareNode) Block() *ast.BlockStmt { + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.block +} + +// NodeID implements dag.Node and returns the unique ID for the config node. +func (cn *DeclareNode) NodeID() string { return cn.nodeID } diff --git a/pkg/flow/internal/controller/node_declare_component.go b/pkg/flow/internal/controller/node_declare_component.go new file mode 100644 index 000000000000..b389d2c8989e --- /dev/null +++ b/pkg/flow/internal/controller/node_declare_component.go @@ -0,0 +1,361 @@ +package controller + +import ( + "context" + "fmt" + "path" + "path/filepath" + "reflect" + "strings" + "sync" + "time" + + "github.com/go-kit/log" + "github.com/grafana/agent/component" + "github.com/grafana/agent/component/module" + "github.com/grafana/agent/pkg/flow/logging/level" + "github.com/grafana/agent/pkg/flow/tracing" + "github.com/grafana/river/ast" + "github.com/grafana/river/vm" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/atomic" +) + +// DeclareComponentNode is a controller node which manages a module. +// +// DeclareComponentNode manages the underlying module and caches its current +// arguments and exports. +type DeclareComponentNode struct { + id ComponentID + globalID string + label string + componentName string + importLabel string + declareLabel string + nodeID string // Cached from id.String() to avoid allocating new strings every time NodeID is called. + managedOpts component.Options + registry *prometheus.Registry + moduleController ModuleController + OnComponentUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate + + GetModuleInfo func(fullName string, importLabel string, declareLabel string) (ModuleInfo, error) // Retrieve the module config. + lastUpdateTime atomic.Time + + mut sync.RWMutex + block *ast.BlockStmt // Current River block to derive args from + eval *vm.Evaluator + managed *module.ModuleComponent // Inner managed module + args component.Arguments // Evaluated arguments for the managed component + + // NOTE(rfratto): health and exports have their own mutex because they may be + // set asynchronously while mut is still being held (i.e., when calling Evaluate + // and the managed module immediately creates new exports) + + healthMut sync.RWMutex + evalHealth component.Health // Health of the last evaluate + runHealth component.Health // Health of running the component + + exportsMut sync.RWMutex + exports component.Exports // Evaluated exports for the managed module +} + +// ExtractImportAndDeclareLabels extract an importLabel and a declareLabel from a componentName. +func ExtractImportAndDeclareLabels(componentName string) (string, string) { + parts := strings.Split(componentName, ".") + if len(parts) == 0 { + return "", "" + } + // If this is a local declare. + importLabel := "" + declareLabel := parts[0] + // If this is an imported module. + if len(parts) > 1 { + importLabel = parts[0] + declareLabel = parts[1] + } + return importLabel, declareLabel +} + +var _ NodeWithDependants = (*DeclareComponentNode)(nil) +var _ ComponentNode = (*DeclareComponentNode)(nil) + +// NewDeclareComponentNode creates a new DeclareComponentNode from an initial ast.BlockStmt. +// The underlying managed module isn't created until Evaluate is called. +func NewDeclareComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetModuleInfo func(string, string, string) (ModuleInfo, error)) *DeclareComponentNode { + var ( + id = BlockComponentID(b) + nodeID = id.String() + ) + + initHealth := component.Health{ + Health: component.HealthTypeUnknown, + Message: "node declare component created", + UpdateTime: time.Now(), + } + + // We need to generate a globally unique component ID to give to the + // component and for use with telemetry data which doesn't support + // reconstructing the global ID. For everything else (HTTP, data), we can + // just use the controller-local ID as those values are guaranteed to be + // globally unique. + globalID := nodeID + if globals.ControllerID != "" { + globalID = path.Join(globals.ControllerID, nodeID) + } + + componentName := b.GetBlockName() + + importLabel, declareLabel := ExtractImportAndDeclareLabels(componentName) + + cn := &DeclareComponentNode{ + id: id, + globalID: globalID, + label: b.Label, + nodeID: nodeID, + componentName: componentName, + importLabel: importLabel, + declareLabel: declareLabel, + moduleController: globals.NewModuleController(globalID), + OnComponentUpdate: globals.OnComponentUpdate, + GetModuleInfo: GetModuleInfo, + + block: b, + eval: vm.New(b.Body), + + evalHealth: initHealth, + runHealth: initHealth, + } + cn.managedOpts = getDeclareManagedOptions(globals, cn) + + return cn +} + +func getDeclareManagedOptions(globals ComponentGlobals, cn *DeclareComponentNode) component.Options { + cn.registry = prometheus.NewRegistry() + return component.Options{ + ID: cn.globalID, + Logger: log.With(globals.Logger, "component", cn.globalID), + Registerer: prometheus.WrapRegistererWith(prometheus.Labels{ + "component_id": cn.globalID, + }, cn.registry), + Tracer: tracing.WrapTracer(globals.TraceProvider, cn.globalID), + + DataPath: filepath.Join(globals.DataPath, cn.globalID), + + OnStateChange: cn.setExports, + ModuleController: cn.moduleController, + + GetServiceData: func(name string) (interface{}, error) { + return globals.GetServiceData(name) + }, + } +} + +// ID returns the component ID of the managed component from its River block. +func (cn *DeclareComponentNode) ID() ComponentID { return cn.id } + +// Label returns the label for the block or "" if none was specified. +func (cn *DeclareComponentNode) Label() string { return cn.label } + +// NodeID implements dag.Node and returns the unique ID for this node. The +// NodeID is the string representation of the component's ID from its River +// block. +func (cn *DeclareComponentNode) NodeID() string { return cn.nodeID } + +// UpdateBlock updates the River block used to construct arguments for the +// managed module. The new block isn't used until the next time Evaluate is +// invoked. +// +// UpdateBlock will panic if the block does not match the component ID of the +// DeclareComponentNode. +func (cn *DeclareComponentNode) UpdateBlock(b *ast.BlockStmt) { + if !BlockComponentID(b).Equals(cn.id) { + panic("UpdateBlock called with an River block with a different component ID") + } + + cn.mut.Lock() + defer cn.mut.Unlock() + cn.block = b + cn.eval = vm.New(b.Body) +} + +// Evaluate implements BlockNode and updates the arguments by re-evaluating its River block with the provided scope and the module content by +// retrieving it from the corresponding import or declare node for the managed module. +// The managed module will be built the first time Evaluate is called. +// +// Evaluate will return an error if the River block cannot be evaluated, if +// decoding to arguments fails or if the module content cannot be retrieved. +func (cn *DeclareComponentNode) Evaluate(scope *vm.Scope) error { + err := cn.evaluate(scope) + + switch err { + case nil: + cn.setEvalHealth(component.HealthTypeHealthy, "component evaluated") + default: + msg := fmt.Sprintf("component evaluation failed: %s", err) + cn.setEvalHealth(component.HealthTypeUnhealthy, msg) + } + return err +} + +func (cn *DeclareComponentNode) evaluate(scope *vm.Scope) error { + cn.mut.Lock() + defer cn.mut.Unlock() + + var args map[string]any + if err := cn.eval.Evaluate(scope, &args); err != nil { + return fmt.Errorf("decoding River: %w", err) + } + + if cn.managed == nil { + // We haven't built the managed module successfully yet. + managed, err := module.NewModuleComponent(cn.managedOpts) + if err != nil { + return fmt.Errorf("building module: %w", err) + } + cn.managed = managed + } + + moduleInfo, err := cn.GetModuleInfo(cn.componentName, cn.importLabel, cn.declareLabel) + if err != nil { + return fmt.Errorf("retrieving module info: %w", err) + } + + // Reload the module with new config + if err := cn.managed.LoadFlowSource(args, moduleInfo.content, moduleInfo.moduleDefinitions); err != nil { + return fmt.Errorf("updating component: %w", err) + } + return nil +} + +func (cn *DeclareComponentNode) Run(ctx context.Context) error { + cn.mut.RLock() + managed := cn.managed + logger := cn.managedOpts.Logger + cn.mut.RUnlock() + + if managed == nil { + return ErrUnevaluated + } + + cn.setRunHealth(component.HealthTypeHealthy, "started module") + cn.managed.RunFlowController(ctx) + + level.Info(logger).Log("msg", "module exited") + cn.setRunHealth(component.HealthTypeExited, "module shut down") + return nil +} + +// Arguments returns the current arguments of the managed module. +func (cn *DeclareComponentNode) Arguments() component.Arguments { + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.args +} + +// Block implements BlockNode and returns the current block of the managed module. +func (cn *DeclareComponentNode) Block() *ast.BlockStmt { + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.block +} + +// Exports returns the current set of exports from the managed module. +// Exports returns nil if the managed module does not have exports. +func (cn *DeclareComponentNode) Exports() component.Exports { + cn.exportsMut.RLock() + defer cn.exportsMut.RUnlock() + return cn.exports +} + +func (cn *DeclareComponentNode) LastUpdateTime() time.Time { + return cn.lastUpdateTime.Load() +} + +// setExports is called whenever the managed module updates. e must be the +// same type as the registered exports type of the managed module. +func (cn *DeclareComponentNode) setExports(e component.Exports) { + // Some components may aggressively reexport values even though no exposed + // state has changed. This may be done for components which always supply + // exports whenever their arguments are evaluated without tracking internal + // state to see if anything actually changed. + // + // To avoid needlessly reevaluating components we'll ignore unchanged + // exports. + var changed bool + + cn.exportsMut.Lock() + if !reflect.DeepEqual(cn.exports, e) { + changed = true + cn.exports = e + } + cn.exportsMut.Unlock() + + if changed { + // Inform the controller that we have new exports. + cn.lastUpdateTime.Store(time.Now()) + cn.OnComponentUpdate(cn) + } +} + +// CurrentHealth returns the current health of the DeclareComponentNode. +// +// The health of a DeclareComponentNode is determined by combining: +// +// 1. Health from the call to Run(). +// 2. Health from the last call to Evaluate(). +// 3. Health reported from the module. +func (cn *DeclareComponentNode) CurrentHealth() component.Health { + cn.healthMut.RLock() + defer cn.healthMut.RUnlock() + return component.LeastHealthy(cn.runHealth, cn.evalHealth, cn.managed.CurrentHealth()) +} + +// TODO implement debugInfo? +func (cn *DeclareComponentNode) DebugInfo() interface{} { + cn.mut.RLock() + defer cn.mut.RUnlock() + return nil +} + +// setEvalHealth sets the internal health from a call to Evaluate. See Health +// for information on how overall health is calculated. +func (cn *DeclareComponentNode) setEvalHealth(t component.HealthType, msg string) { + cn.healthMut.Lock() + defer cn.healthMut.Unlock() + + cn.evalHealth = component.Health{ + Health: t, + Message: msg, + UpdateTime: time.Now(), + } +} + +// setRunHealth sets the internal health from a call to Run. See Health for +// information on how overall health is calculated. +func (cn *DeclareComponentNode) setRunHealth(t component.HealthType, msg string) { + cn.healthMut.Lock() + defer cn.healthMut.Unlock() + + cn.runHealth = component.Health{ + Health: t, + Message: msg, + UpdateTime: time.Now(), + } +} + +// ModuleIDs returns the current list of modules that this component is +// managing. +func (cn *DeclareComponentNode) ModuleIDs() []string { + return cn.moduleController.ModuleIDs() +} + +// BlockName returns the name of the block. +func (cn *DeclareComponentNode) BlockName() string { + return cn.componentName +} + +// This node does not manage any component. +func (cn *DeclareComponentNode) Component() component.Component { + return nil +} diff --git a/pkg/flow/internal/controller/node_component.go b/pkg/flow/internal/controller/node_native_component.go similarity index 85% rename from pkg/flow/internal/controller/node_component.go rename to pkg/flow/internal/controller/node_native_component.go index b99597809d4b..ef14722caf78 100644 --- a/pkg/flow/internal/controller/node_component.go +++ b/pkg/flow/internal/controller/node_native_component.go @@ -66,7 +66,7 @@ type ComponentGlobals struct { Logger *logging.Logger // Logger shared between all managed components. TraceProvider trace.TracerProvider // Tracer shared between all managed components. DataPath string // Shared directory where component data may be stored - OnComponentUpdate func(cn *ComponentNode) // Informs controller that we need to reevaluate + OnComponentUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate OnExportsChange func(exports map[string]any) // Invoked when the managed component updated its exports Registerer prometheus.Registerer // Registerer for serving agent and component metrics ControllerID string // ID of controller. @@ -74,12 +74,12 @@ type ComponentGlobals struct { GetServiceData func(name string) (interface{}, error) // Get data for a service. } -// ComponentNode is a controller node which manages a user-defined component. +// NativeComponentNode is a controller node which manages a user-defined component. // -// ComponentNode manages the underlying component and caches its current -// arguments and exports. ComponentNode manages the arguments for the component +// NativeComponentNode manages the underlying component and caches its current +// arguments and exports. NativeComponentNode manages the arguments for the component // from a River block. -type ComponentNode struct { +type NativeComponentNode struct { id ComponentID globalID string label string @@ -90,7 +90,7 @@ type ComponentNode struct { registry *prometheus.Registry exportsType reflect.Type moduleController ModuleController - OnComponentUpdate func(cn *ComponentNode) // Informs controller that we need to reevaluate + OnComponentUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate lastUpdateTime atomic.Time mut sync.RWMutex @@ -111,11 +111,12 @@ type ComponentNode struct { exports component.Exports // Evaluated exports for the managed component } -var _ BlockNode = (*ComponentNode)(nil) +var _ NodeWithDependants = (*NativeComponentNode)(nil) +var _ ComponentNode = (*NativeComponentNode)(nil) // NewComponentNode creates a new ComponentNode from an initial ast.BlockStmt. // The underlying managed component isn't created until Evaluate is called. -func NewComponentNode(globals ComponentGlobals, reg component.Registration, b *ast.BlockStmt) *ComponentNode { +func NewComponentNode(globals ComponentGlobals, reg component.Registration, b *ast.BlockStmt) *NativeComponentNode { var ( id = BlockComponentID(b) nodeID = id.String() @@ -137,12 +138,12 @@ func NewComponentNode(globals ComponentGlobals, reg component.Registration, b *a globalID = path.Join(globals.ControllerID, nodeID) } - cn := &ComponentNode{ + cn := &NativeComponentNode{ id: id, globalID: globalID, label: b.Label, nodeID: nodeID, - componentName: strings.Join(b.Name, "."), + componentName: b.GetBlockName(), reg: reg, exportsType: getExportsType(reg), moduleController: globals.NewModuleController(globalID), @@ -163,7 +164,7 @@ func NewComponentNode(globals ComponentGlobals, reg component.Registration, b *a return cn } -func getManagedOptions(globals ComponentGlobals, cn *ComponentNode) component.Options { +func getManagedOptions(globals ComponentGlobals, cn *NativeComponentNode) component.Options { cn.registry = prometheus.NewRegistry() return component.Options{ ID: cn.globalID, @@ -192,29 +193,29 @@ func getExportsType(reg component.Registration) reflect.Type { } // Registration returns the original registration of the component. -func (cn *ComponentNode) Registration() component.Registration { return cn.reg } +func (cn *NativeComponentNode) Registration() component.Registration { return cn.reg } // Component returns the instance of the managed component. Component may be // nil if the ComponentNode has not been successfully evaluated yet. -func (cn *ComponentNode) Component() component.Component { +func (cn *NativeComponentNode) Component() component.Component { cn.mut.RLock() defer cn.mut.RUnlock() return cn.managed } // ID returns the component ID of the managed component from its River block. -func (cn *ComponentNode) ID() ComponentID { return cn.id } +func (cn *NativeComponentNode) ID() ComponentID { return cn.id } // Label returns the label for the block or "" if none was specified. -func (cn *ComponentNode) Label() string { return cn.label } +func (cn *NativeComponentNode) Label() string { return cn.label } // ComponentName returns the component's type, i.e. `local.file.test` returns `local.file`. -func (cn *ComponentNode) ComponentName() string { return cn.componentName } +func (cn *NativeComponentNode) ComponentName() string { return cn.componentName } // NodeID implements dag.Node and returns the unique ID for this node. The // NodeID is the string representation of the component's ID from its River // block. -func (cn *ComponentNode) NodeID() string { return cn.nodeID } +func (cn *NativeComponentNode) NodeID() string { return cn.nodeID } // UpdateBlock updates the River block used to construct arguments for the // managed component. The new block isn't used until the next time Evaluate is @@ -222,7 +223,7 @@ func (cn *ComponentNode) NodeID() string { return cn.nodeID } // // UpdateBlock will panic if the block does not match the component ID of the // ComponentNode. -func (cn *ComponentNode) UpdateBlock(b *ast.BlockStmt) { +func (cn *NativeComponentNode) UpdateBlock(b *ast.BlockStmt) { if !BlockComponentID(b).Equals(cn.id) { panic("UpdateBlock called with an River block with a different component ID") } @@ -239,7 +240,7 @@ func (cn *ComponentNode) UpdateBlock(b *ast.BlockStmt) { // // Evaluate will return an error if the River block cannot be evaluated or if // decoding to arguments fails. -func (cn *ComponentNode) Evaluate(scope *vm.Scope) error { +func (cn *NativeComponentNode) Evaluate(scope *vm.Scope) error { err := cn.evaluate(scope) switch err { @@ -252,7 +253,7 @@ func (cn *ComponentNode) Evaluate(scope *vm.Scope) error { return err } -func (cn *ComponentNode) evaluate(scope *vm.Scope) error { +func (cn *NativeComponentNode) evaluate(scope *vm.Scope) error { cn.mut.Lock() defer cn.mut.Unlock() @@ -299,7 +300,7 @@ func (cn *ComponentNode) evaluate(scope *vm.Scope) error { // // Run will immediately return ErrUnevaluated if Evaluate has never been called // successfully. Otherwise, Run will return nil. -func (cn *ComponentNode) Run(ctx context.Context) error { +func (cn *NativeComponentNode) Run(ctx context.Context) error { cn.mut.RLock() managed := cn.managed cn.mut.RUnlock() @@ -330,14 +331,14 @@ func (cn *ComponentNode) Run(ctx context.Context) error { var ErrUnevaluated = errors.New("managed component not built") // Arguments returns the current arguments of the managed component. -func (cn *ComponentNode) Arguments() component.Arguments { +func (cn *NativeComponentNode) Arguments() component.Arguments { cn.mut.RLock() defer cn.mut.RUnlock() return cn.args } // Block implements BlockNode and returns the current block of the managed component. -func (cn *ComponentNode) Block() *ast.BlockStmt { +func (cn *NativeComponentNode) Block() *ast.BlockStmt { cn.mut.RLock() defer cn.mut.RUnlock() return cn.block @@ -345,15 +346,19 @@ func (cn *ComponentNode) Block() *ast.BlockStmt { // Exports returns the current set of exports from the managed component. // Exports returns nil if the managed component does not have exports. -func (cn *ComponentNode) Exports() component.Exports { +func (cn *NativeComponentNode) Exports() component.Exports { cn.exportsMut.RLock() defer cn.exportsMut.RUnlock() return cn.exports } +func (cn *NativeComponentNode) LastUpdateTime() time.Time { + return cn.lastUpdateTime.Load() +} + // setExports is called whenever the managed component updates. e must be the // same type as the registered exports type of the managed component. -func (cn *ComponentNode) setExports(e component.Exports) { +func (cn *NativeComponentNode) setExports(e component.Exports) { if cn.exportsType == nil { panic(fmt.Sprintf("Component %s called OnStateChange but never registered an Exports type", cn.nodeID)) } @@ -391,7 +396,7 @@ func (cn *ComponentNode) setExports(e component.Exports) { // 1. Health from the call to Run(). // 2. Health from the last call to Evaluate(). // 3. Health reported from the component. -func (cn *ComponentNode) CurrentHealth() component.Health { +func (cn *NativeComponentNode) CurrentHealth() component.Health { cn.healthMut.RLock() defer cn.healthMut.RUnlock() @@ -409,7 +414,7 @@ func (cn *ComponentNode) CurrentHealth() component.Health { } // DebugInfo returns debugging information from the managed component (if any). -func (cn *ComponentNode) DebugInfo() interface{} { +func (cn *NativeComponentNode) DebugInfo() interface{} { cn.mut.RLock() defer cn.mut.RUnlock() @@ -421,7 +426,7 @@ func (cn *ComponentNode) DebugInfo() interface{} { // setEvalHealth sets the internal health from a call to Evaluate. See Health // for information on how overall health is calculated. -func (cn *ComponentNode) setEvalHealth(t component.HealthType, msg string) { +func (cn *NativeComponentNode) setEvalHealth(t component.HealthType, msg string) { cn.healthMut.Lock() defer cn.healthMut.Unlock() @@ -434,7 +439,7 @@ func (cn *ComponentNode) setEvalHealth(t component.HealthType, msg string) { // setRunHealth sets the internal health from a call to Run. See Health for // information on how overall health is calculated. -func (cn *ComponentNode) setRunHealth(t component.HealthType, msg string) { +func (cn *NativeComponentNode) setRunHealth(t component.HealthType, msg string) { cn.healthMut.Lock() defer cn.healthMut.Unlock() @@ -447,6 +452,11 @@ func (cn *ComponentNode) setRunHealth(t component.HealthType, msg string) { // ModuleIDs returns the current list of modules that this component is // managing. -func (cn *ComponentNode) ModuleIDs() []string { +func (cn *NativeComponentNode) ModuleIDs() []string { return cn.moduleController.ModuleIDs() } + +// BlockName returns the name of the block. +func (cn *NativeComponentNode) BlockName() string { + return cn.componentName +} diff --git a/pkg/flow/internal/controller/node_component_test.go b/pkg/flow/internal/controller/node_native_component_test.go similarity index 93% rename from pkg/flow/internal/controller/node_component_test.go rename to pkg/flow/internal/controller/node_native_component_test.go index 6eb46f004601..e2f734352030 100644 --- a/pkg/flow/internal/controller/node_component_test.go +++ b/pkg/flow/internal/controller/node_native_component_test.go @@ -14,7 +14,7 @@ func TestGlobalID(t *testing.T) { NewModuleController: func(id string) ModuleController { return nil }, - }, &ComponentNode{ + }, &NativeComponentNode{ nodeID: "local.id", globalID: "module.file/local.id", }) @@ -28,7 +28,7 @@ func TestLocalID(t *testing.T) { NewModuleController: func(id string) ModuleController { return nil }, - }, &ComponentNode{ + }, &NativeComponentNode{ nodeID: "local.id", globalID: "local.id", }) diff --git a/pkg/flow/internal/controller/node_with_dependants.go b/pkg/flow/internal/controller/node_with_dependants.go new file mode 100644 index 000000000000..a7c47360c7ff --- /dev/null +++ b/pkg/flow/internal/controller/node_with_dependants.go @@ -0,0 +1,18 @@ +package controller + +import ( + "time" + + "github.com/grafana/agent/component" +) + +// NodeWithDependants must be implemented by nodes which can trigger other nodes to be evaluated. +type NodeWithDependants interface { + BlockNode + + LastUpdateTime() time.Time + + Exports() component.Exports + + ID() ComponentID +} diff --git a/pkg/flow/internal/controller/queue.go b/pkg/flow/internal/controller/queue.go index a8cd1b5bae05..d4708cecf806 100644 --- a/pkg/flow/internal/controller/queue.go +++ b/pkg/flow/internal/controller/queue.go @@ -10,8 +10,8 @@ import ( // for later reevaluation. type Queue struct { mut sync.Mutex - queuedSet map[*ComponentNode]struct{} - queuedOrder []*ComponentNode + queuedSet map[NodeWithDependants]struct{} + queuedOrder []NodeWithDependants updateCh chan struct{} } @@ -20,14 +20,14 @@ type Queue struct { func NewQueue() *Queue { return &Queue{ updateCh: make(chan struct{}, 1), - queuedSet: make(map[*ComponentNode]struct{}), - queuedOrder: make([]*ComponentNode, 0), + queuedSet: make(map[NodeWithDependants]struct{}), + queuedOrder: make([]NodeWithDependants, 0), } } // Enqueue inserts a new component into the Queue. Enqueue is a no-op if the // component is already in the Queue. -func (q *Queue) Enqueue(c *ComponentNode) { +func (q *Queue) Enqueue(c NodeWithDependants) { q.mut.Lock() defer q.mut.Unlock() @@ -48,13 +48,13 @@ func (q *Queue) Enqueue(c *ComponentNode) { func (q *Queue) Chan() <-chan struct{} { return q.updateCh } // DequeueAll removes all components from the queue and returns them. -func (q *Queue) DequeueAll() []*ComponentNode { +func (q *Queue) DequeueAll() []NodeWithDependants { q.mut.Lock() defer q.mut.Unlock() all := q.queuedOrder - q.queuedOrder = make([]*ComponentNode, 0) - q.queuedSet = make(map[*ComponentNode]struct{}) + q.queuedOrder = make([]NodeWithDependants, 0) + q.queuedSet = make(map[NodeWithDependants]struct{}) return all } diff --git a/pkg/flow/internal/controller/queue_test.go b/pkg/flow/internal/controller/queue_test.go index c93fb14ef8fc..8fe953ba31ba 100644 --- a/pkg/flow/internal/controller/queue_test.go +++ b/pkg/flow/internal/controller/queue_test.go @@ -9,7 +9,7 @@ import ( ) func TestEnqueueDequeue(t *testing.T) { - tn := &ComponentNode{} + tn := &NativeComponentNode{} q := NewQueue() q.Enqueue(tn) require.Lenf(t, q.queuedSet, 1, "queue should be 1") @@ -26,7 +26,7 @@ func TestDequeue_Empty(t *testing.T) { } func TestDequeue_InOrder(t *testing.T) { - c1, c2, c3 := &ComponentNode{}, &ComponentNode{}, &ComponentNode{} + c1, c2, c3 := &NativeComponentNode{}, &NativeComponentNode{}, &NativeComponentNode{} q := NewQueue() q.Enqueue(c1) q.Enqueue(c2) @@ -41,7 +41,7 @@ func TestDequeue_InOrder(t *testing.T) { } func TestDequeue_NoDuplicates(t *testing.T) { - c1, c2 := &ComponentNode{}, &ComponentNode{} + c1, c2 := &NativeComponentNode{}, &NativeComponentNode{} q := NewQueue() q.Enqueue(c1) q.Enqueue(c1) @@ -58,7 +58,7 @@ func TestDequeue_NoDuplicates(t *testing.T) { } func TestEnqueue_ChannelNotification(t *testing.T) { - c1 := &ComponentNode{} + c1 := &NativeComponentNode{} q := NewQueue() notificationsCount := atomic.Int32{} diff --git a/pkg/flow/internal/import-source/import_file.go b/pkg/flow/internal/import-source/import_file.go new file mode 100644 index 000000000000..178064c91872 --- /dev/null +++ b/pkg/flow/internal/import-source/import_file.go @@ -0,0 +1,86 @@ +package importsource + +import ( + "context" + "fmt" + "reflect" + + "github.com/grafana/agent/component" + "github.com/grafana/agent/component/local/file" + "github.com/grafana/river/vm" +) + +type ImportFile struct { + fileComponent *file.Component + arguments component.Arguments + managedOpts component.Options + eval *vm.Evaluator +} + +var _ ImportSource = (*ImportFile)(nil) + +func NewImportFile(managedOpts component.Options, eval *vm.Evaluator, onContentChange func(string)) *ImportFile { + opts := managedOpts + opts.OnStateChange = func(e component.Exports) { + onContentChange(e.(file.Exports).Content.Value) + } + return &ImportFile{ + managedOpts: opts, + eval: eval, + } +} + +type importFileConfigBlock struct { + LocalFileArguments file.Arguments `river:",squash"` +} + +// SetToDefault implements river.Defaulter. +func (a *importFileConfigBlock) SetToDefault() { + a.LocalFileArguments = file.DefaultArguments +} + +func (im *ImportFile) Evaluate(scope *vm.Scope) error { + var arguments importFileConfigBlock + if err := im.eval.Evaluate(scope, &arguments); err != nil { + return fmt.Errorf("decoding River: %w", err) + } + if im.fileComponent == nil { + var err error + im.fileComponent, err = file.New(im.managedOpts, arguments.LocalFileArguments) + if err != nil { + return fmt.Errorf("creating file component: %w", err) + } + im.arguments = arguments + } + + if reflect.DeepEqual(im.arguments, arguments) { + return nil + } + + // Update the existing managed component + if err := im.fileComponent.Update(arguments); err != nil { + return fmt.Errorf("updating component: %w", err) + } + return nil +} + +func (im *ImportFile) Run(ctx context.Context) error { + return im.fileComponent.Run(ctx) +} + +func (im *ImportFile) Arguments() component.Arguments { + return im.arguments +} + +func (im *ImportFile) Component() component.Component { + return im.fileComponent +} + +func (im *ImportFile) CurrentHealth() component.Health { + return im.fileComponent.CurrentHealth() +} + +// DebugInfo() is not implemented by the file component. +func (im *ImportFile) DebugInfo() interface{} { + return nil +} diff --git a/pkg/flow/internal/import-source/import_git.go b/pkg/flow/internal/import-source/import_git.go new file mode 100644 index 000000000000..b889f3a9d157 --- /dev/null +++ b/pkg/flow/internal/import-source/import_git.go @@ -0,0 +1,281 @@ +package importsource + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "reflect" + "sync" + "time" + + "github.com/go-kit/log" + + "github.com/grafana/agent/component" + "github.com/grafana/agent/pkg/flow/logging/level" + vcs "github.com/grafana/agent/pkg/util/git" + "github.com/grafana/river/vm" +) + +// The difference between this import source and the others is that there is no git component. +// The git logic in the internal package is a copy of the one used in the old module. +type ImportGit struct { + opts component.Options + log log.Logger + eval *vm.Evaluator + mut sync.RWMutex + repo *vcs.GitRepo + repoOpts vcs.GitRepoOptions + args Arguments + onContentChange func(string) + + lastContent string + + argsChanged chan struct{} + + healthMut sync.RWMutex + health component.Health +} + +var ( + _ ImportSource = (*ImportGit)(nil) + _ component.Component = (*ImportGit)(nil) + _ component.HealthComponent = (*ImportGit)(nil) +) + +type Arguments struct { + Repository string `river:"repository,attr"` + Revision string `river:"revision,attr,optional"` + Path string `river:"path,attr"` + PullFrequency time.Duration `river:"pull_frequency,attr,optional"` + GitAuthConfig vcs.GitAuthConfig `river:",squash"` +} + +var DefaultArguments = Arguments{ + Revision: "HEAD", + PullFrequency: time.Minute, +} + +// SetToDefault implements river.Defaulter. +func (args *Arguments) SetToDefault() { + *args = DefaultArguments +} + +func NewImportGit(managedOpts component.Options, eval *vm.Evaluator, onContentChange func(string)) *ImportGit { + return &ImportGit{ + opts: managedOpts, + log: managedOpts.Logger, + eval: eval, + argsChanged: make(chan struct{}, 1), + onContentChange: onContentChange, + } +} + +func (im *ImportGit) Evaluate(scope *vm.Scope) error { + var arguments Arguments + if err := im.eval.Evaluate(scope, &arguments); err != nil { + return fmt.Errorf("decoding River: %w", err) + } + + if reflect.DeepEqual(im.args, arguments) { + return nil + } + + if err := im.Update(arguments); err != nil { + return fmt.Errorf("updating component: %w", err) + } + return nil +} + +func (im *ImportGit) Run(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + ticker *time.Ticker + tickerC <-chan time.Time + ) + + for { + select { + case <-ctx.Done(): + return nil + + case <-im.argsChanged: + im.mut.Lock() + pullFrequency := im.args.PullFrequency + im.mut.Unlock() + ticker, tickerC = im.updateTicker(pullFrequency, ticker, tickerC) + + case <-tickerC: + level.Info(im.log).Log("msg", "updating repository") + im.tickPollFile(ctx) + } + } +} + +func (im *ImportGit) updateTicker(pullFrequency time.Duration, ticker *time.Ticker, tickerC <-chan time.Time) (*time.Ticker, <-chan time.Time) { + level.Info(im.log).Log("msg", "updating repository pull frequency, next pull attempt will be done according to the pullFrequency", "new_frequency", pullFrequency) + + if pullFrequency > 0 { + if ticker == nil { + ticker = time.NewTicker(pullFrequency) + tickerC = ticker.C + } else { + ticker.Reset(pullFrequency) + } + return ticker, tickerC + } + + if ticker != nil { + ticker.Stop() + } + return nil, nil +} + +func (im *ImportGit) tickPollFile(ctx context.Context) { + im.mut.Lock() + err := im.pollFile(ctx, im.args) + pullFrequency := im.args.PullFrequency + im.mut.Unlock() + + im.updateHealth(err) + + if err != nil { + level.Error(im.log).Log("msg", "failed to update repository", "pullFrequency", pullFrequency, "err", err) + } +} + +func (im *ImportGit) updateHealth(err error) { + im.healthMut.Lock() + defer im.healthMut.Unlock() + + if err != nil { + im.health = component.Health{ + Health: component.HealthTypeUnhealthy, + Message: err.Error(), + UpdateTime: time.Now(), + } + } else { + im.health = component.Health{ + Health: component.HealthTypeHealthy, + Message: "module updated", + UpdateTime: time.Now(), + } + } +} + +// Update implements component.Component. +// Only acknowledge the error from Update if it's not a +// vcs.UpdateFailedError; vcs.UpdateFailedError means that the Git repo +// exists, but we were unable to update it. It makes sense to retry on the next poll and it may succeed. +func (im *ImportGit) Update(args component.Arguments) (err error) { + defer func() { + im.updateHealth(err) + }() + im.mut.Lock() + defer im.mut.Unlock() + + newArgs := args.(Arguments) + + // TODO(rfratto): store in a repo-specific directory so changing repositories + // doesn't risk break the module loader if there's a SHA collision between + // the two different repositories. + repoPath := filepath.Join(im.opts.DataPath, "repo") + + repoOpts := vcs.GitRepoOptions{ + Repository: newArgs.Repository, + Revision: newArgs.Revision, + Auth: newArgs.GitAuthConfig, + } + + // Create or update the repo field. + // Failure to update repository makes the module loader temporarily use cached contents on disk + if im.repo == nil || !reflect.DeepEqual(repoOpts, im.repoOpts) { + r, err := vcs.NewGitRepo(context.Background(), repoPath, repoOpts) + if err != nil { + if errors.As(err, &vcs.UpdateFailedError{}) { + level.Error(im.log).Log("msg", "failed to update repository", "err", err) + im.updateHealth(err) + } else { + return err + } + } + im.repo = r + im.repoOpts = repoOpts + } + + if err := im.pollFile(context.Background(), newArgs); err != nil { + if errors.As(err, &vcs.UpdateFailedError{}) { + level.Error(im.log).Log("msg", "failed to poll file from repository", "err", err) + // We don't update the health here because it will be updated via the defer call. + // This is not very good because if we reassign the err before exiting the function it will not update the health correctly. + // TODO improve the error health handling. + } else { + return err + } + } + + // Schedule an update for handling the changed arguments. + select { + case im.argsChanged <- struct{}{}: + default: + } + + im.args = newArgs + return nil +} + +// pollFile fetches the latest content from the repository and updates the +// controller. pollFile must only be called with im.mut held. +func (im *ImportGit) pollFile(ctx context.Context, args Arguments) error { + // Make sure our repo is up-to-date. + if err := im.repo.Update(ctx); err != nil { + return err + } + + // Finally, configure our controller. + bb, err := im.repo.ReadFile(args.Path) + if err != nil { + return err + } + content := string(bb) + if im.lastContent != content { + im.onContentChange(content) + im.lastContent = content + } + return nil +} + +// CurrentHealth implements component.HealthComponent. +func (im *ImportGit) CurrentHealth() component.Health { + im.healthMut.RLock() + defer im.healthMut.RUnlock() + return im.health +} + +// DebugInfo implements component.DebugComponent. +func (im *ImportGit) DebugInfo() interface{} { + type DebugInfo struct { + SHA string `river:"sha,attr"` + RepoError string `river:"repo_error,attr,optional"` + } + + im.mut.RLock() + defer im.mut.RUnlock() + + rev, err := im.repo.CurrentRevision() + if err != nil { + return DebugInfo{RepoError: err.Error()} + } else { + return DebugInfo{SHA: rev} + } +} + +func (im *ImportGit) Arguments() component.Arguments { + return im.args +} + +func (im *ImportGit) Component() component.Component { + return im +} diff --git a/pkg/flow/internal/import-source/import_http.go b/pkg/flow/internal/import-source/import_http.go new file mode 100644 index 000000000000..9bc0ece91fbc --- /dev/null +++ b/pkg/flow/internal/import-source/import_http.go @@ -0,0 +1,87 @@ +package importsource + +import ( + "context" + "fmt" + "reflect" + + "github.com/grafana/agent/component" + "github.com/grafana/agent/component/remote/http" + remote_http "github.com/grafana/agent/component/remote/http" + "github.com/grafana/river/vm" +) + +type ImportHTTP struct { + managedRemoteHTTP *remote_http.Component + arguments component.Arguments + managedOpts component.Options + eval *vm.Evaluator +} + +var _ ImportSource = (*ImportHTTP)(nil) + +func NewImportHTTP(managedOpts component.Options, eval *vm.Evaluator, onContentChange func(string)) *ImportHTTP { + opts := managedOpts + opts.OnStateChange = func(e component.Exports) { + onContentChange(e.(http.Exports).Content.Value) + } + return &ImportHTTP{ + managedOpts: opts, + eval: eval, + } +} + +type ImportHTTPConfigBlock struct { + RemoteHTTPArguments remote_http.Arguments `river:",squash"` +} + +// SetToDefault implements river.Defaulter. +func (a *ImportHTTPConfigBlock) SetToDefault() { + a.RemoteHTTPArguments.SetToDefault() +} + +func (im *ImportHTTP) Evaluate(scope *vm.Scope) error { + var arguments ImportHTTPConfigBlock + if err := im.eval.Evaluate(scope, &arguments); err != nil { + return fmt.Errorf("decoding River: %w", err) + } + if im.managedRemoteHTTP == nil { + var err error + im.managedRemoteHTTP, err = remote_http.New(im.managedOpts, arguments.RemoteHTTPArguments) + if err != nil { + return fmt.Errorf("creating http component: %w", err) + } + im.arguments = arguments + } + + if reflect.DeepEqual(im.arguments, arguments) { + return nil + } + + // Update the existing managed component + if err := im.managedRemoteHTTP.Update(arguments); err != nil { + return fmt.Errorf("updating component: %w", err) + } + return nil +} + +func (im *ImportHTTP) Run(ctx context.Context) error { + return im.managedRemoteHTTP.Run(ctx) +} + +func (im *ImportHTTP) Arguments() component.Arguments { + return im.arguments +} + +func (im *ImportHTTP) Component() component.Component { + return im.managedRemoteHTTP +} + +func (im *ImportHTTP) CurrentHealth() component.Health { + return im.managedRemoteHTTP.CurrentHealth() +} + +// DebugInfo() is not implemented by the http component. +func (im *ImportHTTP) DebugInfo() interface{} { + return nil +} diff --git a/pkg/flow/internal/import-source/import_source.go b/pkg/flow/internal/import-source/import_source.go new file mode 100644 index 000000000000..1467234c1d7e --- /dev/null +++ b/pkg/flow/internal/import-source/import_source.go @@ -0,0 +1,58 @@ +package importsource + +import ( + "context" + "fmt" + + "github.com/grafana/agent/component" + "github.com/grafana/river/vm" +) + +type SourceType int + +const ( + FILE SourceType = iota + HTTP + GIT +) + +const ( + BlockImportFile = "import.file" + BlockImportHTTP = "import.http" + BlockImportGit = "import.git" +) + +type ImportSource interface { + Evaluate(scope *vm.Scope) error + Run(ctx context.Context) error + Component() component.Component + CurrentHealth() component.Health + DebugInfo() interface{} + Arguments() component.Arguments +} + +func NewImportSource(sourceType SourceType, managedOpts component.Options, eval *vm.Evaluator, onContentChange func(string)) ImportSource { + switch sourceType { + case FILE: + return NewImportFile(managedOpts, eval, onContentChange) + case HTTP: + return NewImportHTTP(managedOpts, eval, onContentChange) + case GIT: + return NewImportGit(managedOpts, eval, onContentChange) + } + // This is a programming error, not a config error so this is ok to panic. + panic(fmt.Errorf("unsupported source type: %v", sourceType)) +} + +func GetSourceType(fullName string) SourceType { + switch fullName { + case BlockImportFile: + return FILE + case BlockImportGit: + return GIT + case BlockImportHTTP: + return HTTP + } + // This is a programming error, not a config error so this is ok to panic. + panic(fmt.Errorf("name does not map to a know source type: %v", fullName)) +} diff --git a/pkg/flow/module.go b/pkg/flow/module.go index ec97aab093d5..d67eefcb0508 100644 --- a/pkg/flow/module.go +++ b/pkg/flow/module.go @@ -128,12 +128,12 @@ func newModule(o *moduleOptions) *module { } // LoadConfig parses River config and loads it. -func (c *module) LoadConfig(config []byte, args map[string]any) error { +func (c *module) LoadConfig(config []byte, args map[string]any, parentModuleDefinitions map[string]string) error { ff, err := ParseSource(c.o.ID, config) if err != nil { return err } - return c.f.LoadSource(ff, args) + return c.f.LoadSource(ff, args, parentModuleDefinitions) } // Run starts the Module. No components within the Module diff --git a/pkg/flow/module_caching_test.go b/pkg/flow/module_caching_test.go index e22e0583cbda..bdfce702b6fb 100644 --- a/pkg/flow/module_caching_test.go +++ b/pkg/flow/module_caching_test.go @@ -60,7 +60,7 @@ func TestUpdates_EmptyModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -121,7 +121,7 @@ func TestUpdates_ThroughModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/flow/module_declare_test.go b/pkg/flow/module_declare_test.go new file mode 100644 index 000000000000..c471bf38beea --- /dev/null +++ b/pkg/flow/module_declare_test.go @@ -0,0 +1,227 @@ +package flow_test + +import ( + "context" + "testing" + "time" + + "github.com/grafana/agent/pkg/flow" + "github.com/grafana/agent/pkg/flow/internal/testcomponents" + "github.com/stretchr/testify/require" +) + +type testCase struct { + name string + config string + expected int +} + +func TestDeclareComponent(t *testing.T) { + tt := []testCase{ + { + name: "BasicDeclare", + config: ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + } + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + test "myModule" { + input = testcomponents.count.inc.count + } + + testcomponents.summation "sum" { + input = test.myModule.output + } + `, + expected: 10, + }, + { + name: "NestedDeclares", + config: ` + declare "test" { + argument "input" { + optional = false + } + + declare "nested" { + argument "input" { + optional = false + } + export "output" { + value = argument.input.value + } + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + nested "default" { + input = testcomponents.passthrough.pt.output + } + + export "output" { + value = nested.default.output + } + } + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + test "myModule" { + input = testcomponents.count.inc.count + } + + testcomponents.summation "sum" { + input = test.myModule.output + } + `, + expected: 10, + }, + { + name: "DeclaredInParentDepth1", + config: ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + rootDeclare "default" { + input = testcomponents.passthrough.pt.output + } + + export "output" { + value = rootDeclare.default.output + } + } + declare "rootDeclare" { + argument "input" { + optional = false + } + export "output" { + value = argument.input.value + } + } + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + test "myModule" { + input = testcomponents.count.inc.count + } + + testcomponents.summation "sum" { + input = test.myModule.output + } + `, + expected: 10, + }, + { + name: "DeclaredInParentDepth2", + config: ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + declare "anotherDeclare" { + argument "input" { + optional = false + } + rootDeclare "default" { + input = argument.input.value + } + export "output" { + value = rootDeclare.default.output + } + } + + anotherDeclare "myOtherDeclare" { + input = testcomponents.passthrough.pt.output + } + + export "output" { + value = anotherDeclare.myOtherDeclare.output + } + } + declare "rootDeclare" { + argument "input" { + optional = false + } + export "output" { + value = argument.input.value + } + } + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + test "myModule" { + input = testcomponents.count.inc.count + } + + testcomponents.summation "sum" { + input = test.myModule.output + } + `, + expected: 10, + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + ctrl := flow.New(testOptions(t)) + f, err := flow.ParseSource(t.Name(), []byte(tc.config)) + require.NoError(t, err) + require.NotNil(t, f) + + err = ctrl.LoadSource(f, nil, nil) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + ctrl.Run(ctx) + close(done) + }() + defer func() { + cancel() + <-done + }() + + require.Eventually(t, func() bool { + export := getExport[testcomponents.SummationExports](t, ctrl, "", "testcomponents.summation.sum") + return export.LastAdded == tc.expected + }, 3*time.Second, 10*time.Millisecond) + }) + } +} diff --git a/pkg/flow/module_fail_test.go b/pkg/flow/module_fail_test.go index 28fb0923a892..5ff28f20c9f0 100644 --- a/pkg/flow/module_fail_test.go +++ b/pkg/flow/module_fail_test.go @@ -15,7 +15,7 @@ func TestIDRemovalIfFailedToLoad(t *testing.T) { fullContent := "test.fail.module \"t1\" { content = \"\" }" fl, err := ParseSource("test", []byte(fullContent)) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, nil) require.NoError(t, err) ctx := context.Background() ctx, cnc := context.WithTimeout(ctx, 600*time.Second) diff --git a/pkg/flow/module_import_test.go b/pkg/flow/module_import_test.go new file mode 100644 index 000000000000..87f182bbac7f --- /dev/null +++ b/pkg/flow/module_import_test.go @@ -0,0 +1,741 @@ +package flow_test + +import ( + "context" + "os" + "testing" + "time" + + "github.com/grafana/agent/pkg/flow" + "github.com/grafana/agent/pkg/flow/internal/testcomponents" + "github.com/stretchr/testify/require" + + _ "github.com/grafana/agent/component/module/string" +) + +func TestImportModule(t *testing.T) { + const defaultModuleUpdate = ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = -10 + } + } +` + testCases := []struct { + name string + module string + otherModule string + config string + updateModule func(filename string) string + updateFile string + }{ + { + name: "TestImportModule", + module: ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + }`, + config: ` + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + import.file "testImport" { + filename = "module" + } + + testImport.test "myModule" { + input = testcomponents.count.inc.count + } + + testcomponents.summation "sum" { + input = testImport.test.myModule.output + } + `, + updateModule: func(filename string) string { + return defaultModuleUpdate + }, + updateFile: "module", + }, + { + name: "TestImportModuleNoArgs", + module: ` + declare "test" { + testcomponents.passthrough "pt" { + input = 10 + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + }`, + config: ` + import.file "testImport" { + filename = "module" + } + + testImport.test "myModule" { + } + + testcomponents.summation "sum" { + input = testImport.test.myModule.output + } + `, + updateModule: func(filename string) string { + return ` + declare "test" { + testcomponents.passthrough "pt" { + input = -10 + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + } + ` + }, + updateFile: "module", + }, + { + name: "TestImportModuleInDeclare", + module: ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + } + `, + config: ` + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + import.file "testImport" { + filename = "module" + } + + declare "anotherModule" { + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + testImport.test "myModule" { + input = testcomponents.count.inc.count + } + + export "output" { + value = testImport.test.myModule.output + } + } + + anotherModule "myOtherModule" {} + + testcomponents.summation "sum" { + input = anotherModule.myOtherModule.output + } + `, + updateModule: func(filename string) string { + return defaultModuleUpdate + }, + updateFile: "module", + }, + { + name: "TestImportModuleInNestedDeclare", + module: ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + } + `, + config: ` + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + import.file "testImport" { + filename = "module" + } + + declare "yetAgainAnotherModule" { + declare "anotherModule" { + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + testImport.test "myModule" { + input = testcomponents.count.inc.count + } + + export "output" { + value = testImport.test.myModule.output + } + } + anotherModule "myOtherModule" {} + + export "output" { + value = anotherModule.myOtherModule.output + } + } + + yetAgainAnotherModule "default" {} + + testcomponents.summation "sum" { + input = yetAgainAnotherModule.default.output + } + `, + updateModule: func(filename string) string { + return defaultModuleUpdate + }, + updateFile: "module", + }, + { + name: "TestImportModuleWithImportBlock", + module: ` + import.file "otherModule" { + filename = "other_module" + } + declare "anotherModule" { + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + otherModule.test "default" { + input = testcomponents.count.inc.count + } + + export "output" { + value = otherModule.test.default.output + } + } + `, + otherModule: ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + } + `, + config: ` + import.file "testImport" { + filename = "module" + } + + testImport.anotherModule "myOtherModule" {} + + testcomponents.summation "sum" { + input = testImport.anotherModule.myOtherModule.output + } + `, + updateModule: func(filename string) string { + return defaultModuleUpdate + }, + updateFile: "other_module", + }, + { + name: "TestImportModuleWithNestedDeclareUsingModule", + module: ` + import.file "default" { + filename = "other_module" + } + declare "anotherModule" { + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + declare "blabla" { + argument "input" {} + default.test "default" { + input = argument.input.value + } + + export "output" { + value = default.test.default.output + } + } + + blabla "default" { + input = testcomponents.count.inc.count + } + + export "output" { + value = blabla.default.output + } + } + `, + otherModule: ` + declare "test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + } + `, + config: ` + import.file "testImport" { + filename = "module" + } + + testImport.anotherModule "myOtherModule" {} + + testcomponents.summation "sum" { + input = testImport.anotherModule.myOtherModule.output + } + `, + }, + { + name: "TestImportModuleWithNestedDeclareDependency", + module: ` + declare "other_test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + } + + declare "test" { + argument "input" { + optional = false + } + + other_test "default" { + input = argument.input.value + } + + export "output" { + value = other_test.default.output + } + } + `, + config: ` + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + import.file "testImport" { + filename = "module" + } + + declare "anotherModule" { + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + testImport.test "myModule" { + input = testcomponents.count.inc.count + } + + export "output" { + value = testImport.test.myModule.output + } + } + + anotherModule "myOtherModule" {} + + testcomponents.summation "sum" { + input = anotherModule.myOtherModule.output + } + `, + updateModule: func(filename string) string { + return ` + declare "other_test" { + argument "input" { + optional = false + } + export "output" { + value = -10 + } + } + + declare "test" { + argument "input" { + optional = false + } + + other_test "default" { + input = argument.input.value + } + + export "output" { + value = other_test.default.output + } + } + ` + }, + updateFile: "module", + }, + { + name: "TestImportModuleWithMoreNesting", + module: ` + import.file "importOtherTest" { + filename = "other_module" + } + declare "test" { + argument "input" { + optional = false + } + + importOtherTest.other_test "default" { + input = argument.input.value + } + + export "output" { + value = importOtherTest.other_test.default.output + } + } + `, + otherModule: ` + declare "other_test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + }`, + config: ` + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + import.file "testImport" { + filename = "module" + } + + declare "anotherModule" { + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + testImport.test "myModule" { + input = testcomponents.count.inc.count + } + + export "output" { + value = testImport.test.myModule.output + } + } + + anotherModule "myOtherModule" {} + + testcomponents.summation "sum" { + input = anotherModule.myOtherModule.output + } + `, + updateModule: func(filename string) string { + return ` + declare "other_test" { + argument "input" { + optional = false + } + export "output" { + value = -10 + } + } + ` + }, + updateFile: "other_module", + }, + { + name: "TestImportModuleWithMoreNestingAndMoreNesting", + module: ` + import.file "importOtherTest" { + filename = "other_module" + } + declare "test" { + argument "input" { + optional = false + } + + declare "anotherOne" { + argument "input" { + optional = false + } + importOtherTest.other_test "default" { + input = argument.input.value + } + export "output" { + value = importOtherTest.other_test.default.output + } + } + + anotherOne "default" { + input = argument.input.value + } + + export "output" { + value = anotherOne.default.output + } + } + `, + otherModule: ` + declare "other_test" { + argument "input" { + optional = false + } + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + }`, + config: ` + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + import.file "testImport" { + filename = "module" + } + + declare "anotherModule" { + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + testImport.test "myModule" { + input = testcomponents.count.inc.count + } + + export "output" { + value = testImport.test.myModule.output + } + } + + anotherModule "myOtherModule" {} + + testcomponents.summation "sum" { + input = anotherModule.myOtherModule.output + } + `, + updateModule: func(filename string) string { + return ` + declare "other_test" { + argument "input" { + optional = false + } + export "output" { + value = -10 + } + } + ` + }, + updateFile: "other_module", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filename := "module" + require.NoError(t, os.WriteFile(filename, []byte(tc.module), 0664)) + defer os.Remove(filename) + + otherFilename := "other_module" + if tc.otherModule != "" { + require.NoError(t, os.WriteFile(otherFilename, []byte(tc.otherModule), 0664)) + defer os.Remove(otherFilename) + } + + ctrl := flow.New(testOptions(t)) + f, err := flow.ParseSource(t.Name(), []byte(tc.config)) + require.NoError(t, err) + require.NotNil(t, f) + + err = ctrl.LoadSource(f, nil, nil) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + ctrl.Run(ctx) + close(done) + }() + defer func() { + cancel() + <-done + }() + + // Check for initial condition + require.Eventually(t, func() bool { + export := getExport[testcomponents.SummationExports](t, ctrl, "", "testcomponents.summation.sum") + return export.LastAdded == 10 + }, 3*time.Second, 10*time.Millisecond) + + // Update module if needed + if tc.updateModule != nil { + newModule := tc.updateModule(tc.updateFile) + require.NoError(t, os.WriteFile(tc.updateFile, []byte(newModule), 0664)) + + require.Eventually(t, func() bool { + export := getExport[testcomponents.SummationExports](t, ctrl, "", "testcomponents.summation.sum") + return export.LastAdded == -10 + }, 3*time.Second, 10*time.Millisecond) + } + }) + } +} + +func TestImportModuleError(t *testing.T) { + testCases := []struct { + name string + module string + otherModule string + config string + expectedError string + }{ + { + name: "TestImportedModuleTriesAccessingDeclareOnRoot", + module: ` + declare "test" { + argument "input" { + optional = false + } + + cantAccessThis "default" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } + }`, + config: ` + declare "cantAccessThis" { + export "output" { + value = -1 + } + } + testcomponents.count "inc" { + frequency = "10ms" + max = 10 + } + + import.file "testImport" { + filename = "module" + } + + testImport.test "myModule" { + input = testcomponents.count.inc.count + } + + testcomponents.summation "sum" { + input = testImport.test.myModule.output + } + `, + expectedError: `Unrecognized component name "cantAccessThis"`, + }, // TODO: add more tests + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filename := "module" + require.NoError(t, os.WriteFile(filename, []byte(tc.module), 0664)) + defer os.Remove(filename) + + otherFilename := "other_module" + if tc.otherModule != "" { + require.NoError(t, os.WriteFile(otherFilename, []byte(tc.otherModule), 0664)) + defer os.Remove(otherFilename) + } + + ctrl := flow.New(testOptions(t)) + f, err := flow.ParseSource(t.Name(), []byte(tc.config)) + require.NoError(t, err) + require.NotNil(t, f) + + err = ctrl.LoadSource(f, nil, nil) + require.ErrorContains(t, err, tc.expectedError) + }) + } +} diff --git a/pkg/flow/module_test.go b/pkg/flow/module_test.go index 4e4ddb9faaa8..2730d77d9932 100644 --- a/pkg/flow/module_test.go +++ b/pkg/flow/module_test.go @@ -144,7 +144,7 @@ func TestArgsNotInModules(t *testing.T) { defer cleanUpController(f) fl, err := ParseSource("test", []byte("argument \"arg\"{}")) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, nil) require.ErrorContains(t, err, "argument blocks only allowed inside a module") } @@ -154,7 +154,7 @@ func TestExportsNotInModules(t *testing.T) { defer cleanUpController(f) fl, err := ParseSource("test", []byte("export \"arg\"{ value = 1}")) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, nil) require.ErrorContains(t, err, "export blocks only allowed inside a module") } @@ -165,7 +165,7 @@ func TestExportsWhenNotUsed(t *testing.T) { fullContent := "test.module \"t1\" { content = \"" + content + "\" }" fl, err := ParseSource("test", []byte(fullContent)) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, nil) require.NoError(t, err) ctx := context.Background() ctx, cnc := context.WithTimeout(ctx, 1*time.Second) @@ -296,7 +296,7 @@ func (t *testModule) Run(ctx context.Context) error { return err } - err = m.LoadConfig([]byte(t.content), t.args) + err = m.LoadConfig([]byte(t.content), t.args, nil) if err != nil { return err } diff --git a/pkg/flow/source.go b/pkg/flow/source.go index acd7d2ce2f58..143ea8998f3e 100644 --- a/pkg/flow/source.go +++ b/pkg/flow/source.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/grafana/agent/pkg/config/encoder" + "github.com/grafana/agent/pkg/flow/internal/controller" "github.com/grafana/river/ast" "github.com/grafana/river/diag" "github.com/grafana/river/parser" @@ -21,6 +22,7 @@ type Source struct { // The Flow controller can interpret them. components []*ast.BlockStmt configBlocks []*ast.BlockStmt + declares []controller.Declare } // ParseSource parses the River file specified by bb into a File. name should be @@ -45,6 +47,7 @@ func ParseSource(name string, bb []byte) (*Source, error) { var ( components []*ast.BlockStmt configs []*ast.BlockStmt + declares []controller.Declare ) for _, stmt := range node.Body { @@ -60,7 +63,9 @@ func ParseSource(name string, bb []byte) (*Source, error) { case *ast.BlockStmt: fullName := strings.Join(stmt.Name, ".") switch fullName { - case "logging", "tracing", "argument", "export": + case "declare": + declares = append(declares, controller.Declare{Block: stmt, Content: string(bb[stmt.LCurlyPos.Position().Offset+1 : stmt.RCurlyPos.Position().Offset-1])}) + case "logging", "tracing", "argument", "export", "import.file", "import.git", "import.http": configs = append(configs, stmt) default: components = append(components, stmt) @@ -79,6 +84,7 @@ func ParseSource(name string, bb []byte) (*Source, error) { return &Source{ components: components, configBlocks: configs, + declares: declares, sourceMap: map[string][]byte{name: bb}, hash: sha256.Sum256(bb), }, nil @@ -120,6 +126,7 @@ func ParseSources(sources map[string][]byte) (*Source, error) { mergedSource.components = append(mergedSource.components, sourceFragment.components...) mergedSource.configBlocks = append(mergedSource.configBlocks, sourceFragment.configBlocks...) + mergedSource.declares = append(mergedSource.declares, sourceFragment.declares...) } mergedSource.hash = [32]byte(hash.Sum(nil)) diff --git a/pkg/flow/source_test.go b/pkg/flow/source_test.go index fa79c8c1e9e1..840642ba203a 100644 --- a/pkg/flow/source_test.go +++ b/pkg/flow/source_test.go @@ -89,7 +89,7 @@ func TestParseSources_DuplicateComponent(t *testing.T) { require.NoError(t, err) ctrl := New(testOptions(t)) defer cleanUpController(ctrl) - err = ctrl.LoadSource(s, nil) + err = ctrl.LoadSource(s, nil, nil) diagErrs, ok := err.(diag.Diagnostics) require.True(t, ok) require.Len(t, diagErrs, 2) @@ -120,7 +120,7 @@ func TestParseSources_UniqueComponent(t *testing.T) { require.NoError(t, err) ctrl := New(testOptions(t)) defer cleanUpController(ctrl) - err = ctrl.LoadSource(s, nil) + err = ctrl.LoadSource(s, nil, nil) require.NoError(t, err) } diff --git a/component/module/git/internal/vcs/auth.go b/pkg/util/git/auth.go similarity index 100% rename from component/module/git/internal/vcs/auth.go rename to pkg/util/git/auth.go diff --git a/component/module/git/internal/vcs/errors.go b/pkg/util/git/errors.go similarity index 100% rename from component/module/git/internal/vcs/errors.go rename to pkg/util/git/errors.go diff --git a/component/module/git/internal/vcs/git.go b/pkg/util/git/git.go similarity index 100% rename from component/module/git/internal/vcs/git.go rename to pkg/util/git/git.go diff --git a/component/module/git/internal/vcs/git_test.go b/pkg/util/git/git_test.go similarity index 97% rename from component/module/git/internal/vcs/git_test.go rename to pkg/util/git/git_test.go index 7680c857db0e..e8c232885f5a 100644 --- a/component/module/git/internal/vcs/git_test.go +++ b/pkg/util/git/git_test.go @@ -6,7 +6,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" - "github.com/grafana/agent/component/module/git/internal/vcs" + vcs "github.com/grafana/agent/pkg/util/git" "github.com/stretchr/testify/require" ) From aca2289a1acf16497e92fe753e95cc8b9089d84d Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 15 Jan 2024 11:42:14 +0100 Subject: [PATCH 02/36] minor changes following review --- component/module/git/git.go | 2 +- {pkg/util/git => internal/vcs}/auth.go | 0 {pkg/util/git => internal/vcs}/errors.go | 0 {pkg/util/git => internal/vcs}/git.go | 0 {pkg/util/git => internal/vcs}/git_test.go | 2 +- pkg/flow/flow.go | 6 +- pkg/flow/flow_components.go | 14 ++-- .../internal/controller/component_node.go | 1 + pkg/flow/internal/controller/declare.go | 2 +- pkg/flow/internal/controller/loader.go | 5 +- pkg/flow/internal/controller/loader_test.go | 20 +++--- pkg/flow/internal/controller/node_config.go | 2 +- .../internal/controller/node_config_import.go | 60 ++++++++--------- .../controller/node_declare_component.go | 44 ++++++------- .../controller/node_native_component.go | 66 +++++++++---------- .../import_file.go | 0 .../import_git.go | 2 +- .../import_http.go | 0 .../import_source.go | 18 ++--- 19 files changed, 121 insertions(+), 123 deletions(-) rename {pkg/util/git => internal/vcs}/auth.go (100%) rename {pkg/util/git => internal/vcs}/errors.go (100%) rename {pkg/util/git => internal/vcs}/git.go (100%) rename {pkg/util/git => internal/vcs}/git_test.go (98%) rename pkg/flow/internal/{import-source => importsource}/import_file.go (100%) rename pkg/flow/internal/{import-source => importsource}/import_git.go (99%) rename pkg/flow/internal/{import-source => importsource}/import_http.go (100%) rename pkg/flow/internal/{import-source => importsource}/import_source.go (91%) diff --git a/component/module/git/git.go b/component/module/git/git.go index a7d7c5f27dd9..b30569b67bb5 100644 --- a/component/module/git/git.go +++ b/component/module/git/git.go @@ -12,8 +12,8 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/component" "github.com/grafana/agent/component/module" + "github.com/grafana/agent/internal/vcs" "github.com/grafana/agent/pkg/flow/logging/level" - vcs "github.com/grafana/agent/pkg/util/git" ) func init() { diff --git a/pkg/util/git/auth.go b/internal/vcs/auth.go similarity index 100% rename from pkg/util/git/auth.go rename to internal/vcs/auth.go diff --git a/pkg/util/git/errors.go b/internal/vcs/errors.go similarity index 100% rename from pkg/util/git/errors.go rename to internal/vcs/errors.go diff --git a/pkg/util/git/git.go b/internal/vcs/git.go similarity index 100% rename from pkg/util/git/git.go rename to internal/vcs/git.go diff --git a/pkg/util/git/git_test.go b/internal/vcs/git_test.go similarity index 98% rename from pkg/util/git/git_test.go rename to internal/vcs/git_test.go index e8c232885f5a..a7614eb9507f 100644 --- a/pkg/util/git/git_test.go +++ b/internal/vcs/git_test.go @@ -6,7 +6,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" - vcs "github.com/grafana/agent/pkg/util/git" + "github.com/grafana/agent/internal/vcs" "github.com/stretchr/testify/require" ) diff --git a/pkg/flow/flow.go b/pkg/flow/flow.go index 8d0d776fcd6f..963f2eebb824 100644 --- a/pkg/flow/flow.go +++ b/pkg/flow/flow.go @@ -185,8 +185,8 @@ func newController(o controllerOptions) *Flow { Logger: log, TraceProvider: tracer, DataPath: o.DataPath, - OnComponentUpdate: func(cn controller.NodeWithDependants) { - // Changed components should be queued for reevaluation. + OnNodeWithDependantsUpdate: func(cn controller.NodeWithDependants) { + // Changed node with dependants should be queued for reevaluation. f.updateQueue.Enqueue(cn) }, OnExportsChange: o.OnExportsChange, @@ -248,7 +248,7 @@ func (f *Flow) Run(ctx context.Context) { components = f.loader.Components() services = f.loader.Services() imports = f.loader.Imports() - declareComponents = f.loader.DeclareComponent() + declareComponents = f.loader.DeclareComponents() runnables = make([]controller.RunnableNode, 0, len(components)+len(services)+len(imports)+len(declareComponents)) ) diff --git a/pkg/flow/flow_components.go b/pkg/flow/flow_components.go index e4f4c7734b0a..33995d02c603 100644 --- a/pkg/flow/flow_components.go +++ b/pkg/flow/flow_components.go @@ -54,23 +54,19 @@ func (f *Flow) ListComponents(moduleID string, opts component.InfoOptions) ([]*c var ( components = f.loader.Components() imports = f.loader.Imports() - declareComponents = f.loader.DeclareComponent() + declareComponents = f.loader.DeclareComponents() graph = f.loader.OriginalGraph() ) - detail := make([]*component.Info, len(components)+len(imports)+len(declareComponents)) - idx := 0 + detail := make([]*component.Info, 0, len(components)+len(imports)+len(declareComponents)) for _, component := range components { - detail[idx] = f.getComponentDetail(component, graph, opts) - idx++ + detail = append(detail, f.getComponentDetail(component, graph, opts)) } for _, importNode := range imports { - detail[idx] = f.getComponentDetail(importNode, graph, opts) - idx++ + detail = append(detail, f.getComponentDetail(importNode, graph, opts)) } for _, declareComponent := range declareComponents { - detail[idx] = f.getComponentDetail(declareComponent, graph, opts) - idx++ + detail = append(detail, f.getComponentDetail(declareComponent, graph, opts)) } return detail, nil } diff --git a/pkg/flow/internal/controller/component_node.go b/pkg/flow/internal/controller/component_node.go index 269e313b99fa..cd7d3050483a 100644 --- a/pkg/flow/internal/controller/component_node.go +++ b/pkg/flow/internal/controller/component_node.go @@ -5,6 +5,7 @@ import ( "github.com/grafana/agent/pkg/flow/internal/dag" ) +// ComponentNode is a dag.Node that manages a component. type ComponentNode interface { dag.Node diff --git a/pkg/flow/internal/controller/declare.go b/pkg/flow/internal/controller/declare.go index e97554396c32..657f8eb974fc 100644 --- a/pkg/flow/internal/controller/declare.go +++ b/pkg/flow/internal/controller/declare.go @@ -2,7 +2,7 @@ package controller import "github.com/grafana/river/ast" -// Should this be defined somewhere else? +// Declare represents the content of a declare block as AST and as plain string. type Declare struct { Block *ast.BlockStmt Content string diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 2c4ddb098b34..c640341c73ca 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -282,6 +282,7 @@ func (l *Loader) Cleanup(stopWorkerPool bool) { func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []Declare) (dag.Graph, diag.Diagnostics) { var g dag.Graph + // Reset the module cache before loading a new graph. This step is crucial to ensure that references to outdated modules, not included in the new configuration, are removed. l.moduleReferences = make(map[string][]ModuleReference) // Split component blocks into blocks for components and services. @@ -525,7 +526,7 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo }) continue } - g.Add(NewComponentNode(l.globals, registration, block)) + g.Add(NewNativeComponentNode(l.globals, registration, block)) } } } @@ -634,7 +635,7 @@ func (l *Loader) Components() []*NativeComponentNode { return l.componentNodes } -func (l *Loader) DeclareComponent() []*DeclareComponentNode { +func (l *Loader) DeclareComponents() []*DeclareComponentNode { l.mut.RLock() defer l.mut.RUnlock() return l.declareComponentNodes diff --git a/pkg/flow/internal/controller/loader_test.go b/pkg/flow/internal/controller/loader_test.go index 9351dda73657..c25ea23c3091 100644 --- a/pkg/flow/internal/controller/loader_test.go +++ b/pkg/flow/internal/controller/loader_test.go @@ -70,11 +70,11 @@ func TestLoader(t *testing.T) { l, _ := logging.New(os.Stderr, logging.DefaultOptions) return controller.LoaderOptions{ ComponentGlobals: controller.ComponentGlobals{ - Logger: l, - TraceProvider: noop.NewTracerProvider(), - DataPath: t.TempDir(), - OnComponentUpdate: func(cn controller.NodeWithDependants) { /* no-op */ }, - Registerer: prometheus.NewRegistry(), + Logger: l, + TraceProvider: noop.NewTracerProvider(), + DataPath: t.TempDir(), + OnNodeWithDependantsUpdate: func(cn controller.NodeWithDependants) { /* no-op */ }, + Registerer: prometheus.NewRegistry(), NewModuleController: func(id string) controller.ModuleController { return nil }, @@ -204,11 +204,11 @@ func TestScopeWithFailingComponent(t *testing.T) { l, _ := logging.New(os.Stderr, logging.DefaultOptions) return controller.LoaderOptions{ ComponentGlobals: controller.ComponentGlobals{ - Logger: l, - TraceProvider: noop.NewTracerProvider(), - DataPath: t.TempDir(), - OnComponentUpdate: func(cn controller.NodeWithDependants) { /* no-op */ }, - Registerer: prometheus.NewRegistry(), + Logger: l, + TraceProvider: noop.NewTracerProvider(), + DataPath: t.TempDir(), + OnNodeWithDependantsUpdate: func(cn controller.NodeWithDependants) { /* no-op */ }, + Registerer: prometheus.NewRegistry(), NewModuleController: func(id string) controller.ModuleController { return fakeModuleController{} }, diff --git a/pkg/flow/internal/controller/node_config.go b/pkg/flow/internal/controller/node_config.go index 7b84067cea4c..0b679f074109 100644 --- a/pkg/flow/internal/controller/node_config.go +++ b/pkg/flow/internal/controller/node_config.go @@ -3,7 +3,7 @@ package controller import ( "fmt" - importsource "github.com/grafana/agent/pkg/flow/internal/import-source" + "github.com/grafana/agent/pkg/flow/internal/importsource" "github.com/grafana/river/ast" "github.com/grafana/river/diag" ) diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 856e1cb1bdf8..432b4c8660b0 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -11,7 +11,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/component" - importsource "github.com/grafana/agent/pkg/flow/internal/import-source" + "github.com/grafana/agent/pkg/flow/internal/importsource" "github.com/grafana/agent/pkg/flow/logging/level" "github.com/grafana/agent/pkg/flow/tracing" "github.com/grafana/river/ast" @@ -22,19 +22,19 @@ import ( ) type ImportConfigNode struct { - id ComponentID - label string - nodeID string - componentName string - globalID string - globals ComponentGlobals // Need a copy of the globals to create other import nodes. - source importsource.ImportSource - registry *prometheus.Registry - importedDeclares map[string]string - importConfigNodesChildren map[string]*ImportConfigNode - OnComponentUpdate func(cn NodeWithDependants) - logger log.Logger - inContentUpdate bool + id ComponentID + label string + nodeID string + componentName string + globalID string + globals ComponentGlobals // Need a copy of the globals to create other import nodes. + source importsource.ImportSource + registry *prometheus.Registry + importedDeclares map[string]string + importConfigNodesChildren map[string]*ImportConfigNode + OnNodeWithDependantsUpdate func(cn NodeWithDependants) + logger log.Logger + inContentUpdate bool mut sync.RWMutex importedContentMut sync.RWMutex @@ -68,17 +68,17 @@ func NewImportConfigNode(block *ast.BlockStmt, globals ComponentGlobals, sourceT globalID = path.Join(globals.ControllerID, nodeID) } cn := &ImportConfigNode{ - id: id, - globalID: globalID, - label: block.Label, - globals: globals, - nodeID: BlockComponentID(block).String(), - componentName: block.GetBlockName(), - importedDeclares: make(map[string]string), - OnComponentUpdate: globals.OnComponentUpdate, - block: block, - evalHealth: initHealth, - runHealth: initHealth, + id: id, + globalID: globalID, + label: block.Label, + globals: globals, + nodeID: BlockComponentID(block).String(), + componentName: block.GetBlockName(), + importedDeclares: make(map[string]string), + OnNodeWithDependantsUpdate: globals.OnNodeWithDependantsUpdate, + block: block, + evalHealth: initHealth, + runHealth: initHealth, } managedOpts := getImportManagedOptions(globals, cn) cn.logger = managedOpts.Logger @@ -177,8 +177,8 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str return } childGlobals := cn.globals - // Children have a special OnComponentUpdate function which will surface all the imported declares to the root import config node. - childGlobals.OnComponentUpdate = cn.OnChildrenContentUpdate + // Children have a special OnNodeWithDependantsUpdate function which will surface all the imported declares to the root import config node. + childGlobals.OnNodeWithDependantsUpdate = cn.OnChildrenContentUpdate cn.importConfigNodesChildren[stmt.Label] = NewImportConfigNode(stmt, childGlobals, sourceType) } @@ -205,7 +205,7 @@ func (cn *ImportConfigNode) onContentUpdate(content string) { return } cn.lastUpdateTime.Store(time.Now()) - cn.OnComponentUpdate(cn) + cn.OnNodeWithDependantsUpdate(cn) } // evaluateChildren evaluates the import nodes managed by this import node. @@ -255,9 +255,9 @@ func (cn *ImportConfigNode) OnChildrenContentUpdate(child NodeWithDependants) { cn.importedDeclares[label] = content } } - // This avoids OnComponentUpdate to be called multiple times in a row when the content changes. + // This avoids OnNodeWithDependantsUpdate to be called multiple times in a row when the content changes. if !cn.inContentUpdate { - cn.OnComponentUpdate(cn) + cn.OnNodeWithDependantsUpdate(cn) } } diff --git a/pkg/flow/internal/controller/node_declare_component.go b/pkg/flow/internal/controller/node_declare_component.go index b389d2c8989e..6bd459452e88 100644 --- a/pkg/flow/internal/controller/node_declare_component.go +++ b/pkg/flow/internal/controller/node_declare_component.go @@ -26,17 +26,17 @@ import ( // DeclareComponentNode manages the underlying module and caches its current // arguments and exports. type DeclareComponentNode struct { - id ComponentID - globalID string - label string - componentName string - importLabel string - declareLabel string - nodeID string // Cached from id.String() to avoid allocating new strings every time NodeID is called. - managedOpts component.Options - registry *prometheus.Registry - moduleController ModuleController - OnComponentUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate + id ComponentID + globalID string + label string + componentName string + importLabel string + declareLabel string + nodeID string // Cached from id.String() to avoid allocating new strings every time NodeID is called. + managedOpts component.Options + registry *prometheus.Registry + moduleController ModuleController + OnNodeWithDependantsUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate GetModuleInfo func(fullName string, importLabel string, declareLabel string) (ModuleInfo, error) // Retrieve the module config. lastUpdateTime atomic.Time @@ -108,16 +108,16 @@ func NewDeclareComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetModu importLabel, declareLabel := ExtractImportAndDeclareLabels(componentName) cn := &DeclareComponentNode{ - id: id, - globalID: globalID, - label: b.Label, - nodeID: nodeID, - componentName: componentName, - importLabel: importLabel, - declareLabel: declareLabel, - moduleController: globals.NewModuleController(globalID), - OnComponentUpdate: globals.OnComponentUpdate, - GetModuleInfo: GetModuleInfo, + id: id, + globalID: globalID, + label: b.Label, + nodeID: nodeID, + componentName: componentName, + importLabel: importLabel, + declareLabel: declareLabel, + moduleController: globals.NewModuleController(globalID), + OnNodeWithDependantsUpdate: globals.OnNodeWithDependantsUpdate, + GetModuleInfo: GetModuleInfo, block: b, eval: vm.New(b.Body), @@ -294,7 +294,7 @@ func (cn *DeclareComponentNode) setExports(e component.Exports) { if changed { // Inform the controller that we have new exports. cn.lastUpdateTime.Store(time.Now()) - cn.OnComponentUpdate(cn) + cn.OnNodeWithDependantsUpdate(cn) } } diff --git a/pkg/flow/internal/controller/node_native_component.go b/pkg/flow/internal/controller/node_native_component.go index ef14722caf78..f7ff2c9592ca 100644 --- a/pkg/flow/internal/controller/node_native_component.go +++ b/pkg/flow/internal/controller/node_native_component.go @@ -63,15 +63,15 @@ type DialFunc func(ctx context.Context, network, address string) (net.Conn, erro // ComponentGlobals are used by ComponentNodes to build managed components. All // ComponentNodes should use the same ComponentGlobals. type ComponentGlobals struct { - Logger *logging.Logger // Logger shared between all managed components. - TraceProvider trace.TracerProvider // Tracer shared between all managed components. - DataPath string // Shared directory where component data may be stored - OnComponentUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate - OnExportsChange func(exports map[string]any) // Invoked when the managed component updated its exports - Registerer prometheus.Registerer // Registerer for serving agent and component metrics - ControllerID string // ID of controller. - NewModuleController func(id string) ModuleController // Func to generate a module controller. - GetServiceData func(name string) (interface{}, error) // Get data for a service. + Logger *logging.Logger // Logger shared between all managed components. + TraceProvider trace.TracerProvider // Tracer shared between all managed components. + DataPath string // Shared directory where component data may be stored + OnNodeWithDependantsUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate + OnExportsChange func(exports map[string]any) // Invoked when the managed component updated its exports + Registerer prometheus.Registerer // Registerer for serving agent and component metrics + ControllerID string // ID of controller. + NewModuleController func(id string) ModuleController // Func to generate a module controller. + GetServiceData func(name string) (interface{}, error) // Get data for a service. } // NativeComponentNode is a controller node which manages a user-defined component. @@ -80,18 +80,18 @@ type ComponentGlobals struct { // arguments and exports. NativeComponentNode manages the arguments for the component // from a River block. type NativeComponentNode struct { - id ComponentID - globalID string - label string - componentName string - nodeID string // Cached from id.String() to avoid allocating new strings every time NodeID is called. - reg component.Registration - managedOpts component.Options - registry *prometheus.Registry - exportsType reflect.Type - moduleController ModuleController - OnComponentUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate - lastUpdateTime atomic.Time + id ComponentID + globalID string + label string + componentName string + nodeID string // Cached from id.String() to avoid allocating new strings every time NodeID is called. + reg component.Registration + managedOpts component.Options + registry *prometheus.Registry + exportsType reflect.Type + moduleController ModuleController + OnNodeWithDependantsUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate + lastUpdateTime atomic.Time mut sync.RWMutex block *ast.BlockStmt // Current River block to derive args from @@ -114,9 +114,9 @@ type NativeComponentNode struct { var _ NodeWithDependants = (*NativeComponentNode)(nil) var _ ComponentNode = (*NativeComponentNode)(nil) -// NewComponentNode creates a new ComponentNode from an initial ast.BlockStmt. +// NewNativeComponentNode creates a new NewNativeComponentNode from an initial ast.BlockStmt. // The underlying managed component isn't created until Evaluate is called. -func NewComponentNode(globals ComponentGlobals, reg component.Registration, b *ast.BlockStmt) *NativeComponentNode { +func NewNativeComponentNode(globals ComponentGlobals, reg component.Registration, b *ast.BlockStmt) *NativeComponentNode { var ( id = BlockComponentID(b) nodeID = id.String() @@ -139,15 +139,15 @@ func NewComponentNode(globals ComponentGlobals, reg component.Registration, b *a } cn := &NativeComponentNode{ - id: id, - globalID: globalID, - label: b.Label, - nodeID: nodeID, - componentName: b.GetBlockName(), - reg: reg, - exportsType: getExportsType(reg), - moduleController: globals.NewModuleController(globalID), - OnComponentUpdate: globals.OnComponentUpdate, + id: id, + globalID: globalID, + label: b.Label, + nodeID: nodeID, + componentName: b.GetBlockName(), + reg: reg, + exportsType: getExportsType(reg), + moduleController: globals.NewModuleController(globalID), + OnNodeWithDependantsUpdate: globals.OnNodeWithDependantsUpdate, block: b, eval: vm.New(b.Body), @@ -385,7 +385,7 @@ func (cn *NativeComponentNode) setExports(e component.Exports) { if changed { // Inform the controller that we have new exports. cn.lastUpdateTime.Store(time.Now()) - cn.OnComponentUpdate(cn) + cn.OnNodeWithDependantsUpdate(cn) } } diff --git a/pkg/flow/internal/import-source/import_file.go b/pkg/flow/internal/importsource/import_file.go similarity index 100% rename from pkg/flow/internal/import-source/import_file.go rename to pkg/flow/internal/importsource/import_file.go diff --git a/pkg/flow/internal/import-source/import_git.go b/pkg/flow/internal/importsource/import_git.go similarity index 99% rename from pkg/flow/internal/import-source/import_git.go rename to pkg/flow/internal/importsource/import_git.go index b889f3a9d157..236089f9ff59 100644 --- a/pkg/flow/internal/import-source/import_git.go +++ b/pkg/flow/internal/importsource/import_git.go @@ -12,8 +12,8 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/component" + "github.com/grafana/agent/internal/vcs" "github.com/grafana/agent/pkg/flow/logging/level" - vcs "github.com/grafana/agent/pkg/util/git" "github.com/grafana/river/vm" ) diff --git a/pkg/flow/internal/import-source/import_http.go b/pkg/flow/internal/importsource/import_http.go similarity index 100% rename from pkg/flow/internal/import-source/import_http.go rename to pkg/flow/internal/importsource/import_http.go diff --git a/pkg/flow/internal/import-source/import_source.go b/pkg/flow/internal/importsource/import_source.go similarity index 91% rename from pkg/flow/internal/import-source/import_source.go rename to pkg/flow/internal/importsource/import_source.go index 1467234c1d7e..4841c12a7e61 100644 --- a/pkg/flow/internal/import-source/import_source.go +++ b/pkg/flow/internal/importsource/import_source.go @@ -11,9 +11,9 @@ import ( type SourceType int const ( - FILE SourceType = iota - HTTP - GIT + File SourceType = iota + Http + Git ) const ( @@ -33,11 +33,11 @@ type ImportSource interface { func NewImportSource(sourceType SourceType, managedOpts component.Options, eval *vm.Evaluator, onContentChange func(string)) ImportSource { switch sourceType { - case FILE: + case File: return NewImportFile(managedOpts, eval, onContentChange) - case HTTP: + case Http: return NewImportHTTP(managedOpts, eval, onContentChange) - case GIT: + case Git: return NewImportGit(managedOpts, eval, onContentChange) } // This is a programming error, not a config error so this is ok to panic. @@ -47,11 +47,11 @@ func NewImportSource(sourceType SourceType, managedOpts component.Options, eval func GetSourceType(fullName string) SourceType { switch fullName { case BlockImportFile: - return FILE + return File case BlockImportGit: - return GIT + return Git case BlockImportHTTP: - return HTTP + return Http } // This is a programming error, not a config error so this is ok to panic. panic(fmt.Errorf("name does not map to a know source type: %v", fullName)) From 8f50ca1839f9e806aa99f30610711ab3a84cf070 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Mon, 15 Jan 2024 14:32:26 +0100 Subject: [PATCH 03/36] use Declare type instead of plain string to use the AST instead of parsing the content several times --- pkg/flow/internal/controller/declare.go | 5 +++ pkg/flow/internal/controller/loader.go | 10 +++--- pkg/flow/internal/controller/loader_test.go | 2 +- pkg/flow/internal/controller/module_info.go | 34 ++++++++---------- .../internal/controller/module_references.go | 30 +++++----------- .../internal/controller/node_config_import.go | 35 +++++++++++++------ pkg/flow/internal/controller/node_declare.go | 24 ++++++------- pkg/flow/source.go | 6 ++-- 8 files changed, 71 insertions(+), 75 deletions(-) diff --git a/pkg/flow/internal/controller/declare.go b/pkg/flow/internal/controller/declare.go index 657f8eb974fc..342ba4ef1a33 100644 --- a/pkg/flow/internal/controller/declare.go +++ b/pkg/flow/internal/controller/declare.go @@ -7,3 +7,8 @@ type Declare struct { Block *ast.BlockStmt Content string } + +// NewDeclare creates a new Declare from its AST and its plain string content. +func NewDeclare(block *ast.BlockStmt, content string) *Declare { + return &Declare{Block: block, Content: content} +} diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index c640341c73ca..7c527d1d98cf 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -127,7 +127,7 @@ func NewLoader(opts LoaderOptions) *Loader { // The provided parentContext can be used to provide global variables and // functions to components. A child context will be constructed from the parent // to expose values of other components. -func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []Declare, parentModuleDefinitions map[string]string) diag.Diagnostics { +func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare, parentModuleDefinitions map[string]string) diag.Diagnostics { start := time.Now() l.mut.Lock() defer l.mut.Unlock() @@ -279,7 +279,7 @@ func (l *Loader) Cleanup(stopWorkerPool bool) { } // loadNewGraph creates a new graph from the provided blocks and validates it. -func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []Declare) (dag.Graph, diag.Diagnostics) { +func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare) (dag.Graph, diag.Diagnostics) { var g dag.Graph // Reset the module cache before loading a new graph. This step is crucial to ensure that references to outdated modules, not included in the new configuration, are removed. @@ -344,11 +344,11 @@ func (l *Loader) splitComponentBlocks(blocks []*ast.BlockStmt) (componentBlocks, return componentBlocks, serviceBlocks } -func (l *Loader) populateDeclareNodes(g *dag.Graph, declares []Declare) diag.Diagnostics { +func (l *Loader) populateDeclareNodes(g *dag.Graph, declares []*Declare) diag.Diagnostics { var diags diag.Diagnostics l.declareNodes = map[string]*DeclareNode{} for _, declare := range declares { - node := NewDeclareNode(declare.Block, declare.Content) + node := NewDeclareNode(declare) if g.GetByID(node.NodeID()) != nil { diags.Add(diag.Diagnostic{ Severity: diag.SeverityLevelError, @@ -548,7 +548,7 @@ func (l *Loader) wireModuleReferences(g *dag.Graph, dc *DeclareComponentNode, de references = deps } else { var err error - references, err = GetModuleReferences(declareNode.content, l.importNodes, l.declareNodes, l.parentModuleDefinitions) + references, err = GetModuleReferences(declareNode.Declare(), l.importNodes, l.declareNodes, l.parentModuleDefinitions) if err != nil { return err } diff --git a/pkg/flow/internal/controller/loader_test.go b/pkg/flow/internal/controller/loader_test.go index c25ea23c3091..cac91b7926bb 100644 --- a/pkg/flow/internal/controller/loader_test.go +++ b/pkg/flow/internal/controller/loader_test.go @@ -230,7 +230,7 @@ func applyFromContent(t *testing.T, l *controller.Loader, componentBytes []byte, diags diag.Diagnostics componentBlocks []*ast.BlockStmt configBlocks []*ast.BlockStmt = nil - declares []controller.Declare + declares []*controller.Declare ) componentBlocks, diags = fileToBlock(t, componentBytes) diff --git a/pkg/flow/internal/controller/module_info.go b/pkg/flow/internal/controller/module_info.go index 7273a3cce685..9abf23e791b6 100644 --- a/pkg/flow/internal/controller/module_info.go +++ b/pkg/flow/internal/controller/module_info.go @@ -27,11 +27,7 @@ func getLocalModuleInfo( if err != nil { return moduleInfo, err } - - content, err = node.ModuleContent() - if err != nil { - return moduleInfo, err - } + content = node.Declare().Content } else if c, ok := parentModuleDefinitions[componentName]; ok { content = c moduleInfo.moduleDefinitions = parentModuleDefinitions @@ -47,28 +43,25 @@ func getLocalModuleDefinitions(componentName string, parentModuleDefinitions map[string]string, ) (map[string]string, error) { - moduleReferences := make(map[string]string) + moduleDefinitions := make(map[string]string) for _, moduleDependency := range localModuleReferences[componentName] { if moduleDependency.importNode != nil { - for importModulePath, importModuleContent := range moduleDependency.importNode.importedDeclares { - moduleReferences[moduleDependency.importNode.label+"."+importModulePath] = importModuleContent + for importModulePath, importModuleDeclare := range moduleDependency.importNode.ImportedDeclares() { + moduleDefinitions[moduleDependency.importNode.label+"."+importModulePath] = importModuleDeclare.Content } } else if moduleDependency.declareNode != nil { - ref, err := moduleDependency.declareNode.ModuleContent() - if err != nil { - return moduleReferences, nil - } - moduleReferences[moduleDependency.declareLabel] = ref + def := moduleDependency.declareNode.Declare().Content + moduleDefinitions[moduleDependency.declareLabel] = def } else { // Nested declares have access to their parents module definitions. if c, ok := parentModuleDefinitions[moduleDependency.componentName]; ok { - moduleReferences[moduleDependency.componentName] = c + moduleDefinitions[moduleDependency.componentName] = c } else { - return moduleReferences, fmt.Errorf("could not find the required module dependency %s for the module %s", moduleDependency.componentName, componentName) + return moduleDefinitions, fmt.Errorf("could not find the required module dependency %s for the module %s", moduleDependency.componentName, componentName) } } } - return moduleReferences, nil + return moduleDefinitions, nil } func getImportedModuleInfo( @@ -81,13 +74,16 @@ func getImportedModuleInfo( var moduleInfo ModuleInfo var content string - var err error if node, exists := importNodes[importLabel]; exists { - moduleInfo.moduleDefinitions = node.importedDeclares - content, err = node.ModuleContent(declareLabel) + moduleInfo.moduleDefinitions = make(map[string]string, len(node.ImportedDeclares())) + for importDeclarePath, importedDeclare := range node.ImportedDeclares() { + moduleInfo.moduleDefinitions[importDeclarePath] = importedDeclare.Content + } + declare, err := node.GetImportedDeclareByLabel(declareLabel) if err != nil { return moduleInfo, err } + content = declare.Content } else if c, ok := parentModuleDefinitions[componentName]; ok { content = c moduleInfo.moduleDefinitions = filterParentModuleDefinitions(importLabel, parentModuleDefinitions) diff --git a/pkg/flow/internal/controller/module_references.go b/pkg/flow/internal/controller/module_references.go index 2d975be9d169..34382fefecab 100644 --- a/pkg/flow/internal/controller/module_references.go +++ b/pkg/flow/internal/controller/module_references.go @@ -4,7 +4,6 @@ import ( "strings" "github.com/grafana/river/ast" - "github.com/grafana/river/parser" ) type ModuleReference struct { @@ -15,19 +14,17 @@ type ModuleReference struct { declareNode *DeclareNode } -// This function will parse the provided river content and collect references to known modules. +// GetModuleReferences traverses the AST of the provided declare and collects references to known modules. +// Panics if declare is nil. func GetModuleReferences( - content string, + declare *Declare, importNodes map[string]*ImportConfigNode, declareNodes map[string]*DeclareNode, parentModuleDefinitions map[string]string, ) ([]ModuleReference, error) { uniqueReferences := make(map[string]ModuleReference) - err := getModuleReferences(content, importNodes, declareNodes, uniqueReferences, parentModuleDefinitions) - if err != nil { - return nil, err - } + getModuleReferences(declare.Block.Body, importNodes, declareNodes, uniqueReferences, parentModuleDefinitions) references := make([]ModuleReference, 0, len(uniqueReferences)) for _, ref := range uniqueReferences { @@ -38,29 +35,19 @@ func GetModuleReferences( } func getModuleReferences( - content string, + stmts ast.Body, importNodes map[string]*ImportConfigNode, declareNodes map[string]*DeclareNode, uniqueReferences map[string]ModuleReference, parentModuleDefinitions map[string]string, -) error { - - node, err := parser.ParseFile("", []byte(content)) - if err != nil { - return err - } - - for _, stmt := range node.Body { +) { + for _, stmt := range stmts { switch stmt := stmt.(type) { case *ast.BlockStmt: componentName := strings.Join(stmt.Name, ".") switch componentName { case "declare": - declareContent := content[stmt.LCurlyPos.Position().Offset+1 : stmt.RCurlyPos.Position().Offset-1] - err = getModuleReferences(declareContent, importNodes, declareNodes, uniqueReferences, parentModuleDefinitions) - if err != nil { - return err - } + getModuleReferences(stmt.Body, importNodes, declareNodes, uniqueReferences, parentModuleDefinitions) default: potentialImportLabel, potentialDeclareLabel := ExtractImportAndDeclareLabels(componentName) if declareNode, ok := declareNodes[potentialDeclareLabel]; ok { @@ -73,5 +60,4 @@ func getModuleReferences( } } } - return nil } diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 432b4c8660b0..89dc627cdf2f 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -30,7 +30,7 @@ type ImportConfigNode struct { globals ComponentGlobals // Need a copy of the globals to create other import nodes. source importsource.ImportSource registry *prometheus.Registry - importedDeclares map[string]string + importedDeclares map[string]*Declare importConfigNodesChildren map[string]*ImportConfigNode OnNodeWithDependantsUpdate func(cn NodeWithDependants) logger log.Logger @@ -74,7 +74,6 @@ func NewImportConfigNode(block *ast.BlockStmt, globals ComponentGlobals, sourceT globals: globals, nodeID: BlockComponentID(block).String(), componentName: block.GetBlockName(), - importedDeclares: make(map[string]string), OnNodeWithDependantsUpdate: globals.OnNodeWithDependantsUpdate, block: block, evalHealth: initHealth, @@ -166,7 +165,7 @@ func (cn *ImportConfigNode) processDeclareBlock(stmt *ast.BlockStmt, content str level.Error(cn.logger).Log("msg", "declare block redefined", "name", stmt.Label) return } - cn.importedDeclares[stmt.Label] = content[stmt.LCurlyPos.Position().Offset+1 : stmt.RCurlyPos.Position().Offset-1] + cn.importedDeclares[stmt.Label] = NewDeclare(stmt, content[stmt.LCurlyPos.Position().Offset+1:stmt.RCurlyPos.Position().Offset-1]) } // processDeclareBlock processes an import block. @@ -190,7 +189,7 @@ func (cn *ImportConfigNode) onContentUpdate(content string) { defer func() { cn.inContentUpdate = false }() - cn.importedDeclares = make(map[string]string) + cn.importedDeclares = make(map[string]*Declare) // We recreate the nodes when the content changes. Can we copy instead for optimization? cn.importConfigNodesChildren = make(map[string]*ImportConfigNode) node, err := parser.ParseFile(cn.label, []byte(content)) @@ -250,9 +249,9 @@ func (cn *ImportConfigNode) runChildren(ctx context.Context) error { func (cn *ImportConfigNode) OnChildrenContentUpdate(child NodeWithDependants) { switch child := child.(type) { case *ImportConfigNode: - for importedDeclareLabel, content := range child.importedDeclares { + for importedDeclareLabel, importedDeclare := range child.importedDeclares { label := child.label + "." + importedDeclareLabel - cn.importedDeclares[label] = content + cn.importedDeclares[label] = importedDeclare } } // This avoids OnNodeWithDependantsUpdate to be called multiple times in a row when the content changes. @@ -261,14 +260,14 @@ func (cn *ImportConfigNode) OnChildrenContentUpdate(child NodeWithDependants) { } } -// ModuleContent returns the content of a declare block imported by the node. -func (cn *ImportConfigNode) ModuleContent(declareLabel string) (string, error) { +// GetImportedDeclareByLabel returns a declare block imported by the node. +func (cn *ImportConfigNode) GetImportedDeclareByLabel(declareLabel string) (*Declare, error) { cn.importedContentMut.Lock() defer cn.importedContentMut.Unlock() - if content, ok := cn.importedDeclares[declareLabel]; ok { - return content, nil + if declare, ok := cn.importedDeclares[declareLabel]; ok { + return declare, nil } - return "", fmt.Errorf("declareLabel %s not found in imported node %s", declareLabel, cn.label) + return nil, fmt.Errorf("declareLabel %s not found in imported node %s", declareLabel, cn.label) } // Run runs the managed component in the calling goroutine until ctx is @@ -367,6 +366,20 @@ func (cn *ImportConfigNode) Component() component.Component { return cn.source.Component() } +// ImportedDeclares returns all declare blocks that it imported. +func (cn *ImportConfigNode) ImportedDeclares() map[string]*Declare { + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.importedDeclares +} + +// ImportConfigNodesChildren returns the ImportConfigNodesChildren of this ImportConfigNode. +func (cn *ImportConfigNode) ImportConfigNodesChildren() map[string]*ImportConfigNode { + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.importConfigNodesChildren +} + // CurrentHealth returns the current health of the ComponentNode. // // The health of a ComponentNode is determined by combining: diff --git a/pkg/flow/internal/controller/node_declare.go b/pkg/flow/internal/controller/node_declare.go index ec9ff4596f83..d9e8ce6f5a0b 100644 --- a/pkg/flow/internal/controller/node_declare.go +++ b/pkg/flow/internal/controller/node_declare.go @@ -11,30 +11,26 @@ type DeclareNode struct { label string nodeID string componentName string - content string - - mut sync.RWMutex - block *ast.BlockStmt + declare *Declare + mut sync.RWMutex } var _ BlockNode = (*DeclareNode)(nil) // NewDeclareNode creates a new declare node with a content which will be loaded by declare component nodes. -func NewDeclareNode(block *ast.BlockStmt, content string) *DeclareNode { +func NewDeclareNode(declare *Declare) *DeclareNode { return &DeclareNode{ - label: block.Label, - nodeID: BlockComponentID(block).String(), - componentName: block.GetBlockName(), - content: content, - - block: block, + label: declare.Block.Label, + nodeID: BlockComponentID(declare.Block).String(), + componentName: declare.Block.GetBlockName(), + declare: declare, } } -func (cn *DeclareNode) ModuleContent() (string, error) { +func (cn *DeclareNode) Declare() *Declare { cn.mut.Lock() defer cn.mut.Unlock() - return cn.content, nil + return cn.declare } // Evaluate does nothing for this node. @@ -48,7 +44,7 @@ func (cn *DeclareNode) Label() string { return cn.label } func (cn *DeclareNode) Block() *ast.BlockStmt { cn.mut.RLock() defer cn.mut.RUnlock() - return cn.block + return cn.declare.Block } // NodeID implements dag.Node and returns the unique ID for the config node. diff --git a/pkg/flow/source.go b/pkg/flow/source.go index 143ea8998f3e..312a56f61410 100644 --- a/pkg/flow/source.go +++ b/pkg/flow/source.go @@ -22,7 +22,7 @@ type Source struct { // The Flow controller can interpret them. components []*ast.BlockStmt configBlocks []*ast.BlockStmt - declares []controller.Declare + declares []*controller.Declare } // ParseSource parses the River file specified by bb into a File. name should be @@ -47,7 +47,7 @@ func ParseSource(name string, bb []byte) (*Source, error) { var ( components []*ast.BlockStmt configs []*ast.BlockStmt - declares []controller.Declare + declares []*controller.Declare ) for _, stmt := range node.Body { @@ -64,7 +64,7 @@ func ParseSource(name string, bb []byte) (*Source, error) { fullName := strings.Join(stmt.Name, ".") switch fullName { case "declare": - declares = append(declares, controller.Declare{Block: stmt, Content: string(bb[stmt.LCurlyPos.Position().Offset+1 : stmt.RCurlyPos.Position().Offset-1])}) + declares = append(declares, controller.NewDeclare(stmt, string(bb[stmt.LCurlyPos.Position().Offset+1:stmt.RCurlyPos.Position().Offset-1]))) case "logging", "tracing", "argument", "export", "import.file", "import.git", "import.http": configs = append(configs, stmt) default: From 69a1d6924f12b2994d051cafbd93b5af1ebac2aa Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 16 Jan 2024 02:55:06 +0100 Subject: [PATCH 04/36] introduces a new componentNode interface --- pkg/flow/flow.go | 13 ++--- pkg/flow/flow_components.go | 20 +++----- .../internal/controller/component_node.go | 32 ++---------- pkg/flow/internal/controller/loader.go | 50 ++++--------------- pkg/flow/internal/controller/metrics.go | 2 +- .../internal/controller/node_config_import.go | 7 ++- .../controller/node_declare_component.go | 7 ++- .../controller/node_native_component.go | 5 ++ .../controller/node_with_component.go | 47 +++++++++++++++++ 9 files changed, 90 insertions(+), 93 deletions(-) create mode 100644 pkg/flow/internal/controller/node_with_component.go diff --git a/pkg/flow/flow.go b/pkg/flow/flow.go index 963f2eebb824..a982a24af3c5 100644 --- a/pkg/flow/flow.go +++ b/pkg/flow/flow.go @@ -245,12 +245,11 @@ func (f *Flow) Run(ctx context.Context) { level.Info(f.log).Log("msg", "scheduling loaded components and services") var ( - components = f.loader.Components() - services = f.loader.Services() - imports = f.loader.Imports() - declareComponents = f.loader.DeclareComponents() + components = f.loader.Components() + services = f.loader.Services() + imports = f.loader.Imports() - runnables = make([]controller.RunnableNode, 0, len(components)+len(services)+len(imports)+len(declareComponents)) + runnables = make([]controller.RunnableNode, 0, len(components)+len(services)+len(imports)) ) for _, c := range components { runnables = append(runnables, c) @@ -260,10 +259,6 @@ func (f *Flow) Run(ctx context.Context) { runnables = append(runnables, i) } - for _, d := range declareComponents { - runnables = append(runnables, d) - } - // Only the root controller should run services, since modules share the // same service instance as the root. if !f.opts.IsModule { diff --git a/pkg/flow/flow_components.go b/pkg/flow/flow_components.go index 33995d02c603..94389946adc0 100644 --- a/pkg/flow/flow_components.go +++ b/pkg/flow/flow_components.go @@ -29,7 +29,7 @@ func (f *Flow) GetComponent(id component.ID, opts component.InfoOptions) (*compo return nil, component.ErrComponentNotFound } - cn, ok := node.(controller.ComponentNode) + cn, ok := node.(controller.NodeWithComponent) if !ok { return nil, fmt.Errorf("%q is not a ComponentNode", id) } @@ -52,26 +52,22 @@ func (f *Flow) ListComponents(moduleID string, opts component.InfoOptions) ([]*c } var ( - components = f.loader.Components() - imports = f.loader.Imports() - declareComponents = f.loader.DeclareComponents() - graph = f.loader.OriginalGraph() + components = f.loader.Components() + imports = f.loader.Imports() + graph = f.loader.OriginalGraph() ) - detail := make([]*component.Info, 0, len(components)+len(imports)+len(declareComponents)) + detail := make([]*component.Info, 0, len(components)+len(imports)) for _, component := range components { detail = append(detail, f.getComponentDetail(component, graph, opts)) } for _, importNode := range imports { detail = append(detail, f.getComponentDetail(importNode, graph, opts)) } - for _, declareComponent := range declareComponents { - detail = append(detail, f.getComponentDetail(declareComponent, graph, opts)) - } return detail, nil } -func (f *Flow) getComponentDetail(cn controller.ComponentNode, graph *dag.Graph, opts component.InfoOptions) *component.Info { +func (f *Flow) getComponentDetail(cn controller.NodeWithComponent, graph *dag.Graph, opts component.InfoOptions) *component.Info { var references, referencedBy []string // Skip over any edge which isn't between two component nodes. This is a @@ -83,12 +79,12 @@ func (f *Flow) getComponentDetail(cn controller.ComponentNode, graph *dag.Graph, // // TODO(rfratto): add support for config block nodes in the API and UI. for _, dep := range graph.Dependencies(cn) { - if _, ok := dep.(controller.ComponentNode); ok { + if _, ok := dep.(controller.NodeWithComponent); ok { references = append(references, dep.NodeID()) } } for _, dep := range graph.Dependants(cn) { - if _, ok := dep.(controller.ComponentNode); ok { + if _, ok := dep.(controller.NodeWithComponent); ok { referencedBy = append(referencedBy, dep.NodeID()) } } diff --git a/pkg/flow/internal/controller/component_node.go b/pkg/flow/internal/controller/component_node.go index cd7d3050483a..f42e6430378d 100644 --- a/pkg/flow/internal/controller/component_node.go +++ b/pkg/flow/internal/controller/component_node.go @@ -1,35 +1,11 @@ package controller -import ( - "github.com/grafana/agent/component" - "github.com/grafana/agent/pkg/flow/internal/dag" -) +import "github.com/grafana/river/ast" // ComponentNode is a dag.Node that manages a component. type ComponentNode interface { - dag.Node + NodeWithComponent - // CurrentHealth returns the current health of the node. - CurrentHealth() component.Health - - // DebugInfo returns debugging information from the managed component (if any). - DebugInfo() interface{} - - // Arguments returns the current arguments of the managed component. - Arguments() component.Arguments - - // Exports returns the current set of exports from the managed component. - Exports() component.Exports - - // Component returns the instance of the managed component. - Component() component.Component - - // ModuleIDs returns the current list of modules that this component is managing. - ModuleIDs() []string - - // Label returns the label for the block or "" if none was specified. - Label() string - - // BlockName returns the name of the block. - BlockName() string + // UpdateBlock updates the River block used to construct arguments for the managed component. + UpdateBlock(b *ast.BlockStmt) } diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 7c527d1d98cf..99f9fad7bb58 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -41,9 +41,8 @@ type Loader struct { mut sync.RWMutex graph *dag.Graph originalGraph *dag.Graph - componentNodes []*NativeComponentNode + componentNodes []ComponentNode serviceNodes []*ServiceNode - declareComponentNodes []*DeclareComponentNode importNodes map[string]*ImportConfigNode declareNodes map[string]*DeclareNode parentModuleDefinitions map[string]string @@ -146,10 +145,9 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co } var ( - components = make([]*NativeComponentNode, 0, len(componentBlocks)) - componentIDs = make([]ComponentID, 0, len(componentBlocks)) - services = make([]*ServiceNode, 0, len(l.services)) - declareComponentNodes = make([]*DeclareComponentNode, 0, len(l.declareComponentNodes)) + components = make([]ComponentNode, 0) + componentIDs = make([]ComponentID, 0, len(componentBlocks)) + services = make([]*ServiceNode, 0, len(l.services)) ) tracer := l.tracer.Tracer("") @@ -180,7 +178,7 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co var err error switch n := n.(type) { - case *NativeComponentNode: + case ComponentNode: components = append(components, n) componentIDs = append(componentIDs, n.ID()) @@ -213,22 +211,6 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co }) } } - case *DeclareComponentNode: - declareComponentNodes = append(declareComponentNodes, n) - componentIDs = append(componentIDs, n.ID()) - if err = l.evaluate(logger, n); err != nil { - var evalDiags diag.Diagnostics - if errors.As(err, &evalDiags) { - diags = append(diags, evalDiags...) - } else { - diags.Add(diag.Diagnostic{ - Severity: diag.SeverityLevelError, - Message: fmt.Sprintf("Failed to build declared component: %s", err), - StartPos: ast.StartPos(n.Block()).Position(), - EndPos: ast.EndPos(n.Block()).Position(), - }) - } - } case BlockNode: if err = l.evaluate(logger, n); err != nil { diags.Add(diag.Diagnostic{ @@ -253,7 +235,6 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co return nil }) - l.declareComponentNodes = declareComponentNodes l.componentNodes = components l.serviceNodes = services l.graph = &newGraph @@ -490,14 +471,10 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo } blockMap[id] = block - // Check the graph from the previous call to Load to see we can copy an - // existing instance of NativeComponentNode and DeclareComponentNode. + // Check the graph from the previous call to Load to see we can copy an existing instance of ComponentNode. if exist := l.graph.GetByID(id); exist != nil { switch v := exist.(type) { - case *NativeComponentNode: - v.UpdateBlock(block) - g.Add(v) - case *DeclareComponentNode: + case ComponentNode: v.UpdateBlock(block) g.Add(v) } @@ -629,18 +606,12 @@ func (l *Loader) Variables() map[string]interface{} { } // Components returns the current set of loaded components. -func (l *Loader) Components() []*NativeComponentNode { +func (l *Loader) Components() []ComponentNode { l.mut.RLock() defer l.mut.RUnlock() return l.componentNodes } -func (l *Loader) DeclareComponents() []*DeclareComponentNode { - l.mut.RLock() - defer l.mut.RUnlock() - return l.declareComponentNodes -} - // Services returns the current set of service nodes. func (l *Loader) Services() []*ServiceNode { l.mut.RLock() @@ -813,14 +784,11 @@ func (l *Loader) evaluate(logger log.Logger, bn BlockNode) error { // mut must be held when calling postEvaluate. func (l *Loader) postEvaluate(logger log.Logger, bn BlockNode, err error) error { switch c := bn.(type) { - case *NativeComponentNode: + case ComponentNode: // Always update the cache both the arguments and exports, since both might // change when a component gets re-evaluated. We also want to cache the arguments and exports in case of an error l.cache.CacheArguments(c.ID(), c.Arguments()) l.cache.CacheExports(c.ID(), c.Exports()) - case *DeclareComponentNode: - l.cache.CacheArguments(c.ID(), c.Arguments()) - l.cache.CacheExports(c.ID(), c.Exports()) case *ArgumentConfigNode: if _, found := l.cache.moduleArguments[c.Label()]; !found { if c.Optional() { diff --git a/pkg/flow/internal/controller/metrics.go b/pkg/flow/internal/controller/metrics.go index 1c5a558ccc1b..7c5275cfbcfc 100644 --- a/pkg/flow/internal/controller/metrics.go +++ b/pkg/flow/internal/controller/metrics.go @@ -112,7 +112,7 @@ func (cc *controllerCollector) Collect(ch chan<- prometheus.Metric) { for _, component := range cc.l.Components() { health := component.CurrentHealth().Health.String() componentsByHealth[health]++ - component.registry.Collect(ch) + component.Registry().Collect(ch) } for health, count := range componentsByHealth { diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 89dc627cdf2f..f9b9de656e66 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -48,7 +48,7 @@ type ImportConfigNode struct { var _ NodeWithDependants = (*ImportConfigNode)(nil) var _ RunnableNode = (*ImportConfigNode)(nil) -var _ ComponentNode = (*ImportConfigNode)(nil) +var _ NodeWithComponent = (*ImportConfigNode)(nil) // NewImportConfigNode creates a new ImportConfigNode from an initial ast.BlockStmt. // The underlying config isn't applied until Evaluate is called. @@ -407,3 +407,8 @@ func (cn *ImportConfigNode) ModuleIDs() []string { func (cn *ImportConfigNode) BlockName() string { return cn.componentName } + +// Registry returns the prometheus registry of the component. +func (cn *ImportConfigNode) Registry() *prometheus.Registry { + return cn.registry +} diff --git a/pkg/flow/internal/controller/node_declare_component.go b/pkg/flow/internal/controller/node_declare_component.go index 6bd459452e88..ab051b93c49d 100644 --- a/pkg/flow/internal/controller/node_declare_component.go +++ b/pkg/flow/internal/controller/node_declare_component.go @@ -163,7 +163,7 @@ func (cn *DeclareComponentNode) Label() string { return cn.label } func (cn *DeclareComponentNode) NodeID() string { return cn.nodeID } // UpdateBlock updates the River block used to construct arguments for the -// managed module. The new block isn't used until the next time Evaluate is +// managed component. The new block isn't used until the next time Evaluate is // invoked. // // UpdateBlock will panic if the block does not match the component ID of the @@ -359,3 +359,8 @@ func (cn *DeclareComponentNode) BlockName() string { func (cn *DeclareComponentNode) Component() component.Component { return nil } + +// Registry returns the prometheus registry of the component. +func (cn *DeclareComponentNode) Registry() *prometheus.Registry { + return cn.registry +} diff --git a/pkg/flow/internal/controller/node_native_component.go b/pkg/flow/internal/controller/node_native_component.go index f7ff2c9592ca..177e1ce0b50d 100644 --- a/pkg/flow/internal/controller/node_native_component.go +++ b/pkg/flow/internal/controller/node_native_component.go @@ -460,3 +460,8 @@ func (cn *NativeComponentNode) ModuleIDs() []string { func (cn *NativeComponentNode) BlockName() string { return cn.componentName } + +// Registry returns the prometheus registry of the component. +func (cn *NativeComponentNode) Registry() *prometheus.Registry { + return cn.registry +} diff --git a/pkg/flow/internal/controller/node_with_component.go b/pkg/flow/internal/controller/node_with_component.go new file mode 100644 index 000000000000..aef514f0e248 --- /dev/null +++ b/pkg/flow/internal/controller/node_with_component.go @@ -0,0 +1,47 @@ +package controller + +import ( + "context" + + "github.com/grafana/agent/component" + "github.com/prometheus/client_golang/prometheus" +) + +// NodeWithComponent is ? +// TODO we need a better name +type NodeWithComponent interface { + BlockNode + + // CurrentHealth returns the current health of the node. + CurrentHealth() component.Health + + // DebugInfo returns debugging information from the managed component (if any). + DebugInfo() interface{} + + // Arguments returns the current arguments of the managed component. + Arguments() component.Arguments + + // Exports returns the current set of exports from the managed component. + Exports() component.Exports + + // Component returns the instance of the managed component. + Component() component.Component + + // ModuleIDs returns the current list of modules that this component is managing. + ModuleIDs() []string + + // Label returns the label for the block or "" if none was specified. + Label() string + + // BlockName returns the name of the block. + BlockName() string + + // Run runs the managed component in the calling goroutine until ctx is canceled. + Run(ctx context.Context) error + + // ID returns the component ID of the managed component from its River block. + ID() ComponentID + + // Registry returns the prometheus registry of the component. + Registry() *prometheus.Registry +} From f020082fe743ce76bbbc4d30b5cb5e92dde9a357 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 16 Jan 2024 13:13:45 +0100 Subject: [PATCH 05/36] rename according to new terminology --- component/module/module.go | 26 ++-- pkg/flow/flow.go | 4 +- pkg/flow/flow_test.go | 2 +- .../controller/custom_component_config.go | 105 ++++++++++++++++ .../custom_component_dependencies.go | 63 ++++++++++ pkg/flow/internal/controller/loader.go | 83 ++++++------- pkg/flow/internal/controller/module_info.go | 107 ---------------- .../internal/controller/module_references.go | 63 ---------- ...component.go => node_builtin_component.go} | 64 +++++----- ...test.go => node_builtin_component_test.go} | 4 +- ..._component.go => node_custom_component.go} | 116 +++++++++--------- pkg/flow/internal/controller/queue_test.go | 8 +- pkg/flow/module.go | 4 +- pkg/flow/module_declare_test.go | 2 +- 14 files changed, 325 insertions(+), 326 deletions(-) create mode 100644 pkg/flow/internal/controller/custom_component_config.go create mode 100644 pkg/flow/internal/controller/custom_component_dependencies.go delete mode 100644 pkg/flow/internal/controller/module_info.go delete mode 100644 pkg/flow/internal/controller/module_references.go rename pkg/flow/internal/controller/{node_native_component.go => node_builtin_component.go} (86%) rename pkg/flow/internal/controller/{node_native_component_test.go => node_builtin_component_test.go} (93%) rename pkg/flow/internal/controller/{node_declare_component.go => node_custom_component.go} (67%) diff --git a/component/module/module.go b/component/module/module.go index 40e2d19d138c..5271a497b907 100644 --- a/component/module/module.go +++ b/component/module/module.go @@ -16,11 +16,11 @@ type ModuleComponent struct { opts component.Options mod component.Module - mut sync.RWMutex - health component.Health - latestContent string - latestArgs map[string]any - latestParentModuleDefinitions map[string]string + mut sync.RWMutex + health component.Health + latestContent string + latestArgs map[string]any + latestParentDeclareContents map[string]string } // Exports holds values which are exported from the run module. @@ -57,12 +57,12 @@ func NewModuleComponentDeprecated(o component.Options) (*ModuleComponent, error) // LoadFlowSource loads the flow controller with the current component source. // It will set the component health in addition to return the error so that the consumer can rely on either or both. // If the content is the same as the last time it was successfully loaded, it will not be reloaded. -func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue string, parentModuleDefinitions map[string]string) error { - if reflect.DeepEqual(args, c.getLatestArgs()) && contentValue == c.getLatestContent() && reflect.DeepEqual(args, c.getLatestParentModuleDefinitions()) { +func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue string, parentDeclareContents map[string]string) error { + if reflect.DeepEqual(args, c.getLatestArgs()) && contentValue == c.getLatestContent() && reflect.DeepEqual(args, c.getLatestParentDeclareContents()) { return nil } - err := c.mod.LoadConfig([]byte(contentValue), args, parentModuleDefinitions) + err := c.mod.LoadConfig([]byte(contentValue), args, parentDeclareContents) if err != nil { c.setHealth(component.Health{ Health: component.HealthTypeUnhealthy, @@ -75,7 +75,7 @@ func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue strin c.setLatestArgs(args) c.setLatestContent(contentValue) - c.setLatestParentModuleDefinitions(parentModuleDefinitions) + c.setLatestParentDeclareContents(parentDeclareContents) c.setHealth(component.Health{ Health: component.HealthTypeHealthy, Message: "module content loaded", @@ -119,16 +119,16 @@ func (c *ModuleComponent) getLatestContent() string { return c.latestContent } -func (c *ModuleComponent) setLatestParentModuleDefinitions(parentModuleDefinitions map[string]string) { +func (c *ModuleComponent) setLatestParentDeclareContents(parentDeclareContents map[string]string) { c.mut.Lock() defer c.mut.Unlock() - c.latestParentModuleDefinitions = parentModuleDefinitions + c.latestParentDeclareContents = parentDeclareContents } -func (c *ModuleComponent) getLatestParentModuleDefinitions() map[string]string { +func (c *ModuleComponent) getLatestParentDeclareContents() map[string]string { c.mut.RLock() defer c.mut.RUnlock() - return c.latestParentModuleDefinitions + return c.latestParentDeclareContents } func (c *ModuleComponent) setLatestArgs(args map[string]any) { diff --git a/pkg/flow/flow.go b/pkg/flow/flow.go index a982a24af3c5..41746d59d229 100644 --- a/pkg/flow/flow.go +++ b/pkg/flow/flow.go @@ -281,11 +281,11 @@ func (f *Flow) Run(ctx context.Context) { // // The controller will only start running components after Load is called once // without any configuration errors. -func (f *Flow) LoadSource(source *Source, args map[string]any, parentModuleDefinitions map[string]string) error { +func (f *Flow) LoadSource(source *Source, args map[string]any, parentDeclareContents map[string]string) error { f.loadMut.Lock() defer f.loadMut.Unlock() - diags := f.loader.Apply(args, source.components, source.configBlocks, source.declares, parentModuleDefinitions) + diags := f.loader.Apply(args, source.components, source.configBlocks, source.declares, parentDeclareContents) if !f.loadedOnce.Load() && diags.HasErrors() { // The first call to Load should not run any components if there were // errors in the configuration file. diff --git a/pkg/flow/flow_test.go b/pkg/flow/flow_test.go index 17a2b60d41d9..3217b17faff7 100644 --- a/pkg/flow/flow_test.go +++ b/pkg/flow/flow_test.go @@ -59,7 +59,7 @@ func getFields(t *testing.T, g *dag.Graph, nodeID string) (component.Arguments, n := g.GetByID(nodeID) require.NotNil(t, n, "couldn't find node %q in graph", nodeID) - uc := n.(*controller.NativeComponentNode) + uc := n.(*controller.BuiltinComponentNode) return uc.Arguments(), uc.Exports() } diff --git a/pkg/flow/internal/controller/custom_component_config.go b/pkg/flow/internal/controller/custom_component_config.go new file mode 100644 index 000000000000..303da2cb4912 --- /dev/null +++ b/pkg/flow/internal/controller/custom_component_config.go @@ -0,0 +1,105 @@ +package controller + +import ( + "fmt" + "strings" +) + +type CustomComponentConfig struct { + declareContent string + additionalDeclareContents map[string]string +} + +func getLocalCustomComponentConfig( + declareNodes map[string]*DeclareNode, + customComponentDependencies map[string][]CustomComponentDependency, + parentDeclareContents map[string]string, + componentName string, + declareLabel string, +) (CustomComponentConfig, error) { + + var customComponentConfig CustomComponentConfig + var err error + + if node, exists := declareNodes[declareLabel]; exists { + customComponentConfig.additionalDeclareContents, err = getLocalAdditionalDeclareContents(componentName, customComponentDependencies, parentDeclareContents) + if err != nil { + return customComponentConfig, err + } + customComponentConfig.declareContent = node.Declare().Content + } else if declareContent, ok := parentDeclareContents[componentName]; ok { + customComponentConfig.additionalDeclareContents = parentDeclareContents + customComponentConfig.declareContent = declareContent + } else { + return customComponentConfig, fmt.Errorf("could not find a corresponding declare for the custom component %s", componentName) + } + return customComponentConfig, nil +} + +func getLocalAdditionalDeclareContents(componentName string, + customComponentDependencies map[string][]CustomComponentDependency, + parentDeclareContents map[string]string, +) (map[string]string, error) { + + additionalDeclareContents := make(map[string]string) + for _, customComponentDependency := range customComponentDependencies[componentName] { + if customComponentDependency.importNode != nil { + for importedDeclareLabel, importedDeclare := range customComponentDependency.importNode.ImportedDeclares() { + // The label of the importNode is added as a prefix to the declare label to create a scope. + // This is useful in the scenario where a custom component of an imported declare is defined inside of a local declare. + // In this case, this custom component should only have have access to the imported declares of its corresponding import node. + additionalDeclareContents[customComponentDependency.importNode.label+"."+importedDeclareLabel] = importedDeclare.Content + } + } else if customComponentDependency.declareNode != nil { + additionalDeclareContents[customComponentDependency.declareLabel] = customComponentDependency.declareNode.Declare().Content + } else { + // Nested declares have access to declare contents defined in their parents. + if declareContent, ok := parentDeclareContents[customComponentDependency.componentName]; ok { + additionalDeclareContents[customComponentDependency.componentName] = declareContent + } else { + return additionalDeclareContents, fmt.Errorf("could not find the required declare content %s for the custom component %s", customComponentDependency.componentName, componentName) + } + } + } + return additionalDeclareContents, nil +} + +func getImportedCustomComponentConfig( + importNodes map[string]*ImportConfigNode, + parentDeclareContents map[string]string, + componentName string, + declareLabel string, + importLabel string, +) (CustomComponentConfig, error) { + + var customComponentConfig CustomComponentConfig + if node, exists := importNodes[importLabel]; exists { + customComponentConfig.additionalDeclareContents = make(map[string]string, len(node.ImportedDeclares())) + for importedDeclareLabel, importedDeclare := range node.ImportedDeclares() { + customComponentConfig.additionalDeclareContents[importedDeclareLabel] = importedDeclare.Content + } + declare, err := node.GetImportedDeclareByLabel(declareLabel) + if err != nil { + return customComponentConfig, err + } + customComponentConfig.declareContent = declare.Content + } else if declareContent, ok := parentDeclareContents[componentName]; ok { + customComponentConfig.additionalDeclareContents = filterParentDeclareContents(importLabel, parentDeclareContents) + customComponentConfig.declareContent = declareContent + } else { + return customComponentConfig, fmt.Errorf("could not find a corresponding imported declare for the custom component %s", componentName) + } + return customComponentConfig, nil +} + +// filterParentDeclareContents prevents custom components from accessing declared content out of their scope. +func filterParentDeclareContents(importLabel string, parentDeclareContents map[string]string) map[string]string { + filteredParentDeclareContents := make(map[string]string) + for declareLabel, declareContent := range parentDeclareContents { + // The scope is defined by the importLabel prefix in the declareLabel of the declare block. + if strings.HasPrefix(declareLabel, importLabel) { + filteredParentDeclareContents[strings.TrimPrefix(declareLabel, importLabel+".")] = declareContent + } + } + return filteredParentDeclareContents +} diff --git a/pkg/flow/internal/controller/custom_component_dependencies.go b/pkg/flow/internal/controller/custom_component_dependencies.go new file mode 100644 index 000000000000..ef6377707f77 --- /dev/null +++ b/pkg/flow/internal/controller/custom_component_dependencies.go @@ -0,0 +1,63 @@ +package controller + +import ( + "strings" + + "github.com/grafana/river/ast" +) + +type CustomComponentDependency struct { + componentName string + importLabel string + declareLabel string + importNode *ImportConfigNode + declareNode *DeclareNode +} + +// GetCustomComponentDependencies traverses the AST of the provided declare and collects references to known custom components. +// Panics if declare is nil. +func GetCustomComponentDependencies( + declare *Declare, + importNodes map[string]*ImportConfigNode, + declareNodes map[string]*DeclareNode, + parentDeclareContents map[string]string, +) ([]CustomComponentDependency, error) { + + uniqueReferences := make(map[string]CustomComponentDependency) + getCustomComponentDependencies(declare.Block.Body, importNodes, declareNodes, uniqueReferences, parentDeclareContents) + + references := make([]CustomComponentDependency, 0, len(uniqueReferences)) + for _, ref := range uniqueReferences { + references = append(references, ref) + } + + return references, nil +} + +func getCustomComponentDependencies( + stmts ast.Body, + importNodes map[string]*ImportConfigNode, + declareNodes map[string]*DeclareNode, + uniqueReferences map[string]CustomComponentDependency, + parentDeclareContents map[string]string, +) { + for _, stmt := range stmts { + switch stmt := stmt.(type) { + case *ast.BlockStmt: + componentName := strings.Join(stmt.Name, ".") + switch componentName { + case "declare": + getCustomComponentDependencies(stmt.Body, importNodes, declareNodes, uniqueReferences, parentDeclareContents) + default: + potentialImportLabel, potentialDeclareLabel := ExtractImportAndDeclareLabels(componentName) + if declareNode, ok := declareNodes[potentialDeclareLabel]; ok { + uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: "", declareLabel: potentialDeclareLabel, declareNode: declareNode} + } else if importNode, ok := importNodes[potentialImportLabel]; ok { + uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel, importNode: importNode} + } else if _, ok := parentDeclareContents[componentName]; ok { + uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel} + } + } + } + } +} diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 99f9fad7bb58..5b3950fbde84 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -38,15 +38,15 @@ type Loader struct { // also prevents log spamming with errors. backoffConfig backoff.Config - mut sync.RWMutex - graph *dag.Graph - originalGraph *dag.Graph - componentNodes []ComponentNode - serviceNodes []*ServiceNode - importNodes map[string]*ImportConfigNode - declareNodes map[string]*DeclareNode - parentModuleDefinitions map[string]string - moduleReferences map[string][]ModuleReference + mut sync.RWMutex + graph *dag.Graph + originalGraph *dag.Graph + componentNodes []ComponentNode + serviceNodes []*ServiceNode + importNodes map[string]*ImportConfigNode + declareNodes map[string]*DeclareNode + parentDeclareContents map[string]string + customComponentDependencies map[string][]CustomComponentDependency cache *valueCache blocks []*ast.BlockStmt // Most recently loaded blocks, used for writing @@ -81,16 +81,16 @@ func NewLoader(opts LoaderOptions) *Loader { } l := &Loader{ - log: log.With(globals.Logger, "controller_id", globals.ControllerID), - tracer: tracing.WrapTracerForLoader(globals.TraceProvider, globals.ControllerID), - globals: globals, - services: services, - host: host, - componentReg: reg, - workerPool: opts.WorkerPool, - importNodes: map[string]*ImportConfigNode{}, - declareNodes: map[string]*DeclareNode{}, - moduleReferences: map[string][]ModuleReference{}, + log: log.With(globals.Logger, "controller_id", globals.ControllerID), + tracer: tracing.WrapTracerForLoader(globals.TraceProvider, globals.ControllerID), + globals: globals, + services: services, + host: host, + componentReg: reg, + workerPool: opts.WorkerPool, + importNodes: map[string]*ImportConfigNode{}, + declareNodes: map[string]*DeclareNode{}, + customComponentDependencies: map[string][]CustomComponentDependency{}, // This is a reasonable default which should work for most cases. If a component is completely stuck, we would // retry and log an error every 10 seconds, at most. @@ -126,7 +126,7 @@ func NewLoader(opts LoaderOptions) *Loader { // The provided parentContext can be used to provide global variables and // functions to components. A child context will be constructed from the parent // to expose values of other components. -func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare, parentModuleDefinitions map[string]string) diag.Diagnostics { +func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare, parentDeclareContents map[string]string) diag.Diagnostics { start := time.Now() l.mut.Lock() defer l.mut.Unlock() @@ -138,7 +138,7 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co } l.cache.SyncModuleArgs(args) - l.parentModuleDefinitions = parentModuleDefinitions + l.parentDeclareContents = parentDeclareContents newGraph, diags := l.loadNewGraph(args, componentBlocks, configBlocks, declares) if diags.HasErrors() { return diags @@ -263,8 +263,9 @@ func (l *Loader) Cleanup(stopWorkerPool bool) { func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare) (dag.Graph, diag.Diagnostics) { var g dag.Graph - // Reset the module cache before loading a new graph. This step is crucial to ensure that references to outdated modules, not included in the new configuration, are removed. - l.moduleReferences = make(map[string][]ModuleReference) + // Reset the custom component dependencies cache before loading a new graph. + // This step is crucial to ensure that references to outdated declares, not included in the new configuration, are removed. + l.customComponentDependencies = make(map[string][]CustomComponentDependency) // Split component blocks into blocks for components and services. componentBlocks, serviceBlocks := l.splitComponentBlocks(componentBlocks) @@ -490,8 +491,8 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo continue } firstPart := strings.Split(componentName, ".")[0] - if l.shouldAddDeclareComponentNode(firstPart, componentName) { - g.Add(NewDeclareComponentNode(l.globals, block, l.getModuleInfo)) + if l.shouldAddCustomComponentNode(firstPart, componentName) { + g.Add(NewCustomComponentNode(l.globals, block, l.getModuleInfo)) } else { registration, exists := l.componentReg.Get(componentName) if !exists { @@ -503,7 +504,7 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo }) continue } - g.Add(NewNativeComponentNode(l.globals, registration, block)) + g.Add(NewBuiltinComponentNode(l.globals, registration, block)) } } } @@ -511,27 +512,27 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo return diags } -func (l *Loader) shouldAddDeclareComponentNode(firstPart, componentName string) bool { +func (l *Loader) shouldAddCustomComponentNode(firstPart, componentName string) bool { _, declareExists := l.declareNodes[firstPart] _, importExists := l.importNodes[firstPart] - _, moduleDepExists := l.parentModuleDefinitions[componentName] + _, parentDeclareContentExists := l.parentDeclareContents[componentName] - return declareExists || importExists || moduleDepExists + return declareExists || importExists || parentDeclareContentExists } -func (l *Loader) wireModuleReferences(g *dag.Graph, dc *DeclareComponentNode, declareNode *DeclareNode) error { - var references []ModuleReference - if deps, ok := l.moduleReferences[declareNode.label]; ok { +func (l *Loader) wireModuleReferences(g *dag.Graph, dc *CustomComponentNode, declareNode *DeclareNode) error { + var references []CustomComponentDependency + if deps, ok := l.customComponentDependencies[declareNode.label]; ok { references = deps } else { var err error - references, err = GetModuleReferences(declareNode.Declare(), l.importNodes, l.declareNodes, l.parentModuleDefinitions) + references, err = GetCustomComponentDependencies(declareNode.Declare(), l.importNodes, l.declareNodes, l.parentDeclareContents) if err != nil { return err } - l.moduleReferences[declareNode.label] = references + l.customComponentDependencies[declareNode.label] = references } - // Add edges between the DeclareComponentNode and all import nodes that it needs. + // Add edges between the CustomComponentNode and all import nodes that it needs. for _, ref := range references { if ref.importNode != nil { g.AddEdge(dag.Edge{From: dc, To: ref.importNode}) @@ -563,8 +564,8 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { case *DeclareNode: // A DeclareNode has no edge, it only holds a static content. continue - case *DeclareComponentNode: - err := l.wireDeclareComponentNode(g, n) + case *CustomComponentNode: + err := l.wireCustomComponentNode(g, n) if err != nil { diags.Add(diag.Diagnostic{ Severity: diag.SeverityLevelError, @@ -587,7 +588,7 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { return diags } -func (l *Loader) wireDeclareComponentNode(g *dag.Graph, dc *DeclareComponentNode) error { +func (l *Loader) wireCustomComponentNode(g *dag.Graph, dc *CustomComponentNode) error { if declareNode, exists := l.declareNodes[dc.declareLabel]; exists { err := l.wireModuleReferences(g, dc, declareNode) if err != nil { @@ -808,11 +809,11 @@ func (l *Loader) postEvaluate(logger log.Logger, bn BlockNode, err error) error return nil } -func (l *Loader) getModuleInfo(componentName string, importLabel string, declareLabel string) (ModuleInfo, error) { +func (l *Loader) getModuleInfo(componentName string, importLabel string, declareLabel string) (CustomComponentConfig, error) { if importLabel == "" { - return getLocalModuleInfo(l.declareNodes, l.moduleReferences, l.parentModuleDefinitions, componentName, declareLabel) + return getLocalCustomComponentConfig(l.declareNodes, l.customComponentDependencies, l.parentDeclareContents, componentName, declareLabel) } - return getImportedModuleInfo(l.importNodes, l.parentModuleDefinitions, componentName, declareLabel, importLabel) + return getImportedCustomComponentConfig(l.importNodes, l.parentDeclareContents, componentName, declareLabel, importLabel) } func multierrToDiags(errors error) diag.Diagnostics { diff --git a/pkg/flow/internal/controller/module_info.go b/pkg/flow/internal/controller/module_info.go deleted file mode 100644 index 9abf23e791b6..000000000000 --- a/pkg/flow/internal/controller/module_info.go +++ /dev/null @@ -1,107 +0,0 @@ -package controller - -import ( - "fmt" - "strings" -) - -type ModuleInfo struct { - content string - moduleDefinitions map[string]string -} - -func getLocalModuleInfo( - declareNodes map[string]*DeclareNode, - moduleReferences map[string][]ModuleReference, - parentModuleDefinitions map[string]string, - componentName string, - declareLabel string, -) (ModuleInfo, error) { - - var moduleInfo ModuleInfo - var content string - var err error - - if node, exists := declareNodes[declareLabel]; exists { - moduleInfo.moduleDefinitions, err = getLocalModuleDefinitions(componentName, moduleReferences, parentModuleDefinitions) - if err != nil { - return moduleInfo, err - } - content = node.Declare().Content - } else if c, ok := parentModuleDefinitions[componentName]; ok { - content = c - moduleInfo.moduleDefinitions = parentModuleDefinitions - } else { - return moduleInfo, fmt.Errorf("could not find a definition for the declared module %s", componentName) - } - moduleInfo.content = content - return moduleInfo, nil -} - -func getLocalModuleDefinitions(componentName string, - localModuleReferences map[string][]ModuleReference, - parentModuleDefinitions map[string]string, -) (map[string]string, error) { - - moduleDefinitions := make(map[string]string) - for _, moduleDependency := range localModuleReferences[componentName] { - if moduleDependency.importNode != nil { - for importModulePath, importModuleDeclare := range moduleDependency.importNode.ImportedDeclares() { - moduleDefinitions[moduleDependency.importNode.label+"."+importModulePath] = importModuleDeclare.Content - } - } else if moduleDependency.declareNode != nil { - def := moduleDependency.declareNode.Declare().Content - moduleDefinitions[moduleDependency.declareLabel] = def - } else { - // Nested declares have access to their parents module definitions. - if c, ok := parentModuleDefinitions[moduleDependency.componentName]; ok { - moduleDefinitions[moduleDependency.componentName] = c - } else { - return moduleDefinitions, fmt.Errorf("could not find the required module dependency %s for the module %s", moduleDependency.componentName, componentName) - } - } - } - return moduleDefinitions, nil -} - -func getImportedModuleInfo( - importNodes map[string]*ImportConfigNode, - parentModuleDefinitions map[string]string, - componentName string, - declareLabel string, - importLabel string, -) (ModuleInfo, error) { - - var moduleInfo ModuleInfo - var content string - if node, exists := importNodes[importLabel]; exists { - moduleInfo.moduleDefinitions = make(map[string]string, len(node.ImportedDeclares())) - for importDeclarePath, importedDeclare := range node.ImportedDeclares() { - moduleInfo.moduleDefinitions[importDeclarePath] = importedDeclare.Content - } - declare, err := node.GetImportedDeclareByLabel(declareLabel) - if err != nil { - return moduleInfo, err - } - content = declare.Content - } else if c, ok := parentModuleDefinitions[componentName]; ok { - content = c - moduleInfo.moduleDefinitions = filterParentModuleDefinitions(importLabel, parentModuleDefinitions) - } else { - return moduleInfo, fmt.Errorf("could not find a definition for the imported module %s", componentName) - } - moduleInfo.content = content - return moduleInfo, nil -} - -// filterParentModuleDefinitions prevents modules from accessing other module definitions which are not in their scope. -func filterParentModuleDefinitions(importLabel string, parentModuleDefinitions map[string]string) map[string]string { - filteredParentModuleDefinitions := make(map[string]string) - for importPath, content := range parentModuleDefinitions { - // The scope is defined by the importLabel prefix in the importPath of the modules. - if strings.HasPrefix(importPath, importLabel) { - filteredParentModuleDefinitions[strings.TrimPrefix(importPath, importLabel+".")] = content - } - } - return filteredParentModuleDefinitions -} diff --git a/pkg/flow/internal/controller/module_references.go b/pkg/flow/internal/controller/module_references.go deleted file mode 100644 index 34382fefecab..000000000000 --- a/pkg/flow/internal/controller/module_references.go +++ /dev/null @@ -1,63 +0,0 @@ -package controller - -import ( - "strings" - - "github.com/grafana/river/ast" -) - -type ModuleReference struct { - componentName string - importLabel string - declareLabel string - importNode *ImportConfigNode - declareNode *DeclareNode -} - -// GetModuleReferences traverses the AST of the provided declare and collects references to known modules. -// Panics if declare is nil. -func GetModuleReferences( - declare *Declare, - importNodes map[string]*ImportConfigNode, - declareNodes map[string]*DeclareNode, - parentModuleDefinitions map[string]string, -) ([]ModuleReference, error) { - - uniqueReferences := make(map[string]ModuleReference) - getModuleReferences(declare.Block.Body, importNodes, declareNodes, uniqueReferences, parentModuleDefinitions) - - references := make([]ModuleReference, 0, len(uniqueReferences)) - for _, ref := range uniqueReferences { - references = append(references, ref) - } - - return references, nil -} - -func getModuleReferences( - stmts ast.Body, - importNodes map[string]*ImportConfigNode, - declareNodes map[string]*DeclareNode, - uniqueReferences map[string]ModuleReference, - parentModuleDefinitions map[string]string, -) { - for _, stmt := range stmts { - switch stmt := stmt.(type) { - case *ast.BlockStmt: - componentName := strings.Join(stmt.Name, ".") - switch componentName { - case "declare": - getModuleReferences(stmt.Body, importNodes, declareNodes, uniqueReferences, parentModuleDefinitions) - default: - potentialImportLabel, potentialDeclareLabel := ExtractImportAndDeclareLabels(componentName) - if declareNode, ok := declareNodes[potentialDeclareLabel]; ok { - uniqueReferences[componentName] = ModuleReference{componentName: componentName, importLabel: "", declareLabel: potentialDeclareLabel, declareNode: declareNode} - } else if importNode, ok := importNodes[potentialImportLabel]; ok { - uniqueReferences[componentName] = ModuleReference{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel, importNode: importNode} - } else if _, ok := parentModuleDefinitions[componentName]; ok { - uniqueReferences[componentName] = ModuleReference{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel} - } - } - } - } -} diff --git a/pkg/flow/internal/controller/node_native_component.go b/pkg/flow/internal/controller/node_builtin_component.go similarity index 86% rename from pkg/flow/internal/controller/node_native_component.go rename to pkg/flow/internal/controller/node_builtin_component.go index 177e1ce0b50d..dead07d03711 100644 --- a/pkg/flow/internal/controller/node_native_component.go +++ b/pkg/flow/internal/controller/node_builtin_component.go @@ -74,12 +74,12 @@ type ComponentGlobals struct { GetServiceData func(name string) (interface{}, error) // Get data for a service. } -// NativeComponentNode is a controller node which manages a user-defined component. +// BuiltinComponentNode is a controller node which manages a user-defined component. // -// NativeComponentNode manages the underlying component and caches its current -// arguments and exports. NativeComponentNode manages the arguments for the component +// BuiltinComponentNode manages the underlying component and caches its current +// arguments and exports. BuiltinComponentNode manages the arguments for the component // from a River block. -type NativeComponentNode struct { +type BuiltinComponentNode struct { id ComponentID globalID string label string @@ -111,12 +111,12 @@ type NativeComponentNode struct { exports component.Exports // Evaluated exports for the managed component } -var _ NodeWithDependants = (*NativeComponentNode)(nil) -var _ ComponentNode = (*NativeComponentNode)(nil) +var _ NodeWithDependants = (*BuiltinComponentNode)(nil) +var _ ComponentNode = (*BuiltinComponentNode)(nil) -// NewNativeComponentNode creates a new NewNativeComponentNode from an initial ast.BlockStmt. +// BuiltinComponentNode creates a new BuiltinComponentNode from an initial ast.BlockStmt. // The underlying managed component isn't created until Evaluate is called. -func NewNativeComponentNode(globals ComponentGlobals, reg component.Registration, b *ast.BlockStmt) *NativeComponentNode { +func NewBuiltinComponentNode(globals ComponentGlobals, reg component.Registration, b *ast.BlockStmt) *BuiltinComponentNode { var ( id = BlockComponentID(b) nodeID = id.String() @@ -138,7 +138,7 @@ func NewNativeComponentNode(globals ComponentGlobals, reg component.Registration globalID = path.Join(globals.ControllerID, nodeID) } - cn := &NativeComponentNode{ + cn := &BuiltinComponentNode{ id: id, globalID: globalID, label: b.Label, @@ -164,7 +164,7 @@ func NewNativeComponentNode(globals ComponentGlobals, reg component.Registration return cn } -func getManagedOptions(globals ComponentGlobals, cn *NativeComponentNode) component.Options { +func getManagedOptions(globals ComponentGlobals, cn *BuiltinComponentNode) component.Options { cn.registry = prometheus.NewRegistry() return component.Options{ ID: cn.globalID, @@ -193,29 +193,29 @@ func getExportsType(reg component.Registration) reflect.Type { } // Registration returns the original registration of the component. -func (cn *NativeComponentNode) Registration() component.Registration { return cn.reg } +func (cn *BuiltinComponentNode) Registration() component.Registration { return cn.reg } // Component returns the instance of the managed component. Component may be // nil if the ComponentNode has not been successfully evaluated yet. -func (cn *NativeComponentNode) Component() component.Component { +func (cn *BuiltinComponentNode) Component() component.Component { cn.mut.RLock() defer cn.mut.RUnlock() return cn.managed } // ID returns the component ID of the managed component from its River block. -func (cn *NativeComponentNode) ID() ComponentID { return cn.id } +func (cn *BuiltinComponentNode) ID() ComponentID { return cn.id } // Label returns the label for the block or "" if none was specified. -func (cn *NativeComponentNode) Label() string { return cn.label } +func (cn *BuiltinComponentNode) Label() string { return cn.label } // ComponentName returns the component's type, i.e. `local.file.test` returns `local.file`. -func (cn *NativeComponentNode) ComponentName() string { return cn.componentName } +func (cn *BuiltinComponentNode) ComponentName() string { return cn.componentName } // NodeID implements dag.Node and returns the unique ID for this node. The // NodeID is the string representation of the component's ID from its River // block. -func (cn *NativeComponentNode) NodeID() string { return cn.nodeID } +func (cn *BuiltinComponentNode) NodeID() string { return cn.nodeID } // UpdateBlock updates the River block used to construct arguments for the // managed component. The new block isn't used until the next time Evaluate is @@ -223,7 +223,7 @@ func (cn *NativeComponentNode) NodeID() string { return cn.nodeID } // // UpdateBlock will panic if the block does not match the component ID of the // ComponentNode. -func (cn *NativeComponentNode) UpdateBlock(b *ast.BlockStmt) { +func (cn *BuiltinComponentNode) UpdateBlock(b *ast.BlockStmt) { if !BlockComponentID(b).Equals(cn.id) { panic("UpdateBlock called with an River block with a different component ID") } @@ -240,7 +240,7 @@ func (cn *NativeComponentNode) UpdateBlock(b *ast.BlockStmt) { // // Evaluate will return an error if the River block cannot be evaluated or if // decoding to arguments fails. -func (cn *NativeComponentNode) Evaluate(scope *vm.Scope) error { +func (cn *BuiltinComponentNode) Evaluate(scope *vm.Scope) error { err := cn.evaluate(scope) switch err { @@ -253,7 +253,7 @@ func (cn *NativeComponentNode) Evaluate(scope *vm.Scope) error { return err } -func (cn *NativeComponentNode) evaluate(scope *vm.Scope) error { +func (cn *BuiltinComponentNode) evaluate(scope *vm.Scope) error { cn.mut.Lock() defer cn.mut.Unlock() @@ -300,7 +300,7 @@ func (cn *NativeComponentNode) evaluate(scope *vm.Scope) error { // // Run will immediately return ErrUnevaluated if Evaluate has never been called // successfully. Otherwise, Run will return nil. -func (cn *NativeComponentNode) Run(ctx context.Context) error { +func (cn *BuiltinComponentNode) Run(ctx context.Context) error { cn.mut.RLock() managed := cn.managed cn.mut.RUnlock() @@ -331,14 +331,14 @@ func (cn *NativeComponentNode) Run(ctx context.Context) error { var ErrUnevaluated = errors.New("managed component not built") // Arguments returns the current arguments of the managed component. -func (cn *NativeComponentNode) Arguments() component.Arguments { +func (cn *BuiltinComponentNode) Arguments() component.Arguments { cn.mut.RLock() defer cn.mut.RUnlock() return cn.args } // Block implements BlockNode and returns the current block of the managed component. -func (cn *NativeComponentNode) Block() *ast.BlockStmt { +func (cn *BuiltinComponentNode) Block() *ast.BlockStmt { cn.mut.RLock() defer cn.mut.RUnlock() return cn.block @@ -346,19 +346,19 @@ func (cn *NativeComponentNode) Block() *ast.BlockStmt { // Exports returns the current set of exports from the managed component. // Exports returns nil if the managed component does not have exports. -func (cn *NativeComponentNode) Exports() component.Exports { +func (cn *BuiltinComponentNode) Exports() component.Exports { cn.exportsMut.RLock() defer cn.exportsMut.RUnlock() return cn.exports } -func (cn *NativeComponentNode) LastUpdateTime() time.Time { +func (cn *BuiltinComponentNode) LastUpdateTime() time.Time { return cn.lastUpdateTime.Load() } // setExports is called whenever the managed component updates. e must be the // same type as the registered exports type of the managed component. -func (cn *NativeComponentNode) setExports(e component.Exports) { +func (cn *BuiltinComponentNode) setExports(e component.Exports) { if cn.exportsType == nil { panic(fmt.Sprintf("Component %s called OnStateChange but never registered an Exports type", cn.nodeID)) } @@ -396,7 +396,7 @@ func (cn *NativeComponentNode) setExports(e component.Exports) { // 1. Health from the call to Run(). // 2. Health from the last call to Evaluate(). // 3. Health reported from the component. -func (cn *NativeComponentNode) CurrentHealth() component.Health { +func (cn *BuiltinComponentNode) CurrentHealth() component.Health { cn.healthMut.RLock() defer cn.healthMut.RUnlock() @@ -414,7 +414,7 @@ func (cn *NativeComponentNode) CurrentHealth() component.Health { } // DebugInfo returns debugging information from the managed component (if any). -func (cn *NativeComponentNode) DebugInfo() interface{} { +func (cn *BuiltinComponentNode) DebugInfo() interface{} { cn.mut.RLock() defer cn.mut.RUnlock() @@ -426,7 +426,7 @@ func (cn *NativeComponentNode) DebugInfo() interface{} { // setEvalHealth sets the internal health from a call to Evaluate. See Health // for information on how overall health is calculated. -func (cn *NativeComponentNode) setEvalHealth(t component.HealthType, msg string) { +func (cn *BuiltinComponentNode) setEvalHealth(t component.HealthType, msg string) { cn.healthMut.Lock() defer cn.healthMut.Unlock() @@ -439,7 +439,7 @@ func (cn *NativeComponentNode) setEvalHealth(t component.HealthType, msg string) // setRunHealth sets the internal health from a call to Run. See Health for // information on how overall health is calculated. -func (cn *NativeComponentNode) setRunHealth(t component.HealthType, msg string) { +func (cn *BuiltinComponentNode) setRunHealth(t component.HealthType, msg string) { cn.healthMut.Lock() defer cn.healthMut.Unlock() @@ -452,16 +452,16 @@ func (cn *NativeComponentNode) setRunHealth(t component.HealthType, msg string) // ModuleIDs returns the current list of modules that this component is // managing. -func (cn *NativeComponentNode) ModuleIDs() []string { +func (cn *BuiltinComponentNode) ModuleIDs() []string { return cn.moduleController.ModuleIDs() } // BlockName returns the name of the block. -func (cn *NativeComponentNode) BlockName() string { +func (cn *BuiltinComponentNode) BlockName() string { return cn.componentName } // Registry returns the prometheus registry of the component. -func (cn *NativeComponentNode) Registry() *prometheus.Registry { +func (cn *BuiltinComponentNode) Registry() *prometheus.Registry { return cn.registry } diff --git a/pkg/flow/internal/controller/node_native_component_test.go b/pkg/flow/internal/controller/node_builtin_component_test.go similarity index 93% rename from pkg/flow/internal/controller/node_native_component_test.go rename to pkg/flow/internal/controller/node_builtin_component_test.go index e2f734352030..6a1165b2cc6d 100644 --- a/pkg/flow/internal/controller/node_native_component_test.go +++ b/pkg/flow/internal/controller/node_builtin_component_test.go @@ -14,7 +14,7 @@ func TestGlobalID(t *testing.T) { NewModuleController: func(id string) ModuleController { return nil }, - }, &NativeComponentNode{ + }, &BuiltinComponentNode{ nodeID: "local.id", globalID: "module.file/local.id", }) @@ -28,7 +28,7 @@ func TestLocalID(t *testing.T) { NewModuleController: func(id string) ModuleController { return nil }, - }, &NativeComponentNode{ + }, &BuiltinComponentNode{ nodeID: "local.id", globalID: "local.id", }) diff --git a/pkg/flow/internal/controller/node_declare_component.go b/pkg/flow/internal/controller/node_custom_component.go similarity index 67% rename from pkg/flow/internal/controller/node_declare_component.go rename to pkg/flow/internal/controller/node_custom_component.go index ab051b93c49d..3b77600692ef 100644 --- a/pkg/flow/internal/controller/node_declare_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -21,11 +21,11 @@ import ( "go.uber.org/atomic" ) -// DeclareComponentNode is a controller node which manages a module. +// CustomComponentNode is a controller node which manages a custom component. // -// DeclareComponentNode manages the underlying module and caches its current +// CustomComponentNode manages the underlying custom component and caches its current // arguments and exports. -type DeclareComponentNode struct { +type CustomComponentNode struct { id ComponentID globalID string label string @@ -38,25 +38,25 @@ type DeclareComponentNode struct { moduleController ModuleController OnNodeWithDependantsUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate - GetModuleInfo func(fullName string, importLabel string, declareLabel string) (ModuleInfo, error) // Retrieve the module config. + GetModuleInfo func(fullName string, importLabel string, declareLabel string) (CustomComponentConfig, error) // Retrieve the module config. lastUpdateTime atomic.Time mut sync.RWMutex block *ast.BlockStmt // Current River block to derive args from eval *vm.Evaluator - managed *module.ModuleComponent // Inner managed module + managed *module.ModuleComponent // Inner managed custom component args component.Arguments // Evaluated arguments for the managed component // NOTE(rfratto): health and exports have their own mutex because they may be // set asynchronously while mut is still being held (i.e., when calling Evaluate - // and the managed module immediately creates new exports) + // and the managed custom component immediately creates new exports) healthMut sync.RWMutex evalHealth component.Health // Health of the last evaluate runHealth component.Health // Health of running the component exportsMut sync.RWMutex - exports component.Exports // Evaluated exports for the managed module + exports component.Exports // Evaluated exports for the managed custom component } // ExtractImportAndDeclareLabels extract an importLabel and a declareLabel from a componentName. @@ -68,7 +68,7 @@ func ExtractImportAndDeclareLabels(componentName string) (string, string) { // If this is a local declare. importLabel := "" declareLabel := parts[0] - // If this is an imported module. + // If this is an imported custom component. if len(parts) > 1 { importLabel = parts[0] declareLabel = parts[1] @@ -76,12 +76,12 @@ func ExtractImportAndDeclareLabels(componentName string) (string, string) { return importLabel, declareLabel } -var _ NodeWithDependants = (*DeclareComponentNode)(nil) -var _ ComponentNode = (*DeclareComponentNode)(nil) +var _ NodeWithDependants = (*CustomComponentNode)(nil) +var _ ComponentNode = (*CustomComponentNode)(nil) -// NewDeclareComponentNode creates a new DeclareComponentNode from an initial ast.BlockStmt. -// The underlying managed module isn't created until Evaluate is called. -func NewDeclareComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetModuleInfo func(string, string, string) (ModuleInfo, error)) *DeclareComponentNode { +// NewCustomComponentNode creates a new CustomComponentNode from an initial ast.BlockStmt. +// The underlying managed custom component isn't created until Evaluate is called. +func NewCustomComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetModuleInfo func(string, string, string) (CustomComponentConfig, error)) *CustomComponentNode { var ( id = BlockComponentID(b) nodeID = id.String() @@ -107,7 +107,7 @@ func NewDeclareComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetModu importLabel, declareLabel := ExtractImportAndDeclareLabels(componentName) - cn := &DeclareComponentNode{ + cn := &CustomComponentNode{ id: id, globalID: globalID, label: b.Label, @@ -130,7 +130,7 @@ func NewDeclareComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetModu return cn } -func getDeclareManagedOptions(globals ComponentGlobals, cn *DeclareComponentNode) component.Options { +func getDeclareManagedOptions(globals ComponentGlobals, cn *CustomComponentNode) component.Options { cn.registry = prometheus.NewRegistry() return component.Options{ ID: cn.globalID, @@ -152,23 +152,23 @@ func getDeclareManagedOptions(globals ComponentGlobals, cn *DeclareComponentNode } // ID returns the component ID of the managed component from its River block. -func (cn *DeclareComponentNode) ID() ComponentID { return cn.id } +func (cn *CustomComponentNode) ID() ComponentID { return cn.id } // Label returns the label for the block or "" if none was specified. -func (cn *DeclareComponentNode) Label() string { return cn.label } +func (cn *CustomComponentNode) Label() string { return cn.label } // NodeID implements dag.Node and returns the unique ID for this node. The // NodeID is the string representation of the component's ID from its River // block. -func (cn *DeclareComponentNode) NodeID() string { return cn.nodeID } +func (cn *CustomComponentNode) NodeID() string { return cn.nodeID } // UpdateBlock updates the River block used to construct arguments for the // managed component. The new block isn't used until the next time Evaluate is // invoked. // // UpdateBlock will panic if the block does not match the component ID of the -// DeclareComponentNode. -func (cn *DeclareComponentNode) UpdateBlock(b *ast.BlockStmt) { +// CustomComponentNode. +func (cn *CustomComponentNode) UpdateBlock(b *ast.BlockStmt) { if !BlockComponentID(b).Equals(cn.id) { panic("UpdateBlock called with an River block with a different component ID") } @@ -179,13 +179,13 @@ func (cn *DeclareComponentNode) UpdateBlock(b *ast.BlockStmt) { cn.eval = vm.New(b.Body) } -// Evaluate implements BlockNode and updates the arguments by re-evaluating its River block with the provided scope and the module content by -// retrieving it from the corresponding import or declare node for the managed module. -// The managed module will be built the first time Evaluate is called. +// Evaluate implements BlockNode and updates the arguments by re-evaluating its River block with the provided scope and the custom component by +// retrieving the component definition from the corresponding import or declare node. +// The managed custom component will be built the first time Evaluate is called. // // Evaluate will return an error if the River block cannot be evaluated, if -// decoding to arguments fails or if the module content cannot be retrieved. -func (cn *DeclareComponentNode) Evaluate(scope *vm.Scope) error { +// decoding to arguments fails or if the custom component definition cannot be retrieved. +func (cn *CustomComponentNode) Evaluate(scope *vm.Scope) error { err := cn.evaluate(scope) switch err { @@ -198,7 +198,7 @@ func (cn *DeclareComponentNode) Evaluate(scope *vm.Scope) error { return err } -func (cn *DeclareComponentNode) evaluate(scope *vm.Scope) error { +func (cn *CustomComponentNode) evaluate(scope *vm.Scope) error { cn.mut.Lock() defer cn.mut.Unlock() @@ -208,27 +208,27 @@ func (cn *DeclareComponentNode) evaluate(scope *vm.Scope) error { } if cn.managed == nil { - // We haven't built the managed module successfully yet. + // We haven't built the managed custom component successfully yet. managed, err := module.NewModuleComponent(cn.managedOpts) if err != nil { - return fmt.Errorf("building module: %w", err) + return fmt.Errorf("building custom component: %w", err) } cn.managed = managed } moduleInfo, err := cn.GetModuleInfo(cn.componentName, cn.importLabel, cn.declareLabel) if err != nil { - return fmt.Errorf("retrieving module info: %w", err) + return fmt.Errorf("retrieving custom component info: %w", err) } - // Reload the module with new config - if err := cn.managed.LoadFlowSource(args, moduleInfo.content, moduleInfo.moduleDefinitions); err != nil { + // Reload the custom component with new config + if err := cn.managed.LoadFlowSource(args, moduleInfo.declareContent, moduleInfo.additionalDeclareContents); err != nil { return fmt.Errorf("updating component: %w", err) } return nil } -func (cn *DeclareComponentNode) Run(ctx context.Context) error { +func (cn *CustomComponentNode) Run(ctx context.Context) error { cn.mut.RLock() managed := cn.managed logger := cn.managedOpts.Logger @@ -238,43 +238,43 @@ func (cn *DeclareComponentNode) Run(ctx context.Context) error { return ErrUnevaluated } - cn.setRunHealth(component.HealthTypeHealthy, "started module") + cn.setRunHealth(component.HealthTypeHealthy, "started custom component") cn.managed.RunFlowController(ctx) - level.Info(logger).Log("msg", "module exited") - cn.setRunHealth(component.HealthTypeExited, "module shut down") + level.Info(logger).Log("msg", "custom component exited") + cn.setRunHealth(component.HealthTypeExited, "custom component shut down") return nil } -// Arguments returns the current arguments of the managed module. -func (cn *DeclareComponentNode) Arguments() component.Arguments { +// Arguments returns the current arguments of the managed custom component. +func (cn *CustomComponentNode) Arguments() component.Arguments { cn.mut.RLock() defer cn.mut.RUnlock() return cn.args } -// Block implements BlockNode and returns the current block of the managed module. -func (cn *DeclareComponentNode) Block() *ast.BlockStmt { +// Block implements BlockNode and returns the current block of the managed custom component. +func (cn *CustomComponentNode) Block() *ast.BlockStmt { cn.mut.RLock() defer cn.mut.RUnlock() return cn.block } -// Exports returns the current set of exports from the managed module. -// Exports returns nil if the managed module does not have exports. -func (cn *DeclareComponentNode) Exports() component.Exports { +// Exports returns the current set of exports from the managed custom component. +// Exports returns nil if the managed custom component does not have exports. +func (cn *CustomComponentNode) Exports() component.Exports { cn.exportsMut.RLock() defer cn.exportsMut.RUnlock() return cn.exports } -func (cn *DeclareComponentNode) LastUpdateTime() time.Time { +func (cn *CustomComponentNode) LastUpdateTime() time.Time { return cn.lastUpdateTime.Load() } -// setExports is called whenever the managed module updates. e must be the -// same type as the registered exports type of the managed module. -func (cn *DeclareComponentNode) setExports(e component.Exports) { +// setExports is called whenever the managed custom component updates. e must be the +// same type as the registered exports type of the managed custom component. +func (cn *CustomComponentNode) setExports(e component.Exports) { // Some components may aggressively reexport values even though no exposed // state has changed. This may be done for components which always supply // exports whenever their arguments are evaluated without tracking internal @@ -298,21 +298,21 @@ func (cn *DeclareComponentNode) setExports(e component.Exports) { } } -// CurrentHealth returns the current health of the DeclareComponentNode. +// CurrentHealth returns the current health of the CustomComponentNode. // -// The health of a DeclareComponentNode is determined by combining: +// The health of a CustomComponentNode is determined by combining: // // 1. Health from the call to Run(). // 2. Health from the last call to Evaluate(). -// 3. Health reported from the module. -func (cn *DeclareComponentNode) CurrentHealth() component.Health { +// 3. Health reported from the custom component. +func (cn *CustomComponentNode) CurrentHealth() component.Health { cn.healthMut.RLock() defer cn.healthMut.RUnlock() return component.LeastHealthy(cn.runHealth, cn.evalHealth, cn.managed.CurrentHealth()) } // TODO implement debugInfo? -func (cn *DeclareComponentNode) DebugInfo() interface{} { +func (cn *CustomComponentNode) DebugInfo() interface{} { cn.mut.RLock() defer cn.mut.RUnlock() return nil @@ -320,7 +320,7 @@ func (cn *DeclareComponentNode) DebugInfo() interface{} { // setEvalHealth sets the internal health from a call to Evaluate. See Health // for information on how overall health is calculated. -func (cn *DeclareComponentNode) setEvalHealth(t component.HealthType, msg string) { +func (cn *CustomComponentNode) setEvalHealth(t component.HealthType, msg string) { cn.healthMut.Lock() defer cn.healthMut.Unlock() @@ -333,7 +333,7 @@ func (cn *DeclareComponentNode) setEvalHealth(t component.HealthType, msg string // setRunHealth sets the internal health from a call to Run. See Health for // information on how overall health is calculated. -func (cn *DeclareComponentNode) setRunHealth(t component.HealthType, msg string) { +func (cn *CustomComponentNode) setRunHealth(t component.HealthType, msg string) { cn.healthMut.Lock() defer cn.healthMut.Unlock() @@ -344,23 +344,23 @@ func (cn *DeclareComponentNode) setRunHealth(t component.HealthType, msg string) } } -// ModuleIDs returns the current list of modules that this component is +// ModuleIDs returns the current list of custom components that this component is // managing. -func (cn *DeclareComponentNode) ModuleIDs() []string { +func (cn *CustomComponentNode) ModuleIDs() []string { return cn.moduleController.ModuleIDs() } // BlockName returns the name of the block. -func (cn *DeclareComponentNode) BlockName() string { +func (cn *CustomComponentNode) BlockName() string { return cn.componentName } // This node does not manage any component. -func (cn *DeclareComponentNode) Component() component.Component { +func (cn *CustomComponentNode) Component() component.Component { return nil } // Registry returns the prometheus registry of the component. -func (cn *DeclareComponentNode) Registry() *prometheus.Registry { +func (cn *CustomComponentNode) Registry() *prometheus.Registry { return cn.registry } diff --git a/pkg/flow/internal/controller/queue_test.go b/pkg/flow/internal/controller/queue_test.go index 8fe953ba31ba..1bb4bcbb4438 100644 --- a/pkg/flow/internal/controller/queue_test.go +++ b/pkg/flow/internal/controller/queue_test.go @@ -9,7 +9,7 @@ import ( ) func TestEnqueueDequeue(t *testing.T) { - tn := &NativeComponentNode{} + tn := &BuiltinComponentNode{} q := NewQueue() q.Enqueue(tn) require.Lenf(t, q.queuedSet, 1, "queue should be 1") @@ -26,7 +26,7 @@ func TestDequeue_Empty(t *testing.T) { } func TestDequeue_InOrder(t *testing.T) { - c1, c2, c3 := &NativeComponentNode{}, &NativeComponentNode{}, &NativeComponentNode{} + c1, c2, c3 := &BuiltinComponentNode{}, &BuiltinComponentNode{}, &BuiltinComponentNode{} q := NewQueue() q.Enqueue(c1) q.Enqueue(c2) @@ -41,7 +41,7 @@ func TestDequeue_InOrder(t *testing.T) { } func TestDequeue_NoDuplicates(t *testing.T) { - c1, c2 := &NativeComponentNode{}, &NativeComponentNode{} + c1, c2 := &BuiltinComponentNode{}, &BuiltinComponentNode{} q := NewQueue() q.Enqueue(c1) q.Enqueue(c1) @@ -58,7 +58,7 @@ func TestDequeue_NoDuplicates(t *testing.T) { } func TestEnqueue_ChannelNotification(t *testing.T) { - c1 := &NativeComponentNode{} + c1 := &BuiltinComponentNode{} q := NewQueue() notificationsCount := atomic.Int32{} diff --git a/pkg/flow/module.go b/pkg/flow/module.go index d67eefcb0508..7c12d950d6da 100644 --- a/pkg/flow/module.go +++ b/pkg/flow/module.go @@ -128,12 +128,12 @@ func newModule(o *moduleOptions) *module { } // LoadConfig parses River config and loads it. -func (c *module) LoadConfig(config []byte, args map[string]any, parentModuleDefinitions map[string]string) error { +func (c *module) LoadConfig(config []byte, args map[string]any, parentDeclareContents map[string]string) error { ff, err := ParseSource(c.o.ID, config) if err != nil { return err } - return c.f.LoadSource(ff, args, parentModuleDefinitions) + return c.f.LoadSource(ff, args, parentDeclareContents) } // Run starts the Module. No components within the Module diff --git a/pkg/flow/module_declare_test.go b/pkg/flow/module_declare_test.go index c471bf38beea..a40d537a65a6 100644 --- a/pkg/flow/module_declare_test.go +++ b/pkg/flow/module_declare_test.go @@ -16,7 +16,7 @@ type testCase struct { expected int } -func TestDeclareComponent(t *testing.T) { +func TestDeclare(t *testing.T) { tt := []testCase{ { name: "BasicDeclare", From 130a98e1395c0b99bd78c236c3be315b91fc412a Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 16 Jan 2024 13:40:21 +0100 Subject: [PATCH 06/36] some additional renaming --- pkg/flow/internal/controller/loader.go | 22 +++++++++---------- .../controller/node_custom_component.go | 12 +++++----- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 5b3950fbde84..33ae20d7af0e 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -492,7 +492,7 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo } firstPart := strings.Split(componentName, ".")[0] if l.shouldAddCustomComponentNode(firstPart, componentName) { - g.Add(NewCustomComponentNode(l.globals, block, l.getModuleInfo)) + g.Add(NewCustomComponentNode(l.globals, block, l.getCustomComponentConfig)) } else { registration, exists := l.componentReg.Get(componentName) if !exists { @@ -520,22 +520,22 @@ func (l *Loader) shouldAddCustomComponentNode(firstPart, componentName string) b return declareExists || importExists || parentDeclareContentExists } -func (l *Loader) wireModuleReferences(g *dag.Graph, dc *CustomComponentNode, declareNode *DeclareNode) error { - var references []CustomComponentDependency +func (l *Loader) wireCustomComponentDependencies(g *dag.Graph, dc *CustomComponentNode, declareNode *DeclareNode) error { + var dependencies []CustomComponentDependency if deps, ok := l.customComponentDependencies[declareNode.label]; ok { - references = deps + dependencies = deps } else { var err error - references, err = GetCustomComponentDependencies(declareNode.Declare(), l.importNodes, l.declareNodes, l.parentDeclareContents) + dependencies, err = GetCustomComponentDependencies(declareNode.Declare(), l.importNodes, l.declareNodes, l.parentDeclareContents) if err != nil { return err } - l.customComponentDependencies[declareNode.label] = references + l.customComponentDependencies[declareNode.label] = dependencies } // Add edges between the CustomComponentNode and all import nodes that it needs. - for _, ref := range references { - if ref.importNode != nil { - g.AddEdge(dag.Edge{From: dc, To: ref.importNode}) + for _, dependency := range dependencies { + if dependency.importNode != nil { + g.AddEdge(dag.Edge{From: dc, To: dependency.importNode}) } } return nil @@ -590,7 +590,7 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { func (l *Loader) wireCustomComponentNode(g *dag.Graph, dc *CustomComponentNode) error { if declareNode, exists := l.declareNodes[dc.declareLabel]; exists { - err := l.wireModuleReferences(g, dc, declareNode) + err := l.wireCustomComponentDependencies(g, dc, declareNode) if err != nil { return err } @@ -809,7 +809,7 @@ func (l *Loader) postEvaluate(logger log.Logger, bn BlockNode, err error) error return nil } -func (l *Loader) getModuleInfo(componentName string, importLabel string, declareLabel string) (CustomComponentConfig, error) { +func (l *Loader) getCustomComponentConfig(componentName string, importLabel string, declareLabel string) (CustomComponentConfig, error) { if importLabel == "" { return getLocalCustomComponentConfig(l.declareNodes, l.customComponentDependencies, l.parentDeclareContents, componentName, declareLabel) } diff --git a/pkg/flow/internal/controller/node_custom_component.go b/pkg/flow/internal/controller/node_custom_component.go index 3b77600692ef..921165256be5 100644 --- a/pkg/flow/internal/controller/node_custom_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -38,8 +38,8 @@ type CustomComponentNode struct { moduleController ModuleController OnNodeWithDependantsUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate - GetModuleInfo func(fullName string, importLabel string, declareLabel string) (CustomComponentConfig, error) // Retrieve the module config. - lastUpdateTime atomic.Time + GetCustomComponentConfig func(fullName string, importLabel string, declareLabel string) (CustomComponentConfig, error) // Retrieve the custom component config. + lastUpdateTime atomic.Time mut sync.RWMutex block *ast.BlockStmt // Current River block to derive args from @@ -81,7 +81,7 @@ var _ ComponentNode = (*CustomComponentNode)(nil) // NewCustomComponentNode creates a new CustomComponentNode from an initial ast.BlockStmt. // The underlying managed custom component isn't created until Evaluate is called. -func NewCustomComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetModuleInfo func(string, string, string) (CustomComponentConfig, error)) *CustomComponentNode { +func NewCustomComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetCustomComponentConfig func(string, string, string) (CustomComponentConfig, error)) *CustomComponentNode { var ( id = BlockComponentID(b) nodeID = id.String() @@ -117,7 +117,7 @@ func NewCustomComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetModul declareLabel: declareLabel, moduleController: globals.NewModuleController(globalID), OnNodeWithDependantsUpdate: globals.OnNodeWithDependantsUpdate, - GetModuleInfo: GetModuleInfo, + GetCustomComponentConfig: GetCustomComponentConfig, block: b, eval: vm.New(b.Body), @@ -216,13 +216,13 @@ func (cn *CustomComponentNode) evaluate(scope *vm.Scope) error { cn.managed = managed } - moduleInfo, err := cn.GetModuleInfo(cn.componentName, cn.importLabel, cn.declareLabel) + customComponentConfig, err := cn.GetCustomComponentConfig(cn.componentName, cn.importLabel, cn.declareLabel) if err != nil { return fmt.Errorf("retrieving custom component info: %w", err) } // Reload the custom component with new config - if err := cn.managed.LoadFlowSource(args, moduleInfo.declareContent, moduleInfo.additionalDeclareContents); err != nil { + if err := cn.managed.LoadFlowSource(args, customComponentConfig.declareContent, customComponentConfig.additionalDeclareContents); err != nil { return fmt.Errorf("updating component: %w", err) } return nil From c883d1aa1447c0f095ea44cc58d5841ee73e339d Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 16 Jan 2024 16:45:15 +0100 Subject: [PATCH 07/36] introduce the component node manager --- .../controller/component_node_manager.go | 244 ++++++++++++++++++ .../controller/custom_component_config.go | 105 -------- .../custom_component_dependencies.go | 63 ----- pkg/flow/internal/controller/loader.go | 130 ++++------ .../controller/node_custom_component.go | 8 +- pkg/flow/module_import_test.go | 2 +- 6 files changed, 294 insertions(+), 258 deletions(-) create mode 100644 pkg/flow/internal/controller/component_node_manager.go delete mode 100644 pkg/flow/internal/controller/custom_component_config.go delete mode 100644 pkg/flow/internal/controller/custom_component_dependencies.go diff --git a/pkg/flow/internal/controller/component_node_manager.go b/pkg/flow/internal/controller/component_node_manager.go new file mode 100644 index 000000000000..a3b4b1571880 --- /dev/null +++ b/pkg/flow/internal/controller/component_node_manager.go @@ -0,0 +1,244 @@ +package controller + +import ( + "fmt" + "strings" + + "github.com/grafana/river/ast" +) + +// ComponentNodeManager is a manager that manages component nodes. +type ComponentNodeManager struct { + importNodes map[string]*ImportConfigNode + declareNodes map[string]*DeclareNode + globals ComponentGlobals + componentReg ComponentRegistry + customComponentDependencies map[string][]CustomComponentDependency + parentDeclareContents map[string]string +} + +// NewComponentNodeManager creates a new ComponentNodeManager. +func NewComponentNodeManager(globals ComponentGlobals, componentReg ComponentRegistry) *ComponentNodeManager { + return &ComponentNodeManager{ + importNodes: map[string]*ImportConfigNode{}, + declareNodes: map[string]*DeclareNode{}, + customComponentDependencies: map[string][]CustomComponentDependency{}, + globals: globals, + componentReg: componentReg, + } +} + +// OnReload resets the state of the component node manager. +func (m *ComponentNodeManager) OnReload(parentDeclareContents map[string]string) { + m.parentDeclareContents = parentDeclareContents + m.customComponentDependencies = make(map[string][]CustomComponentDependency) + m.importNodes = map[string]*ImportConfigNode{} + m.declareNodes = map[string]*DeclareNode{} +} + +// CreateComponentNode creates a new builtin component or a new custom component. +func (m *ComponentNodeManager) CreateComponentNode(componentName string, block *ast.BlockStmt) (ComponentNode, error) { + firstPart := strings.Split(componentName, ".")[0] + if m.shouldAddCustomComponentNode(firstPart, componentName) { + return NewCustomComponentNode(m.globals, block, m.getCustomComponentConfig), nil + } else { + registration, exists := m.componentReg.Get(componentName) + if !exists { + return nil, fmt.Errorf("unrecognized component name %q", componentName) + } + return NewBuiltinComponentNode(m.globals, registration, block), nil + } +} + +// GetCustomComponentDependencies retrieves and caches the dependencies that declare might have to other declares. +func (m *ComponentNodeManager) getCustomComponentDependencies(declareNode *DeclareNode) ([]CustomComponentDependency, error) { + var dependencies []CustomComponentDependency + if deps, ok := m.customComponentDependencies[declareNode.label]; ok { + dependencies = deps + } else { + var err error + dependencies, err = m.FindCustomComponentDependencies(declareNode.Declare()) + if err != nil { + return nil, err + } + m.customComponentDependencies[declareNode.label] = dependencies + } + return dependencies, nil +} + +// shouldAddCustomComponentNode searches for a declare corresponding to the given component name. +func (m *ComponentNodeManager) shouldAddCustomComponentNode(firstPart, componentName string) bool { + _, declareExists := m.declareNodes[firstPart] + _, importExists := m.importNodes[firstPart] + _, parentDeclareContentExists := m.parentDeclareContents[componentName] + + return declareExists || importExists || parentDeclareContentExists +} + +func (m *ComponentNodeManager) GetCorrespondingLocalDeclare(cc *CustomComponentNode) (*DeclareNode, bool) { + declareNode, exist := m.declareNodes[cc.declareLabel] + return declareNode, exist +} + +func (m *ComponentNodeManager) GetCorrespondingImportedDeclare(cc *CustomComponentNode) (*ImportConfigNode, bool) { + importNode, exist := m.importNodes[cc.importLabel] + return importNode, exist +} + +// CustomComponentConfig represents the config needed by a custom component to load. +type CustomComponentConfig struct { + declareContent string // represents the corresponding declare as plain string + additionalDeclareContents map[string]string // represents the additional declare that might be needed by the component to build custom components +} + +// getCustomComponentConfig returns the custom component config for a given custom component. +func (m *ComponentNodeManager) getCustomComponentConfig(cc *CustomComponentNode) (CustomComponentConfig, error) { + var customComponentConfig CustomComponentConfig + var found bool + var err error + if cc.importLabel == "" { + customComponentConfig, found = m.getCustomComponentConfigFromLocalDeclares(cc) + if !found { + customComponentConfig, found = m.getCustomComponentConfigFromParent(cc) + } + } else { + customComponentConfig, found, err = m.getCustomComponentConfigFromImportedDeclares(cc) + if err != nil { + return customComponentConfig, err + } + if !found { + customComponentConfig, found = m.getCustomComponentConfigFromParent(cc) + customComponentConfig.additionalDeclareContents = filterParentDeclareContents(cc.importLabel, customComponentConfig.additionalDeclareContents) + } + } + if !found { + return customComponentConfig, fmt.Errorf("custom component config not found for component %s", cc.componentName) + } + return customComponentConfig, nil +} + +// getCustomComponentConfigFromLocalDeclares retrieves the config of a custom component from the local declares. +func (m *ComponentNodeManager) getCustomComponentConfigFromLocalDeclares(cc *CustomComponentNode) (CustomComponentConfig, bool) { + node, exists := m.declareNodes[cc.declareLabel] + if !exists { + return CustomComponentConfig{}, false + } + return CustomComponentConfig{ + declareContent: node.Declare().Content, + additionalDeclareContents: m.getLocalAdditionalDeclareContents(cc.componentName), + }, true +} + +// getCustomComponentConfigFromParent retrieves the config of a custom component from the parent controller. +func (m *ComponentNodeManager) getCustomComponentConfigFromParent(cc *CustomComponentNode) (CustomComponentConfig, bool) { + declareContent, exists := m.parentDeclareContents[cc.componentName] + if !exists { + return CustomComponentConfig{}, false + } + return CustomComponentConfig{ + declareContent: declareContent, + additionalDeclareContents: m.parentDeclareContents, + }, true +} + +// getImportedCustomComponentConfig retrieves the config of a custom component from the imported declares. +func (m *ComponentNodeManager) getCustomComponentConfigFromImportedDeclares(cc *CustomComponentNode) (CustomComponentConfig, bool, error) { + node, exists := m.importNodes[cc.importLabel] + if !exists { + return CustomComponentConfig{}, false, nil + } + declare, err := node.GetImportedDeclareByLabel(cc.declareLabel) + if err != nil { + return CustomComponentConfig{}, false, err + } + return CustomComponentConfig{ + declareContent: declare.Content, + additionalDeclareContents: m.getImportAdditionalDeclareContents(node), + }, true, nil +} + +// getImportAdditionalDeclareContents provides the additional declares that a custom component might need. +func (m *ComponentNodeManager) getImportAdditionalDeclareContents(node *ImportConfigNode) map[string]string { + additionalDeclareContents := make(map[string]string, len(node.ImportedDeclares())) + for importedDeclareLabel, importedDeclare := range node.ImportedDeclares() { + additionalDeclareContents[importedDeclareLabel] = importedDeclare.Content + } + return additionalDeclareContents +} + +// getLocalAdditionalDeclareContents provides the additional declares that a custom component might need. +func (m *ComponentNodeManager) getLocalAdditionalDeclareContents(componentName string) map[string]string { + additionalDeclareContents := make(map[string]string) + for _, customComponentDependency := range m.customComponentDependencies[componentName] { + if customComponentDependency.importNode != nil { + for importedDeclareLabel, importedDeclare := range customComponentDependency.importNode.ImportedDeclares() { + // The label of the importNode is added as a prefix to the declare label to create a scope. + // This is useful in the scenario where a custom component of an imported declare is defined inside of a local declare. + // In this case, this custom component should only have have access to the imported declares of its corresponding import node. + additionalDeclareContents[customComponentDependency.importNode.label+"."+importedDeclareLabel] = importedDeclare.Content + } + } else if customComponentDependency.declareNode != nil { + additionalDeclareContents[customComponentDependency.declareLabel] = customComponentDependency.declareNode.Declare().Content + } else { + additionalDeclareContents[customComponentDependency.componentName] = m.parentDeclareContents[customComponentDependency.componentName] + } + } + return additionalDeclareContents +} + +// filterParentDeclareContents prevents custom components from accessing declared content out of their scope. +func filterParentDeclareContents(importLabel string, parentDeclareContents map[string]string) map[string]string { + filteredParentDeclareContents := make(map[string]string) + for declareLabel, declareContent := range parentDeclareContents { + // The scope is defined by the importLabel prefix in the declareLabel of the declare block. + if strings.HasPrefix(declareLabel, importLabel) { + filteredParentDeclareContents[strings.TrimPrefix(declareLabel, importLabel+".")] = declareContent + } + } + return filteredParentDeclareContents +} + +// CustomComponentDependency represents a dependency that a custom component has to a declare block. +type CustomComponentDependency struct { + componentName string + importLabel string + declareLabel string + importNode *ImportConfigNode + declareNode *DeclareNode +} + +// FindCustomComponentDependencies traverses the AST of the provided declare and collects references to known custom components. +// Panics if declare is nil. +func (m *ComponentNodeManager) FindCustomComponentDependencies(declare *Declare) ([]CustomComponentDependency, error) { + uniqueReferences := make(map[string]CustomComponentDependency) + m.findCustomComponentDependencies(declare.Block.Body, uniqueReferences) + + references := make([]CustomComponentDependency, 0, len(uniqueReferences)) + for _, ref := range uniqueReferences { + references = append(references, ref) + } + + return references, nil +} + +func (m *ComponentNodeManager) findCustomComponentDependencies(stmts ast.Body, uniqueReferences map[string]CustomComponentDependency) { + for _, stmt := range stmts { + switch stmt := stmt.(type) { + case *ast.BlockStmt: + componentName := strings.Join(stmt.Name, ".") + switch componentName { + case "declare": + m.findCustomComponentDependencies(stmt.Body, uniqueReferences) + default: + potentialImportLabel, potentialDeclareLabel := ExtractImportAndDeclareLabels(componentName) + if declareNode, ok := m.declareNodes[potentialDeclareLabel]; ok { + uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: "", declareLabel: potentialDeclareLabel, declareNode: declareNode} + } else if importNode, ok := m.importNodes[potentialImportLabel]; ok { + uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel, importNode: importNode} + } else if _, ok := m.parentDeclareContents[componentName]; ok { + uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel} + } + } + } + } +} diff --git a/pkg/flow/internal/controller/custom_component_config.go b/pkg/flow/internal/controller/custom_component_config.go deleted file mode 100644 index 303da2cb4912..000000000000 --- a/pkg/flow/internal/controller/custom_component_config.go +++ /dev/null @@ -1,105 +0,0 @@ -package controller - -import ( - "fmt" - "strings" -) - -type CustomComponentConfig struct { - declareContent string - additionalDeclareContents map[string]string -} - -func getLocalCustomComponentConfig( - declareNodes map[string]*DeclareNode, - customComponentDependencies map[string][]CustomComponentDependency, - parentDeclareContents map[string]string, - componentName string, - declareLabel string, -) (CustomComponentConfig, error) { - - var customComponentConfig CustomComponentConfig - var err error - - if node, exists := declareNodes[declareLabel]; exists { - customComponentConfig.additionalDeclareContents, err = getLocalAdditionalDeclareContents(componentName, customComponentDependencies, parentDeclareContents) - if err != nil { - return customComponentConfig, err - } - customComponentConfig.declareContent = node.Declare().Content - } else if declareContent, ok := parentDeclareContents[componentName]; ok { - customComponentConfig.additionalDeclareContents = parentDeclareContents - customComponentConfig.declareContent = declareContent - } else { - return customComponentConfig, fmt.Errorf("could not find a corresponding declare for the custom component %s", componentName) - } - return customComponentConfig, nil -} - -func getLocalAdditionalDeclareContents(componentName string, - customComponentDependencies map[string][]CustomComponentDependency, - parentDeclareContents map[string]string, -) (map[string]string, error) { - - additionalDeclareContents := make(map[string]string) - for _, customComponentDependency := range customComponentDependencies[componentName] { - if customComponentDependency.importNode != nil { - for importedDeclareLabel, importedDeclare := range customComponentDependency.importNode.ImportedDeclares() { - // The label of the importNode is added as a prefix to the declare label to create a scope. - // This is useful in the scenario where a custom component of an imported declare is defined inside of a local declare. - // In this case, this custom component should only have have access to the imported declares of its corresponding import node. - additionalDeclareContents[customComponentDependency.importNode.label+"."+importedDeclareLabel] = importedDeclare.Content - } - } else if customComponentDependency.declareNode != nil { - additionalDeclareContents[customComponentDependency.declareLabel] = customComponentDependency.declareNode.Declare().Content - } else { - // Nested declares have access to declare contents defined in their parents. - if declareContent, ok := parentDeclareContents[customComponentDependency.componentName]; ok { - additionalDeclareContents[customComponentDependency.componentName] = declareContent - } else { - return additionalDeclareContents, fmt.Errorf("could not find the required declare content %s for the custom component %s", customComponentDependency.componentName, componentName) - } - } - } - return additionalDeclareContents, nil -} - -func getImportedCustomComponentConfig( - importNodes map[string]*ImportConfigNode, - parentDeclareContents map[string]string, - componentName string, - declareLabel string, - importLabel string, -) (CustomComponentConfig, error) { - - var customComponentConfig CustomComponentConfig - if node, exists := importNodes[importLabel]; exists { - customComponentConfig.additionalDeclareContents = make(map[string]string, len(node.ImportedDeclares())) - for importedDeclareLabel, importedDeclare := range node.ImportedDeclares() { - customComponentConfig.additionalDeclareContents[importedDeclareLabel] = importedDeclare.Content - } - declare, err := node.GetImportedDeclareByLabel(declareLabel) - if err != nil { - return customComponentConfig, err - } - customComponentConfig.declareContent = declare.Content - } else if declareContent, ok := parentDeclareContents[componentName]; ok { - customComponentConfig.additionalDeclareContents = filterParentDeclareContents(importLabel, parentDeclareContents) - customComponentConfig.declareContent = declareContent - } else { - return customComponentConfig, fmt.Errorf("could not find a corresponding imported declare for the custom component %s", componentName) - } - return customComponentConfig, nil -} - -// filterParentDeclareContents prevents custom components from accessing declared content out of their scope. -func filterParentDeclareContents(importLabel string, parentDeclareContents map[string]string) map[string]string { - filteredParentDeclareContents := make(map[string]string) - for declareLabel, declareContent := range parentDeclareContents { - // The scope is defined by the importLabel prefix in the declareLabel of the declare block. - if strings.HasPrefix(declareLabel, importLabel) { - filteredParentDeclareContents[strings.TrimPrefix(declareLabel, importLabel+".")] = declareContent - } - } - return filteredParentDeclareContents -} diff --git a/pkg/flow/internal/controller/custom_component_dependencies.go b/pkg/flow/internal/controller/custom_component_dependencies.go deleted file mode 100644 index ef6377707f77..000000000000 --- a/pkg/flow/internal/controller/custom_component_dependencies.go +++ /dev/null @@ -1,63 +0,0 @@ -package controller - -import ( - "strings" - - "github.com/grafana/river/ast" -) - -type CustomComponentDependency struct { - componentName string - importLabel string - declareLabel string - importNode *ImportConfigNode - declareNode *DeclareNode -} - -// GetCustomComponentDependencies traverses the AST of the provided declare and collects references to known custom components. -// Panics if declare is nil. -func GetCustomComponentDependencies( - declare *Declare, - importNodes map[string]*ImportConfigNode, - declareNodes map[string]*DeclareNode, - parentDeclareContents map[string]string, -) ([]CustomComponentDependency, error) { - - uniqueReferences := make(map[string]CustomComponentDependency) - getCustomComponentDependencies(declare.Block.Body, importNodes, declareNodes, uniqueReferences, parentDeclareContents) - - references := make([]CustomComponentDependency, 0, len(uniqueReferences)) - for _, ref := range uniqueReferences { - references = append(references, ref) - } - - return references, nil -} - -func getCustomComponentDependencies( - stmts ast.Body, - importNodes map[string]*ImportConfigNode, - declareNodes map[string]*DeclareNode, - uniqueReferences map[string]CustomComponentDependency, - parentDeclareContents map[string]string, -) { - for _, stmt := range stmts { - switch stmt := stmt.(type) { - case *ast.BlockStmt: - componentName := strings.Join(stmt.Name, ".") - switch componentName { - case "declare": - getCustomComponentDependencies(stmt.Body, importNodes, declareNodes, uniqueReferences, parentDeclareContents) - default: - potentialImportLabel, potentialDeclareLabel := ExtractImportAndDeclareLabels(componentName) - if declareNode, ok := declareNodes[potentialDeclareLabel]; ok { - uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: "", declareLabel: potentialDeclareLabel, declareNode: declareNode} - } else if importNode, ok := importNodes[potentialImportLabel]; ok { - uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel, importNode: importNode} - } else if _, ok := parentDeclareContents[componentName]; ok { - uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel} - } - } - } - } -} diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 33ae20d7af0e..593b17b2b5f8 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strings" "sync" "time" @@ -25,28 +24,24 @@ import ( // The Loader builds and evaluates ComponentNodes from River blocks. type Loader struct { - log log.Logger - tracer trace.TracerProvider - globals ComponentGlobals - services []service.Service - host service.Host - componentReg ComponentRegistry - workerPool worker.Pool + log log.Logger + tracer trace.TracerProvider + globals ComponentGlobals + services []service.Service + host service.Host + workerPool worker.Pool // backoffConfig is used to backoff when an updated component's dependencies cannot be submitted to worker // pool for evaluation in EvaluateDependants, because the queue is full. This is an unlikely scenario, but when // it happens we should avoid retrying too often to give other goroutines a chance to progress. Having a backoff // also prevents log spamming with errors. backoffConfig backoff.Config - mut sync.RWMutex - graph *dag.Graph - originalGraph *dag.Graph - componentNodes []ComponentNode - serviceNodes []*ServiceNode - importNodes map[string]*ImportConfigNode - declareNodes map[string]*DeclareNode - parentDeclareContents map[string]string - customComponentDependencies map[string][]CustomComponentDependency + mut sync.RWMutex + graph *dag.Graph + originalGraph *dag.Graph + componentNodes []ComponentNode + serviceNodes []*ServiceNode + componentNodeManager *ComponentNodeManager cache *valueCache blocks []*ast.BlockStmt // Most recently loaded blocks, used for writing @@ -81,16 +76,13 @@ func NewLoader(opts LoaderOptions) *Loader { } l := &Loader{ - log: log.With(globals.Logger, "controller_id", globals.ControllerID), - tracer: tracing.WrapTracerForLoader(globals.TraceProvider, globals.ControllerID), - globals: globals, - services: services, - host: host, - componentReg: reg, - workerPool: opts.WorkerPool, - importNodes: map[string]*ImportConfigNode{}, - declareNodes: map[string]*DeclareNode{}, - customComponentDependencies: map[string][]CustomComponentDependency{}, + log: log.With(globals.Logger, "controller_id", globals.ControllerID), + tracer: tracing.WrapTracerForLoader(globals.TraceProvider, globals.ControllerID), + globals: globals, + services: services, + host: host, + workerPool: opts.WorkerPool, + componentNodeManager: NewComponentNodeManager(globals, reg), // This is a reasonable default which should work for most cases. If a component is completely stuck, we would // retry and log an error every 10 seconds, at most. @@ -138,7 +130,8 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co } l.cache.SyncModuleArgs(args) - l.parentDeclareContents = parentDeclareContents + // Reload the component node manager when a new config is applied. + l.componentNodeManager.OnReload(parentDeclareContents) newGraph, diags := l.loadNewGraph(args, componentBlocks, configBlocks, declares) if diags.HasErrors() { return diags @@ -263,10 +256,6 @@ func (l *Loader) Cleanup(stopWorkerPool bool) { func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare) (dag.Graph, diag.Diagnostics) { var g dag.Graph - // Reset the custom component dependencies cache before loading a new graph. - // This step is crucial to ensure that references to outdated declares, not included in the new configuration, are removed. - l.customComponentDependencies = make(map[string][]CustomComponentDependency) - // Split component blocks into blocks for components and services. componentBlocks, serviceBlocks := l.splitComponentBlocks(componentBlocks) @@ -328,7 +317,7 @@ func (l *Loader) splitComponentBlocks(blocks []*ast.BlockStmt) (componentBlocks, func (l *Loader) populateDeclareNodes(g *dag.Graph, declares []*Declare) diag.Diagnostics { var diags diag.Diagnostics - l.declareNodes = map[string]*DeclareNode{} + l.componentNodeManager.declareNodes = map[string]*DeclareNode{} for _, declare := range declares { node := NewDeclareNode(declare) if g.GetByID(node.NodeID()) != nil { @@ -339,7 +328,7 @@ func (l *Loader) populateDeclareNodes(g *dag.Graph, declares []*Declare) diag.Di continue } g.Add(node) - l.declareNodes[node.label] = node + l.componentNodeManager.declareNodes[node.label] = node } return diags } @@ -447,8 +436,7 @@ func (l *Loader) populateConfigBlockNodes(args map[string]any, g *dag.Graph, con g.Add(c) } - l.importNodes = nodeMap.importMap - + l.componentNodeManager.importNodes = nodeMap.importMap return diags } @@ -490,52 +478,31 @@ func (l *Loader) populateComponentNodes(g *dag.Graph, componentBlocks []*ast.Blo }) continue } - firstPart := strings.Split(componentName, ".")[0] - if l.shouldAddCustomComponentNode(firstPart, componentName) { - g.Add(NewCustomComponentNode(l.globals, block, l.getCustomComponentConfig)) - } else { - registration, exists := l.componentReg.Get(componentName) - if !exists { - diags.Add(diag.Diagnostic{ - Severity: diag.SeverityLevelError, - Message: fmt.Sprintf("Unrecognized component name %q", componentName), - StartPos: block.NamePos.Position(), - EndPos: block.NamePos.Add(len(componentName) - 1).Position(), - }) - continue - } - g.Add(NewBuiltinComponentNode(l.globals, registration, block)) + componentNode, err := l.componentNodeManager.CreateComponentNode(componentName, block) + if err != nil { + diags.Add(diag.Diagnostic{ + Severity: diag.SeverityLevelError, + Message: err.Error(), + StartPos: block.NamePos.Position(), + EndPos: block.NamePos.Add(len(componentName) - 1).Position(), + }) + continue } + g.Add(componentNode) } } - return diags } -func (l *Loader) shouldAddCustomComponentNode(firstPart, componentName string) bool { - _, declareExists := l.declareNodes[firstPart] - _, importExists := l.importNodes[firstPart] - _, parentDeclareContentExists := l.parentDeclareContents[componentName] - - return declareExists || importExists || parentDeclareContentExists -} - -func (l *Loader) wireCustomComponentDependencies(g *dag.Graph, dc *CustomComponentNode, declareNode *DeclareNode) error { - var dependencies []CustomComponentDependency - if deps, ok := l.customComponentDependencies[declareNode.label]; ok { - dependencies = deps - } else { - var err error - dependencies, err = GetCustomComponentDependencies(declareNode.Declare(), l.importNodes, l.declareNodes, l.parentDeclareContents) - if err != nil { - return err - } - l.customComponentDependencies[declareNode.label] = dependencies +func (l *Loader) wireCustomComponentDependencies(g *dag.Graph, cc *CustomComponentNode, declareNode *DeclareNode) error { + dependencies, err := l.componentNodeManager.getCustomComponentDependencies(declareNode) + if err != nil { + return err } // Add edges between the CustomComponentNode and all import nodes that it needs. for _, dependency := range dependencies { if dependency.importNode != nil { - g.AddEdge(dag.Edge{From: dc, To: dependency.importNode}) + g.AddEdge(dag.Edge{From: cc, To: dependency.importNode}) } } return nil @@ -588,14 +555,14 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { return diags } -func (l *Loader) wireCustomComponentNode(g *dag.Graph, dc *CustomComponentNode) error { - if declareNode, exists := l.declareNodes[dc.declareLabel]; exists { - err := l.wireCustomComponentDependencies(g, dc, declareNode) +func (l *Loader) wireCustomComponentNode(g *dag.Graph, cc *CustomComponentNode) error { + if declareNode, exists := l.componentNodeManager.GetCorrespondingLocalDeclare(cc); exists { + err := l.wireCustomComponentDependencies(g, cc, declareNode) if err != nil { return err } - } else if importNode, exists := l.importNodes[dc.importLabel]; exists { - g.AddEdge(dag.Edge{From: dc, To: importNode}) + } else if importNode, exists := l.componentNodeManager.GetCorrespondingImportedDeclare(cc); exists { + g.AddEdge(dag.Edge{From: cc, To: importNode}) } return nil } @@ -623,7 +590,7 @@ func (l *Loader) Services() []*ServiceNode { func (l *Loader) Imports() map[string]*ImportConfigNode { l.mut.RLock() defer l.mut.RUnlock() - return l.importNodes + return l.componentNodeManager.importNodes } // Graph returns a copy of the DAG managed by the Loader. @@ -809,13 +776,6 @@ func (l *Loader) postEvaluate(logger log.Logger, bn BlockNode, err error) error return nil } -func (l *Loader) getCustomComponentConfig(componentName string, importLabel string, declareLabel string) (CustomComponentConfig, error) { - if importLabel == "" { - return getLocalCustomComponentConfig(l.declareNodes, l.customComponentDependencies, l.parentDeclareContents, componentName, declareLabel) - } - return getImportedCustomComponentConfig(l.importNodes, l.parentDeclareContents, componentName, declareLabel, importLabel) -} - func multierrToDiags(errors error) diag.Diagnostics { var diags diag.Diagnostics for _, err := range errors.(*multierror.Error).Errors { diff --git a/pkg/flow/internal/controller/node_custom_component.go b/pkg/flow/internal/controller/node_custom_component.go index 921165256be5..f5d0883cc96b 100644 --- a/pkg/flow/internal/controller/node_custom_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -38,7 +38,7 @@ type CustomComponentNode struct { moduleController ModuleController OnNodeWithDependantsUpdate func(cn NodeWithDependants) // Informs controller that we need to reevaluate - GetCustomComponentConfig func(fullName string, importLabel string, declareLabel string) (CustomComponentConfig, error) // Retrieve the custom component config. + GetCustomComponentConfig func(*CustomComponentNode) (CustomComponentConfig, error) // Retrieve the custom component config. lastUpdateTime atomic.Time mut sync.RWMutex @@ -81,7 +81,7 @@ var _ ComponentNode = (*CustomComponentNode)(nil) // NewCustomComponentNode creates a new CustomComponentNode from an initial ast.BlockStmt. // The underlying managed custom component isn't created until Evaluate is called. -func NewCustomComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetCustomComponentConfig func(string, string, string) (CustomComponentConfig, error)) *CustomComponentNode { +func NewCustomComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetCustomComponentConfig func(*CustomComponentNode) (CustomComponentConfig, error)) *CustomComponentNode { var ( id = BlockComponentID(b) nodeID = id.String() @@ -216,9 +216,9 @@ func (cn *CustomComponentNode) evaluate(scope *vm.Scope) error { cn.managed = managed } - customComponentConfig, err := cn.GetCustomComponentConfig(cn.componentName, cn.importLabel, cn.declareLabel) + customComponentConfig, err := cn.GetCustomComponentConfig(cn) if err != nil { - return fmt.Errorf("retrieving custom component info: %w", err) + return fmt.Errorf("retrieving custom component config: %w", err) } // Reload the custom component with new config diff --git a/pkg/flow/module_import_test.go b/pkg/flow/module_import_test.go index 87f182bbac7f..79332838cd82 100644 --- a/pkg/flow/module_import_test.go +++ b/pkg/flow/module_import_test.go @@ -713,7 +713,7 @@ func TestImportModuleError(t *testing.T) { input = testImport.test.myModule.output } `, - expectedError: `Unrecognized component name "cantAccessThis"`, + expectedError: `unrecognized component name "cantAccessThis"`, }, // TODO: add more tests } From cb983c56ab6cd95c8d0849eb5ac0122b1d58f866 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 16 Jan 2024 17:46:55 +0100 Subject: [PATCH 08/36] use deprecation notice on module component constructor --- component/module/file/file.go | 2 +- component/module/git/git.go | 2 +- component/module/http/http.go | 2 +- component/module/module.go | 9 +++++---- component/module/string/string.go | 2 +- pkg/flow/componenttest/testfailmodule.go | 2 +- pkg/flow/internal/controller/node_custom_component.go | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/component/module/file/file.go b/component/module/file/file.go index fdc19804eb69..f939821ffb25 100644 --- a/component/module/file/file.go +++ b/component/module/file/file.go @@ -58,7 +58,7 @@ var ( // New creates a new module.file component. func New(o component.Options, args Arguments) (*Component, error) { - m, err := module.NewModuleComponentDeprecated(o) + m, err := module.NewModuleComponent(o) if err != nil { return nil, err } diff --git a/component/module/git/git.go b/component/module/git/git.go index b30569b67bb5..0565bc7c43b5 100644 --- a/component/module/git/git.go +++ b/component/module/git/git.go @@ -74,7 +74,7 @@ var ( // New creates a new module.git component. func New(o component.Options, args Arguments) (*Component, error) { - m, err := module.NewModuleComponentDeprecated(o) + m, err := module.NewModuleComponent(o) if err != nil { return nil, err } diff --git a/component/module/http/http.go b/component/module/http/http.go index 39855b9b8595..c7906b380c65 100644 --- a/component/module/http/http.go +++ b/component/module/http/http.go @@ -57,7 +57,7 @@ var ( // New creates a new module.http component. func New(o component.Options, args Arguments) (*Component, error) { - m, err := module.NewModuleComponentDeprecated(o) + m, err := module.NewModuleComponent(o) if err != nil { return nil, err } diff --git a/component/module/module.go b/component/module/module.go index 5271a497b907..7ff3af8244e3 100644 --- a/component/module/module.go +++ b/component/module/module.go @@ -30,8 +30,9 @@ type Exports struct { Exports map[string]any `river:"exports,block"` } -// NewModuleComponent initializes a new ModuleComponent. -func NewModuleComponent(o component.Options) (*ModuleComponent, error) { +// NewModuleComponentV2 initializes a new ModuleComponent. +// Compared to the previous constructor, the export is simply map[string]any instead of the Exports type containing the map. +func NewModuleComponentV2(o component.Options) (*ModuleComponent, error) { c := &ModuleComponent{ opts: o, } @@ -42,8 +43,8 @@ func NewModuleComponent(o component.Options) (*ModuleComponent, error) { return c, err } -// TODO: Remove when getting rid of old modules -func NewModuleComponentDeprecated(o component.Options) (*ModuleComponent, error) { +// Deprecated: Use NewModuleComponentV2 instead. +func NewModuleComponent(o component.Options) (*ModuleComponent, error) { c := &ModuleComponent{ opts: o, } diff --git a/component/module/string/string.go b/component/module/string/string.go index 65e2cdf6190d..ff5266b212f2 100644 --- a/component/module/string/string.go +++ b/component/module/string/string.go @@ -42,7 +42,7 @@ var ( // New creates a new module.string component. func New(o component.Options, args Arguments) (*Component, error) { - m, err := module.NewModuleComponentDeprecated(o) + m, err := module.NewModuleComponent(o) if err != nil { return nil, err } diff --git a/pkg/flow/componenttest/testfailmodule.go b/pkg/flow/componenttest/testfailmodule.go index 3481965ef9f5..2f39b3c6f9e5 100644 --- a/pkg/flow/componenttest/testfailmodule.go +++ b/pkg/flow/componenttest/testfailmodule.go @@ -15,7 +15,7 @@ func init() { Exports: mod.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { - m, err := mod.NewModuleComponentDeprecated(opts) + m, err := mod.NewModuleComponent(opts) if err != nil { return nil, err } diff --git a/pkg/flow/internal/controller/node_custom_component.go b/pkg/flow/internal/controller/node_custom_component.go index f5d0883cc96b..9fc0c21f3320 100644 --- a/pkg/flow/internal/controller/node_custom_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -209,7 +209,7 @@ func (cn *CustomComponentNode) evaluate(scope *vm.Scope) error { if cn.managed == nil { // We haven't built the managed custom component successfully yet. - managed, err := module.NewModuleComponent(cn.managedOpts) + managed, err := module.NewModuleComponentV2(cn.managedOpts) if err != nil { return fmt.Errorf("building custom component: %w", err) } From e7fb05e09691b101e7fbe032af2e25eff93eab87 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 16 Jan 2024 17:58:27 +0100 Subject: [PATCH 09/36] fix test after changing error message --- pkg/flow/internal/controller/loader_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/flow/internal/controller/loader_test.go b/pkg/flow/internal/controller/loader_test.go index cac91b7926bb..e016b8fead26 100644 --- a/pkg/flow/internal/controller/loader_test.go +++ b/pkg/flow/internal/controller/loader_test.go @@ -129,7 +129,7 @@ func TestLoader(t *testing.T) { ` l := controller.NewLoader(newLoaderOptions()) diags := applyFromContent(t, l, []byte(invalidFile), nil) - require.ErrorContains(t, diags.ErrorOrNil(), `Unrecognized component name "doesnotexist`) + require.ErrorContains(t, diags.ErrorOrNil(), `unrecognized component name "doesnotexist`) }) t.Run("Partial load with invalid reference", func(t *testing.T) { From e2f9c99cde644b4e59bd27eac58552d7a6a83d3b Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 16 Jan 2024 18:21:19 +0100 Subject: [PATCH 10/36] custom component should return the managed component via Component() --- component/module/module.go | 13 +++++++++++++ .../internal/controller/node_custom_component.go | 7 +++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/component/module/module.go b/component/module/module.go index 7ff3af8244e3..469c5afa2b15 100644 --- a/component/module/module.go +++ b/component/module/module.go @@ -30,6 +30,8 @@ type Exports struct { Exports map[string]any `river:"exports,block"` } +var _ component.Component = (*ModuleComponent)(nil) + // NewModuleComponentV2 initializes a new ModuleComponent. // Compared to the previous constructor, the export is simply map[string]any instead of the Exports type containing the map. func NewModuleComponentV2(o component.Options) (*ModuleComponent, error) { @@ -86,6 +88,17 @@ func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue strin return nil } +// Run implements component.Component. +func (c *ModuleComponent) Run(ctx context.Context) error { + <-ctx.Done() + return nil +} + +// Update implements component.Component. +func (c *ModuleComponent) Update(_ component.Arguments) error { + return nil +} + // RunFlowController runs the flow controller that all module components start. func (c *ModuleComponent) RunFlowController(ctx context.Context) { err := c.mod.Run(ctx) diff --git a/pkg/flow/internal/controller/node_custom_component.go b/pkg/flow/internal/controller/node_custom_component.go index 9fc0c21f3320..54f7a41e8bbe 100644 --- a/pkg/flow/internal/controller/node_custom_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -355,9 +355,12 @@ func (cn *CustomComponentNode) BlockName() string { return cn.componentName } -// This node does not manage any component. +// Component returns the instance of the managed component. Component may be +// nil if the CustomComponentNode has not been successfully evaluated yet. func (cn *CustomComponentNode) Component() component.Component { - return nil + cn.mut.RLock() + defer cn.mut.RUnlock() + return cn.managed } // Registry returns the prometheus registry of the component. From 3c24a3c55d13ea5e1a8d4a854c90cda520f561c8 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 12:10:48 +0100 Subject: [PATCH 11/36] introduces LoaderConfigOptions used to pass options like additional declare contents when loading a new config --- cmd/internal/flowmode/cmd_run.go | 2 +- component/module/file/file.go | 5 +-- component/module/git/git.go | 3 +- component/module/http/http.go | 5 +-- component/module/module.go | 33 ++++++++++--------- component/module/string/string.go | 3 +- component/registry.go | 3 +- converter/internal/test_common/testing.go | 2 +- pkg/flow/componenttest/testfailmodule.go | 5 +-- pkg/flow/config/loader.go | 13 ++++++++ pkg/flow/flow.go | 9 +++-- pkg/flow/flow_services_test.go | 15 +++++---- pkg/flow/flow_test.go | 2 +- pkg/flow/flow_updates_test.go | 10 +++--- .../controller/component_node_manager.go | 32 +++++++++--------- pkg/flow/internal/controller/loader.go | 5 +-- pkg/flow/internal/controller/loader_test.go | 3 +- .../controller/node_custom_component.go | 7 +++- pkg/flow/module.go | 7 ++-- pkg/flow/module_caching_test.go | 4 +-- pkg/flow/module_declare_test.go | 2 +- pkg/flow/module_fail_test.go | 2 +- pkg/flow/module_import_test.go | 4 +-- pkg/flow/module_test.go | 9 ++--- pkg/flow/source_test.go | 4 +-- 25 files changed, 113 insertions(+), 76 deletions(-) create mode 100644 pkg/flow/config/loader.go diff --git a/cmd/internal/flowmode/cmd_run.go b/cmd/internal/flowmode/cmd_run.go index c70905dfcb1f..8bd2d4cebbaa 100644 --- a/cmd/internal/flowmode/cmd_run.go +++ b/cmd/internal/flowmode/cmd_run.go @@ -274,7 +274,7 @@ func (fr *flowRun) Run(configPath string) error { if err != nil { return nil, fmt.Errorf("reading config path %q: %w", configPath, err) } - if err := f.LoadSource(flowSource, nil, nil); err != nil { + if err := f.LoadSource(flowSource, nil); err != nil { return flowSource, fmt.Errorf("error during the initial grafana/agent load: %w", err) } diff --git a/component/module/file/file.go b/component/module/file/file.go index f939821ffb25..d8aeecfb4931 100644 --- a/component/module/file/file.go +++ b/component/module/file/file.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/agent/component" "github.com/grafana/agent/component/local/file" "github.com/grafana/agent/component/module" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/river/rivertypes" ) @@ -88,7 +89,7 @@ func (c *Component) newManagedLocalComponent(o component.Options) (*file.Compone if !c.inUpdate.Load() && c.isCreated.Load() { // Any errors found here are reported via component health - _ = c.mod.LoadFlowSource(c.getArgs().Arguments, c.getContent().Value, nil) + _ = c.mod.LoadFlowSource(c.getArgs().Arguments, c.getContent().Value, config.DefaultLoaderConfigOptions()) } } @@ -135,7 +136,7 @@ func (c *Component) Update(args component.Arguments) error { // Force a content load here and bubble up any error. This will catch problems // on initial load. - return c.mod.LoadFlowSource(newArgs.Arguments, c.getContent().Value, nil) + return c.mod.LoadFlowSource(newArgs.Arguments, c.getContent().Value, config.DefaultLoaderConfigOptions()) } // CurrentHealth implements component.HealthComponent. diff --git a/component/module/git/git.go b/component/module/git/git.go index 0565bc7c43b5..caf1908367cc 100644 --- a/component/module/git/git.go +++ b/component/module/git/git.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/agent/component" "github.com/grafana/agent/component/module" "github.com/grafana/agent/internal/vcs" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/logging/level" ) @@ -239,7 +240,7 @@ func (c *Component) pollFile(ctx context.Context, args Arguments) error { return err } - return c.mod.LoadFlowSource(args.Arguments, string(bb), nil) + return c.mod.LoadFlowSource(args.Arguments, string(bb), config.DefaultLoaderConfigOptions()) } // CurrentHealth implements component.HealthComponent. diff --git a/component/module/http/http.go b/component/module/http/http.go index c7906b380c65..f9c4ae598920 100644 --- a/component/module/http/http.go +++ b/component/module/http/http.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/agent/component" "github.com/grafana/agent/component/module" remote_http "github.com/grafana/agent/component/remote/http" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/river/rivertypes" ) @@ -87,7 +88,7 @@ func (c *Component) newManagedLocalComponent(o component.Options) (*remote_http. if !c.inUpdate.Load() && c.isCreated.Load() { // Any errors found here are reported via component health - _ = c.mod.LoadFlowSource(c.getArgs().Arguments, c.getContent().Value, nil) + _ = c.mod.LoadFlowSource(c.getArgs().Arguments, c.getContent().Value, config.DefaultLoaderConfigOptions()) } } @@ -134,7 +135,7 @@ func (c *Component) Update(args component.Arguments) error { // Force a content load here and bubble up any error. This will catch problems // on initial load. - return c.mod.LoadFlowSource(newArgs.Arguments, c.getContent().Value, nil) + return c.mod.LoadFlowSource(newArgs.Arguments, c.getContent().Value, config.DefaultLoaderConfigOptions()) } // CurrentHealth implements component.HealthComponent. diff --git a/component/module/module.go b/component/module/module.go index 469c5afa2b15..25ccea208028 100644 --- a/component/module/module.go +++ b/component/module/module.go @@ -8,6 +8,7 @@ import ( "time" "github.com/grafana/agent/component" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/logging/level" ) @@ -16,11 +17,11 @@ type ModuleComponent struct { opts component.Options mod component.Module - mut sync.RWMutex - health component.Health - latestContent string - latestArgs map[string]any - latestParentDeclareContents map[string]string + mut sync.RWMutex + health component.Health + latestContent string + latestArgs map[string]any + latestLoaderConfigOptions config.LoaderConfigOptions } // Exports holds values which are exported from the run module. @@ -36,7 +37,8 @@ var _ component.Component = (*ModuleComponent)(nil) // Compared to the previous constructor, the export is simply map[string]any instead of the Exports type containing the map. func NewModuleComponentV2(o component.Options) (*ModuleComponent, error) { c := &ModuleComponent{ - opts: o, + opts: o, + latestLoaderConfigOptions: config.DefaultLoaderConfigOptions(), } var err error c.mod, err = o.ModuleController.NewModule("", func(exports map[string]any) { @@ -48,7 +50,8 @@ func NewModuleComponentV2(o component.Options) (*ModuleComponent, error) { // Deprecated: Use NewModuleComponentV2 instead. func NewModuleComponent(o component.Options) (*ModuleComponent, error) { c := &ModuleComponent{ - opts: o, + opts: o, + latestLoaderConfigOptions: config.DefaultLoaderConfigOptions(), } var err error c.mod, err = o.ModuleController.NewModule("", func(exports map[string]any) { @@ -60,12 +63,12 @@ func NewModuleComponent(o component.Options) (*ModuleComponent, error) { // LoadFlowSource loads the flow controller with the current component source. // It will set the component health in addition to return the error so that the consumer can rely on either or both. // If the content is the same as the last time it was successfully loaded, it will not be reloaded. -func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue string, parentDeclareContents map[string]string) error { - if reflect.DeepEqual(args, c.getLatestArgs()) && contentValue == c.getLatestContent() && reflect.DeepEqual(args, c.getLatestParentDeclareContents()) { +func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue string, options config.LoaderConfigOptions) error { + if reflect.DeepEqual(args, c.getLatestArgs()) && contentValue == c.getLatestContent() && reflect.DeepEqual(options, c.getLatestLoaderConfigOptions()) { return nil } - err := c.mod.LoadConfig([]byte(contentValue), args, parentDeclareContents) + err := c.mod.LoadConfig([]byte(contentValue), args, options) if err != nil { c.setHealth(component.Health{ Health: component.HealthTypeUnhealthy, @@ -78,7 +81,7 @@ func (c *ModuleComponent) LoadFlowSource(args map[string]any, contentValue strin c.setLatestArgs(args) c.setLatestContent(contentValue) - c.setLatestParentDeclareContents(parentDeclareContents) + c.setLatestLoaderConfigOptions(options) c.setHealth(component.Health{ Health: component.HealthTypeHealthy, Message: "module content loaded", @@ -133,16 +136,16 @@ func (c *ModuleComponent) getLatestContent() string { return c.latestContent } -func (c *ModuleComponent) setLatestParentDeclareContents(parentDeclareContents map[string]string) { +func (c *ModuleComponent) setLatestLoaderConfigOptions(options config.LoaderConfigOptions) { c.mut.Lock() defer c.mut.Unlock() - c.latestParentDeclareContents = parentDeclareContents + c.latestLoaderConfigOptions = options } -func (c *ModuleComponent) getLatestParentDeclareContents() map[string]string { +func (c *ModuleComponent) getLatestLoaderConfigOptions() config.LoaderConfigOptions { c.mut.RLock() defer c.mut.RUnlock() - return c.latestParentDeclareContents + return c.latestLoaderConfigOptions } func (c *ModuleComponent) setLatestArgs(args map[string]any) { diff --git a/component/module/string/string.go b/component/module/string/string.go index ff5266b212f2..9ef2d9cb63df 100644 --- a/component/module/string/string.go +++ b/component/module/string/string.go @@ -5,6 +5,7 @@ import ( "github.com/grafana/agent/component" "github.com/grafana/agent/component/module" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/river/rivertypes" ) @@ -66,7 +67,7 @@ func (c *Component) Run(ctx context.Context) error { func (c *Component) Update(args component.Arguments) error { newArgs := args.(Arguments) - return c.mod.LoadFlowSource(newArgs.Arguments, newArgs.Content.Value, nil) + return c.mod.LoadFlowSource(newArgs.Arguments, newArgs.Content.Value, config.DefaultLoaderConfigOptions()) } // CurrentHealth implements component.HealthComponent. diff --git a/component/registry.go b/component/registry.go index 81ead2f9e0d7..0d348d61074d 100644 --- a/component/registry.go +++ b/component/registry.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/go-kit/log" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/regexp" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/trace" @@ -44,7 +45,7 @@ type Module interface { // LoadConfig parses River config and loads it into the Module. // LoadConfig can be called multiple times, and called prior to // [Module.Run]. - LoadConfig(config []byte, args map[string]any, moduleDefinitions map[string]string) error + LoadConfig(config []byte, args map[string]any, options config.LoaderConfigOptions) error // Run starts the Module. No components within the Module // will be run until Run is called. diff --git a/converter/internal/test_common/testing.go b/converter/internal/test_common/testing.go index 3d6a04c03880..03855fc2ca31 100644 --- a/converter/internal/test_common/testing.go +++ b/converter/internal/test_common/testing.go @@ -198,7 +198,7 @@ func attemptLoadingFlowConfig(t *testing.T, river []byte) { labelstore.New(nil, prometheus.DefaultRegisterer), }, }) - err = f.LoadSource(cfg, nil, nil) + err = f.LoadSource(cfg, nil) // Many components will fail to build as e.g. the cert files are missing, so we ignore these errors. // This is not ideal, but we still validate for other potential issues. diff --git a/pkg/flow/componenttest/testfailmodule.go b/pkg/flow/componenttest/testfailmodule.go index 2f39b3c6f9e5..31c7e9350fa2 100644 --- a/pkg/flow/componenttest/testfailmodule.go +++ b/pkg/flow/componenttest/testfailmodule.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/agent/component" mod "github.com/grafana/agent/component/module" + "github.com/grafana/agent/pkg/flow/config" ) func init() { @@ -22,7 +23,7 @@ func init() { if args.(TestFailArguments).Fail { return nil, fmt.Errorf("module told to fail") } - err = m.LoadFlowSource(nil, args.(TestFailArguments).Content, nil) + err = m.LoadFlowSource(nil, args.(TestFailArguments).Content, config.DefaultLoaderConfigOptions()) if err != nil { return nil, err } @@ -58,7 +59,7 @@ func (t *TestFailModule) Run(ctx context.Context) error { func (t *TestFailModule) UpdateContent(content string) error { t.content = content - err := t.mc.LoadFlowSource(nil, t.content, nil) + err := t.mc.LoadFlowSource(nil, t.content, config.DefaultLoaderConfigOptions()) return err } diff --git a/pkg/flow/config/loader.go b/pkg/flow/config/loader.go new file mode 100644 index 000000000000..539e267b212a --- /dev/null +++ b/pkg/flow/config/loader.go @@ -0,0 +1,13 @@ +package config + +// LoaderConfigOptions is used to provide a set of options when a new config is loaded. +type LoaderConfigOptions struct { + // AdditionalDeclareContents can be used to pass custom components definition to the loader. + // This is needed when a custom component is instantiated within a custom component and the corresponding + // declare of the nested custom component is imported/declared at the root. + AdditionalDeclareContents map[string]string +} + +func DefaultLoaderConfigOptions() LoaderConfigOptions { + return LoaderConfigOptions{} +} diff --git a/pkg/flow/flow.go b/pkg/flow/flow.go index 41746d59d229..8e4e5419b137 100644 --- a/pkg/flow/flow.go +++ b/pkg/flow/flow.go @@ -50,6 +50,7 @@ import ( "fmt" "sync" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/internal/controller" "github.com/grafana/agent/pkg/flow/internal/worker" "github.com/grafana/agent/pkg/flow/logging" @@ -281,11 +282,15 @@ func (f *Flow) Run(ctx context.Context) { // // The controller will only start running components after Load is called once // without any configuration errors. -func (f *Flow) LoadSource(source *Source, args map[string]any, parentDeclareContents map[string]string) error { +func (f *Flow) LoadSource(source *Source, args map[string]any) error { + return f.loadSource(source, args, config.DefaultLoaderConfigOptions()) +} + +func (f *Flow) loadSource(source *Source, args map[string]any, options config.LoaderConfigOptions) error { f.loadMut.Lock() defer f.loadMut.Unlock() - diags := f.loader.Apply(args, source.components, source.configBlocks, source.declares, parentDeclareContents) + diags := f.loader.Apply(args, source.components, source.configBlocks, source.declares, options) if !f.loadedOnce.Load() && diags.HasErrors() { // The first call to Load should not run any components if there were // errors in the configuration file. diff --git a/pkg/flow/flow_services_test.go b/pkg/flow/flow_services_test.go index 077a5c2cd4e7..19881f5c7542 100644 --- a/pkg/flow/flow_services_test.go +++ b/pkg/flow/flow_services_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/grafana/agent/component" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/internal/controller" "github.com/grafana/agent/pkg/flow/internal/testcomponents" "github.com/grafana/agent/pkg/flow/internal/testservices" @@ -37,7 +38,7 @@ func TestServices(t *testing.T) { opts.Services = append(opts.Services, svc) ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -88,7 +89,7 @@ func TestServices_Configurable(t *testing.T) { ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(f, nil, nil)) + require.NoError(t, ctrl.LoadSource(f, nil)) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -134,7 +135,7 @@ func TestServices_Configurable_Optional(t *testing.T) { ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -168,7 +169,7 @@ func TestFlow_GetServiceConsumers(t *testing.T) { ctrl := New(opts) defer cleanUpController(ctrl) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) expectConsumers := []service.Consumer{{ Type: service.ConsumerTypeService, @@ -246,7 +247,7 @@ func TestComponents_Using_Services(t *testing.T) { ComponentRegistry: registry, ModuleRegistry: newModuleRegistry(), }) - require.NoError(t, ctrl.LoadSource(f, nil, nil)) + require.NoError(t, ctrl.LoadSource(f, nil)) go ctrl.Run(ctx) require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built") @@ -276,7 +277,7 @@ func TestComponents_Using_Services_In_Modules(t *testing.T) { mod, err := opts.ModuleController.NewModule("", nil) require.NoError(t, err, "Failed to create module") - err = mod.LoadConfig([]byte(`service_consumer "example" {}`), nil, nil) + err = mod.LoadConfig([]byte(`service_consumer "example" {}`), nil, config.DefaultLoaderConfigOptions()) require.NoError(t, err, "Failed to load module config") return &testcomponents.Fake{ @@ -321,7 +322,7 @@ func TestComponents_Using_Services_In_Modules(t *testing.T) { ComponentRegistry: registry, ModuleRegistry: newModuleRegistry(), }) - require.NoError(t, ctrl.LoadSource(f, nil, nil)) + require.NoError(t, ctrl.LoadSource(f, nil)) go ctrl.Run(ctx) require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built") diff --git a/pkg/flow/flow_test.go b/pkg/flow/flow_test.go index 3217b17faff7..42f5a6077e06 100644 --- a/pkg/flow/flow_test.go +++ b/pkg/flow/flow_test.go @@ -42,7 +42,7 @@ func TestController_LoadSource_Evaluation(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) require.Len(t, ctrl.loader.Components(), 4) diff --git a/pkg/flow/flow_updates_test.go b/pkg/flow/flow_updates_test.go index cf77237b7c76..c2349928f06a 100644 --- a/pkg/flow/flow_updates_test.go +++ b/pkg/flow/flow_updates_test.go @@ -42,7 +42,7 @@ func TestController_Updates(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -122,7 +122,7 @@ func TestController_Updates_WithQueueFull(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -195,7 +195,7 @@ func TestController_Updates_WithLag(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -269,7 +269,7 @@ func TestController_Updates_WithOtherLaggingPipeline(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -338,7 +338,7 @@ func TestController_Updates_WithLaggingComponent(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/flow/internal/controller/component_node_manager.go b/pkg/flow/internal/controller/component_node_manager.go index a3b4b1571880..61f865298787 100644 --- a/pkg/flow/internal/controller/component_node_manager.go +++ b/pkg/flow/internal/controller/component_node_manager.go @@ -14,7 +14,7 @@ type ComponentNodeManager struct { globals ComponentGlobals componentReg ComponentRegistry customComponentDependencies map[string][]CustomComponentDependency - parentDeclareContents map[string]string + additionalDeclareContents map[string]string } // NewComponentNodeManager creates a new ComponentNodeManager. @@ -29,8 +29,8 @@ func NewComponentNodeManager(globals ComponentGlobals, componentReg ComponentReg } // OnReload resets the state of the component node manager. -func (m *ComponentNodeManager) OnReload(parentDeclareContents map[string]string) { - m.parentDeclareContents = parentDeclareContents +func (m *ComponentNodeManager) OnReload(additionalDeclareContents map[string]string) { + m.additionalDeclareContents = additionalDeclareContents m.customComponentDependencies = make(map[string][]CustomComponentDependency) m.importNodes = map[string]*ImportConfigNode{} m.declareNodes = map[string]*DeclareNode{} @@ -70,9 +70,9 @@ func (m *ComponentNodeManager) getCustomComponentDependencies(declareNode *Decla func (m *ComponentNodeManager) shouldAddCustomComponentNode(firstPart, componentName string) bool { _, declareExists := m.declareNodes[firstPart] _, importExists := m.importNodes[firstPart] - _, parentDeclareContentExists := m.parentDeclareContents[componentName] + _, additionalDeclareContentExists := m.additionalDeclareContents[componentName] - return declareExists || importExists || parentDeclareContentExists + return declareExists || importExists || additionalDeclareContentExists } func (m *ComponentNodeManager) GetCorrespondingLocalDeclare(cc *CustomComponentNode) (*DeclareNode, bool) { @@ -108,7 +108,7 @@ func (m *ComponentNodeManager) getCustomComponentConfig(cc *CustomComponentNode) } if !found { customComponentConfig, found = m.getCustomComponentConfigFromParent(cc) - customComponentConfig.additionalDeclareContents = filterParentDeclareContents(cc.importLabel, customComponentConfig.additionalDeclareContents) + customComponentConfig.additionalDeclareContents = filterAdditionalDeclareContents(cc.importLabel, customComponentConfig.additionalDeclareContents) } } if !found { @@ -131,13 +131,13 @@ func (m *ComponentNodeManager) getCustomComponentConfigFromLocalDeclares(cc *Cus // getCustomComponentConfigFromParent retrieves the config of a custom component from the parent controller. func (m *ComponentNodeManager) getCustomComponentConfigFromParent(cc *CustomComponentNode) (CustomComponentConfig, bool) { - declareContent, exists := m.parentDeclareContents[cc.componentName] + declareContent, exists := m.additionalDeclareContents[cc.componentName] if !exists { return CustomComponentConfig{}, false } return CustomComponentConfig{ declareContent: declareContent, - additionalDeclareContents: m.parentDeclareContents, + additionalDeclareContents: m.additionalDeclareContents, }, true } @@ -180,22 +180,22 @@ func (m *ComponentNodeManager) getLocalAdditionalDeclareContents(componentName s } else if customComponentDependency.declareNode != nil { additionalDeclareContents[customComponentDependency.declareLabel] = customComponentDependency.declareNode.Declare().Content } else { - additionalDeclareContents[customComponentDependency.componentName] = m.parentDeclareContents[customComponentDependency.componentName] + additionalDeclareContents[customComponentDependency.componentName] = m.additionalDeclareContents[customComponentDependency.componentName] } } return additionalDeclareContents } -// filterParentDeclareContents prevents custom components from accessing declared content out of their scope. -func filterParentDeclareContents(importLabel string, parentDeclareContents map[string]string) map[string]string { - filteredParentDeclareContents := make(map[string]string) - for declareLabel, declareContent := range parentDeclareContents { +// filterAdditionalDeclareContents prevents custom components from accessing declared content out of their scope. +func filterAdditionalDeclareContents(importLabel string, additionalDeclareContents map[string]string) map[string]string { + filteredAdditionalDeclareContents := make(map[string]string) + for declareLabel, declareContent := range additionalDeclareContents { // The scope is defined by the importLabel prefix in the declareLabel of the declare block. if strings.HasPrefix(declareLabel, importLabel) { - filteredParentDeclareContents[strings.TrimPrefix(declareLabel, importLabel+".")] = declareContent + filteredAdditionalDeclareContents[strings.TrimPrefix(declareLabel, importLabel+".")] = declareContent } } - return filteredParentDeclareContents + return filteredAdditionalDeclareContents } // CustomComponentDependency represents a dependency that a custom component has to a declare block. @@ -235,7 +235,7 @@ func (m *ComponentNodeManager) findCustomComponentDependencies(stmts ast.Body, u uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: "", declareLabel: potentialDeclareLabel, declareNode: declareNode} } else if importNode, ok := m.importNodes[potentialImportLabel]; ok { uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel, importNode: importNode} - } else if _, ok := m.parentDeclareContents[componentName]; ok { + } else if _, ok := m.additionalDeclareContents[componentName]; ok { uniqueReferences[componentName] = CustomComponentDependency{componentName: componentName, importLabel: potentialImportLabel, declareLabel: potentialDeclareLabel} } } diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 593b17b2b5f8..503a724113f6 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -8,6 +8,7 @@ import ( "time" "github.com/go-kit/log" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/internal/dag" "github.com/grafana/agent/pkg/flow/internal/worker" "github.com/grafana/agent/pkg/flow/logging/level" @@ -118,7 +119,7 @@ func NewLoader(opts LoaderOptions) *Loader { // The provided parentContext can be used to provide global variables and // functions to components. A child context will be constructed from the parent // to expose values of other components. -func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare, parentDeclareContents map[string]string) diag.Diagnostics { +func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare, options config.LoaderConfigOptions) diag.Diagnostics { start := time.Now() l.mut.Lock() defer l.mut.Unlock() @@ -131,7 +132,7 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co l.cache.SyncModuleArgs(args) // Reload the component node manager when a new config is applied. - l.componentNodeManager.OnReload(parentDeclareContents) + l.componentNodeManager.OnReload(options.AdditionalDeclareContents) newGraph, diags := l.loadNewGraph(args, componentBlocks, configBlocks, declares) if diags.HasErrors() { return diags diff --git a/pkg/flow/internal/controller/loader_test.go b/pkg/flow/internal/controller/loader_test.go index e016b8fead26..537870eb86fa 100644 --- a/pkg/flow/internal/controller/loader_test.go +++ b/pkg/flow/internal/controller/loader_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/grafana/agent/component" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/internal/controller" "github.com/grafana/agent/pkg/flow/internal/dag" "github.com/grafana/agent/pkg/flow/logging" @@ -245,7 +246,7 @@ func applyFromContent(t *testing.T, l *controller.Loader, componentBytes []byte, } } - applyDiags := l.Apply(nil, componentBlocks, configBlocks, declares, nil) + applyDiags := l.Apply(nil, componentBlocks, configBlocks, declares, config.DefaultLoaderConfigOptions()) diags = append(diags, applyDiags...) return diags diff --git a/pkg/flow/internal/controller/node_custom_component.go b/pkg/flow/internal/controller/node_custom_component.go index 54f7a41e8bbe..fea3eb6be89e 100644 --- a/pkg/flow/internal/controller/node_custom_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -13,6 +13,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/agent/component" "github.com/grafana/agent/component/module" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/logging/level" "github.com/grafana/agent/pkg/flow/tracing" "github.com/grafana/river/ast" @@ -221,8 +222,12 @@ func (cn *CustomComponentNode) evaluate(scope *vm.Scope) error { return fmt.Errorf("retrieving custom component config: %w", err) } + loaderConfig := config.LoaderConfigOptions{ + AdditionalDeclareContents: customComponentConfig.additionalDeclareContents, + } + // Reload the custom component with new config - if err := cn.managed.LoadFlowSource(args, customComponentConfig.declareContent, customComponentConfig.additionalDeclareContents); err != nil { + if err := cn.managed.LoadFlowSource(args, customComponentConfig.declareContent, loaderConfig); err != nil { return fmt.Errorf("updating component: %w", err) } return nil diff --git a/pkg/flow/module.go b/pkg/flow/module.go index 7c12d950d6da..689ccefae6c2 100644 --- a/pkg/flow/module.go +++ b/pkg/flow/module.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/grafana/agent/component" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/internal/controller" "github.com/grafana/agent/pkg/flow/internal/worker" "github.com/grafana/agent/pkg/flow/logging" @@ -128,12 +129,12 @@ func newModule(o *moduleOptions) *module { } // LoadConfig parses River config and loads it. -func (c *module) LoadConfig(config []byte, args map[string]any, parentDeclareContents map[string]string) error { - ff, err := ParseSource(c.o.ID, config) +func (c *module) LoadConfig(cfg []byte, args map[string]any, options config.LoaderConfigOptions) error { + ff, err := ParseSource(c.o.ID, cfg) if err != nil { return err } - return c.f.LoadSource(ff, args, parentDeclareContents) + return c.f.loadSource(ff, args, options) } // Run starts the Module. No components within the Module diff --git a/pkg/flow/module_caching_test.go b/pkg/flow/module_caching_test.go index bdfce702b6fb..e22e0583cbda 100644 --- a/pkg/flow/module_caching_test.go +++ b/pkg/flow/module_caching_test.go @@ -60,7 +60,7 @@ func TestUpdates_EmptyModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -121,7 +121,7 @@ func TestUpdates_ThroughModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/flow/module_declare_test.go b/pkg/flow/module_declare_test.go index a40d537a65a6..91abb01f59a6 100644 --- a/pkg/flow/module_declare_test.go +++ b/pkg/flow/module_declare_test.go @@ -204,7 +204,7 @@ func TestDeclare(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/flow/module_fail_test.go b/pkg/flow/module_fail_test.go index 5ff28f20c9f0..28fb0923a892 100644 --- a/pkg/flow/module_fail_test.go +++ b/pkg/flow/module_fail_test.go @@ -15,7 +15,7 @@ func TestIDRemovalIfFailedToLoad(t *testing.T) { fullContent := "test.fail.module \"t1\" { content = \"\" }" fl, err := ParseSource("test", []byte(fullContent)) require.NoError(t, err) - err = f.LoadSource(fl, nil, nil) + err = f.LoadSource(fl, nil) require.NoError(t, err) ctx := context.Background() ctx, cnc := context.WithTimeout(ctx, 600*time.Second) diff --git a/pkg/flow/module_import_test.go b/pkg/flow/module_import_test.go index 79332838cd82..0228c151ff21 100644 --- a/pkg/flow/module_import_test.go +++ b/pkg/flow/module_import_test.go @@ -629,7 +629,7 @@ func TestImportModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -734,7 +734,7 @@ func TestImportModuleError(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil, nil) + err = ctrl.LoadSource(f, nil) require.ErrorContains(t, err, tc.expectedError) }) } diff --git a/pkg/flow/module_test.go b/pkg/flow/module_test.go index 2730d77d9932..dda07b825585 100644 --- a/pkg/flow/module_test.go +++ b/pkg/flow/module_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/grafana/agent/component" + "github.com/grafana/agent/pkg/flow/config" "github.com/grafana/agent/pkg/flow/internal/worker" "github.com/grafana/agent/pkg/flow/logging" "github.com/prometheus/client_golang/prometheus" @@ -144,7 +145,7 @@ func TestArgsNotInModules(t *testing.T) { defer cleanUpController(f) fl, err := ParseSource("test", []byte("argument \"arg\"{}")) require.NoError(t, err) - err = f.LoadSource(fl, nil, nil) + err = f.LoadSource(fl, nil) require.ErrorContains(t, err, "argument blocks only allowed inside a module") } @@ -154,7 +155,7 @@ func TestExportsNotInModules(t *testing.T) { defer cleanUpController(f) fl, err := ParseSource("test", []byte("export \"arg\"{ value = 1}")) require.NoError(t, err) - err = f.LoadSource(fl, nil, nil) + err = f.LoadSource(fl, nil) require.ErrorContains(t, err, "export blocks only allowed inside a module") } @@ -165,7 +166,7 @@ func TestExportsWhenNotUsed(t *testing.T) { fullContent := "test.module \"t1\" { content = \"" + content + "\" }" fl, err := ParseSource("test", []byte(fullContent)) require.NoError(t, err) - err = f.LoadSource(fl, nil, nil) + err = f.LoadSource(fl, nil) require.NoError(t, err) ctx := context.Background() ctx, cnc := context.WithTimeout(ctx, 1*time.Second) @@ -296,7 +297,7 @@ func (t *testModule) Run(ctx context.Context) error { return err } - err = m.LoadConfig([]byte(t.content), t.args, nil) + err = m.LoadConfig([]byte(t.content), t.args, config.DefaultLoaderConfigOptions()) if err != nil { return err } diff --git a/pkg/flow/source_test.go b/pkg/flow/source_test.go index 840642ba203a..fa79c8c1e9e1 100644 --- a/pkg/flow/source_test.go +++ b/pkg/flow/source_test.go @@ -89,7 +89,7 @@ func TestParseSources_DuplicateComponent(t *testing.T) { require.NoError(t, err) ctrl := New(testOptions(t)) defer cleanUpController(ctrl) - err = ctrl.LoadSource(s, nil, nil) + err = ctrl.LoadSource(s, nil) diagErrs, ok := err.(diag.Diagnostics) require.True(t, ok) require.Len(t, diagErrs, 2) @@ -120,7 +120,7 @@ func TestParseSources_UniqueComponent(t *testing.T) { require.NoError(t, err) ctrl := New(testOptions(t)) defer cleanUpController(ctrl) - err = ctrl.LoadSource(s, nil, nil) + err = ctrl.LoadSource(s, nil) require.NoError(t, err) } From b7aa47ee5796a935d99efbc7608af884e4f906c2 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 12:39:53 +0100 Subject: [PATCH 12/36] ignore linter warning on deprecated func --- component/module/file/file.go | 1 + component/module/git/git.go | 1 + component/module/http/http.go | 1 + component/module/string/string.go | 1 + pkg/flow/componenttest/testfailmodule.go | 1 + 5 files changed, 5 insertions(+) diff --git a/component/module/file/file.go b/component/module/file/file.go index d8aeecfb4931..d821354823a4 100644 --- a/component/module/file/file.go +++ b/component/module/file/file.go @@ -59,6 +59,7 @@ var ( // New creates a new module.file component. func New(o component.Options, args Arguments) (*Component, error) { + //nolint:staticcheck m, err := module.NewModuleComponent(o) if err != nil { return nil, err diff --git a/component/module/git/git.go b/component/module/git/git.go index caf1908367cc..e620a25467ca 100644 --- a/component/module/git/git.go +++ b/component/module/git/git.go @@ -75,6 +75,7 @@ var ( // New creates a new module.git component. func New(o component.Options, args Arguments) (*Component, error) { + //nolint:staticcheck m, err := module.NewModuleComponent(o) if err != nil { return nil, err diff --git a/component/module/http/http.go b/component/module/http/http.go index f9c4ae598920..93fb21e60173 100644 --- a/component/module/http/http.go +++ b/component/module/http/http.go @@ -58,6 +58,7 @@ var ( // New creates a new module.http component. func New(o component.Options, args Arguments) (*Component, error) { + //nolint:staticcheck m, err := module.NewModuleComponent(o) if err != nil { return nil, err diff --git a/component/module/string/string.go b/component/module/string/string.go index 9ef2d9cb63df..b53983d36da5 100644 --- a/component/module/string/string.go +++ b/component/module/string/string.go @@ -43,6 +43,7 @@ var ( // New creates a new module.string component. func New(o component.Options, args Arguments) (*Component, error) { + //nolint:staticcheck m, err := module.NewModuleComponent(o) if err != nil { return nil, err diff --git a/pkg/flow/componenttest/testfailmodule.go b/pkg/flow/componenttest/testfailmodule.go index 31c7e9350fa2..32367ac128de 100644 --- a/pkg/flow/componenttest/testfailmodule.go +++ b/pkg/flow/componenttest/testfailmodule.go @@ -16,6 +16,7 @@ func init() { Exports: mod.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { + //nolint:staticcheck m, err := mod.NewModuleComponent(opts) if err != nil { return nil, err From 189a2b203d407d2664e1ee7a11711dcb15ab490c Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 13:53:00 +0100 Subject: [PATCH 13/36] add deprecated doc on module Exports type --- component/module/file/file.go | 5 +++-- component/module/git/git.go | 5 +++-- component/module/http/http.go | 5 +++-- component/module/module.go | 3 +-- component/module/string/string.go | 5 +++-- pkg/flow/componenttest/testfailmodule.go | 5 +++-- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/component/module/file/file.go b/component/module/file/file.go index d821354823a4..22d37bbe44bf 100644 --- a/component/module/file/file.go +++ b/component/module/file/file.go @@ -15,8 +15,9 @@ import ( func init() { component.Register(component.Registration{ - Name: "module.file", - Args: Arguments{}, + Name: "module.file", + Args: Arguments{}, + //nolint:staticcheck Exports: module.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { diff --git a/component/module/git/git.go b/component/module/git/git.go index e620a25467ca..047ef5722f68 100644 --- a/component/module/git/git.go +++ b/component/module/git/git.go @@ -19,8 +19,9 @@ import ( func init() { component.Register(component.Registration{ - Name: "module.git", - Args: Arguments{}, + Name: "module.git", + Args: Arguments{}, + //nolint:staticcheck Exports: module.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { diff --git a/component/module/http/http.go b/component/module/http/http.go index 93fb21e60173..0bcb6f4d9de7 100644 --- a/component/module/http/http.go +++ b/component/module/http/http.go @@ -15,8 +15,9 @@ import ( func init() { component.Register(component.Registration{ - Name: "module.http", - Args: Arguments{}, + Name: "module.http", + Args: Arguments{}, + //nolint:staticcheck Exports: module.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { diff --git a/component/module/module.go b/component/module/module.go index 25ccea208028..4c0a4c62bb27 100644 --- a/component/module/module.go +++ b/component/module/module.go @@ -24,8 +24,7 @@ type ModuleComponent struct { latestLoaderConfigOptions config.LoaderConfigOptions } -// Exports holds values which are exported from the run module. -// This export type is deprecated. +// Deprecated: Exports holds values which are exported from the run module. New modules use map[string]any directly. type Exports struct { // Exports exported from the running module. Exports map[string]any `river:"exports,block"` diff --git a/component/module/string/string.go b/component/module/string/string.go index b53983d36da5..1d6550085413 100644 --- a/component/module/string/string.go +++ b/component/module/string/string.go @@ -11,8 +11,9 @@ import ( func init() { component.Register(component.Registration{ - Name: "module.string", - Args: Arguments{}, + Name: "module.string", + Args: Arguments{}, + //nolint:staticcheck Exports: module.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { diff --git a/pkg/flow/componenttest/testfailmodule.go b/pkg/flow/componenttest/testfailmodule.go index 32367ac128de..f7d624166f75 100644 --- a/pkg/flow/componenttest/testfailmodule.go +++ b/pkg/flow/componenttest/testfailmodule.go @@ -11,8 +11,9 @@ import ( func init() { component.Register(component.Registration{ - Name: "test.fail.module", - Args: TestFailArguments{}, + Name: "test.fail.module", + Args: TestFailArguments{}, + //nolint:staticcheck Exports: mod.Exports{}, Build: func(opts component.Options, args component.Arguments) (component.Component, error) { From 7bfc196fb54558891c6b17105941f5b11ba0315b Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 14:13:30 +0100 Subject: [PATCH 14/36] rename function that decides if a component is a custom one --- pkg/flow/internal/controller/component_node_manager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/flow/internal/controller/component_node_manager.go b/pkg/flow/internal/controller/component_node_manager.go index 61f865298787..6def6914f142 100644 --- a/pkg/flow/internal/controller/component_node_manager.go +++ b/pkg/flow/internal/controller/component_node_manager.go @@ -38,8 +38,7 @@ func (m *ComponentNodeManager) OnReload(additionalDeclareContents map[string]str // CreateComponentNode creates a new builtin component or a new custom component. func (m *ComponentNodeManager) CreateComponentNode(componentName string, block *ast.BlockStmt) (ComponentNode, error) { - firstPart := strings.Split(componentName, ".")[0] - if m.shouldAddCustomComponentNode(firstPart, componentName) { + if m.isCustomComponent(componentName) { return NewCustomComponentNode(m.globals, block, m.getCustomComponentConfig), nil } else { registration, exists := m.componentReg.Get(componentName) @@ -66,8 +65,9 @@ func (m *ComponentNodeManager) getCustomComponentDependencies(declareNode *Decla return dependencies, nil } -// shouldAddCustomComponentNode searches for a declare corresponding to the given component name. -func (m *ComponentNodeManager) shouldAddCustomComponentNode(firstPart, componentName string) bool { +// isCustomComponent searches for a declare corresponding to the given component name. +func (m *ComponentNodeManager) isCustomComponent(componentName string) bool { + firstPart := strings.Split(componentName, ".")[0] _, declareExists := m.declareNodes[firstPart] _, importExists := m.importNodes[firstPart] _, additionalDeclareContentExists := m.additionalDeclareContents[componentName] From 1cb8a319e26a133e4bf75b5de6d03c3acc1c6a15 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 14:25:36 +0100 Subject: [PATCH 15/36] rename firstPart to namespace --- pkg/flow/internal/controller/component_node_manager.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/flow/internal/controller/component_node_manager.go b/pkg/flow/internal/controller/component_node_manager.go index 6def6914f142..1e43c3a7bacb 100644 --- a/pkg/flow/internal/controller/component_node_manager.go +++ b/pkg/flow/internal/controller/component_node_manager.go @@ -67,9 +67,9 @@ func (m *ComponentNodeManager) getCustomComponentDependencies(declareNode *Decla // isCustomComponent searches for a declare corresponding to the given component name. func (m *ComponentNodeManager) isCustomComponent(componentName string) bool { - firstPart := strings.Split(componentName, ".")[0] - _, declareExists := m.declareNodes[firstPart] - _, importExists := m.importNodes[firstPart] + namespace := strings.Split(componentName, ".")[0] + _, declareExists := m.declareNodes[namespace] + _, importExists := m.importNodes[namespace] _, additionalDeclareContentExists := m.additionalDeclareContents[componentName] return declareExists || importExists || additionalDeclareContentExists From b93184e79d281ccd4f9a8a588ecc730c66ef5140 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 14:40:28 +0100 Subject: [PATCH 16/36] make declare fields private --- .../internal/controller/component_node_manager.go | 12 ++++++------ pkg/flow/internal/controller/declare.go | 6 +++--- pkg/flow/internal/controller/node_declare.go | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/flow/internal/controller/component_node_manager.go b/pkg/flow/internal/controller/component_node_manager.go index 1e43c3a7bacb..555a86002308 100644 --- a/pkg/flow/internal/controller/component_node_manager.go +++ b/pkg/flow/internal/controller/component_node_manager.go @@ -124,7 +124,7 @@ func (m *ComponentNodeManager) getCustomComponentConfigFromLocalDeclares(cc *Cus return CustomComponentConfig{}, false } return CustomComponentConfig{ - declareContent: node.Declare().Content, + declareContent: node.Declare().content, additionalDeclareContents: m.getLocalAdditionalDeclareContents(cc.componentName), }, true } @@ -152,7 +152,7 @@ func (m *ComponentNodeManager) getCustomComponentConfigFromImportedDeclares(cc * return CustomComponentConfig{}, false, err } return CustomComponentConfig{ - declareContent: declare.Content, + declareContent: declare.content, additionalDeclareContents: m.getImportAdditionalDeclareContents(node), }, true, nil } @@ -161,7 +161,7 @@ func (m *ComponentNodeManager) getCustomComponentConfigFromImportedDeclares(cc * func (m *ComponentNodeManager) getImportAdditionalDeclareContents(node *ImportConfigNode) map[string]string { additionalDeclareContents := make(map[string]string, len(node.ImportedDeclares())) for importedDeclareLabel, importedDeclare := range node.ImportedDeclares() { - additionalDeclareContents[importedDeclareLabel] = importedDeclare.Content + additionalDeclareContents[importedDeclareLabel] = importedDeclare.content } return additionalDeclareContents } @@ -175,10 +175,10 @@ func (m *ComponentNodeManager) getLocalAdditionalDeclareContents(componentName s // The label of the importNode is added as a prefix to the declare label to create a scope. // This is useful in the scenario where a custom component of an imported declare is defined inside of a local declare. // In this case, this custom component should only have have access to the imported declares of its corresponding import node. - additionalDeclareContents[customComponentDependency.importNode.label+"."+importedDeclareLabel] = importedDeclare.Content + additionalDeclareContents[customComponentDependency.importNode.label+"."+importedDeclareLabel] = importedDeclare.content } } else if customComponentDependency.declareNode != nil { - additionalDeclareContents[customComponentDependency.declareLabel] = customComponentDependency.declareNode.Declare().Content + additionalDeclareContents[customComponentDependency.declareLabel] = customComponentDependency.declareNode.Declare().content } else { additionalDeclareContents[customComponentDependency.componentName] = m.additionalDeclareContents[customComponentDependency.componentName] } @@ -211,7 +211,7 @@ type CustomComponentDependency struct { // Panics if declare is nil. func (m *ComponentNodeManager) FindCustomComponentDependencies(declare *Declare) ([]CustomComponentDependency, error) { uniqueReferences := make(map[string]CustomComponentDependency) - m.findCustomComponentDependencies(declare.Block.Body, uniqueReferences) + m.findCustomComponentDependencies(declare.block.Body, uniqueReferences) references := make([]CustomComponentDependency, 0, len(uniqueReferences)) for _, ref := range uniqueReferences { diff --git a/pkg/flow/internal/controller/declare.go b/pkg/flow/internal/controller/declare.go index 342ba4ef1a33..3bfbdc9f15ea 100644 --- a/pkg/flow/internal/controller/declare.go +++ b/pkg/flow/internal/controller/declare.go @@ -4,11 +4,11 @@ import "github.com/grafana/river/ast" // Declare represents the content of a declare block as AST and as plain string. type Declare struct { - Block *ast.BlockStmt - Content string + block *ast.BlockStmt + content string } // NewDeclare creates a new Declare from its AST and its plain string content. func NewDeclare(block *ast.BlockStmt, content string) *Declare { - return &Declare{Block: block, Content: content} + return &Declare{block: block, content: content} } diff --git a/pkg/flow/internal/controller/node_declare.go b/pkg/flow/internal/controller/node_declare.go index d9e8ce6f5a0b..fc0f4a5c2747 100644 --- a/pkg/flow/internal/controller/node_declare.go +++ b/pkg/flow/internal/controller/node_declare.go @@ -20,9 +20,9 @@ var _ BlockNode = (*DeclareNode)(nil) // NewDeclareNode creates a new declare node with a content which will be loaded by declare component nodes. func NewDeclareNode(declare *Declare) *DeclareNode { return &DeclareNode{ - label: declare.Block.Label, - nodeID: BlockComponentID(declare.Block).String(), - componentName: declare.Block.GetBlockName(), + label: declare.block.Label, + nodeID: BlockComponentID(declare.block).String(), + componentName: declare.block.GetBlockName(), declare: declare, } } @@ -44,7 +44,7 @@ func (cn *DeclareNode) Label() string { return cn.label } func (cn *DeclareNode) Block() *ast.BlockStmt { cn.mut.RLock() defer cn.mut.RUnlock() - return cn.declare.Block + return cn.declare.block } // NodeID implements dag.Node and returns the unique ID for the config node. From d40e409372cd0258b0becb8b9e2d3c435d0580e6 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 14:50:34 +0100 Subject: [PATCH 17/36] add comment regarding the redundancy in the Declare struct --- pkg/flow/internal/controller/declare.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/flow/internal/controller/declare.go b/pkg/flow/internal/controller/declare.go index 3bfbdc9f15ea..20465ddaef9f 100644 --- a/pkg/flow/internal/controller/declare.go +++ b/pkg/flow/internal/controller/declare.go @@ -4,7 +4,9 @@ import "github.com/grafana/river/ast" // Declare represents the content of a declare block as AST and as plain string. type Declare struct { - block *ast.BlockStmt + block *ast.BlockStmt + // TODO: we would not need this content field if the content of the block was saved in ast.BlockStmt when parsing. + // Not only it looks redundant but it allows discrepancies between the block and the content. content string } From ce8ec22dcc6d1262daaad47fa2ed9b5a847b8ff5 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 15:36:09 +0100 Subject: [PATCH 18/36] rename blocks in loader to prevent confusion between componentBlocks and serviceBlocks --- pkg/flow/flow.go | 2 +- pkg/flow/internal/controller/loader.go | 15 +++++++-------- pkg/flow/internal/controller/loader_test.go | 12 ++++++------ pkg/flow/source.go | 16 ++++++++-------- pkg/flow/source_test.go | 12 ++++++------ 5 files changed, 28 insertions(+), 29 deletions(-) diff --git a/pkg/flow/flow.go b/pkg/flow/flow.go index 8e4e5419b137..55f474b02519 100644 --- a/pkg/flow/flow.go +++ b/pkg/flow/flow.go @@ -290,7 +290,7 @@ func (f *Flow) loadSource(source *Source, args map[string]any, options config.Lo f.loadMut.Lock() defer f.loadMut.Unlock() - diags := f.loader.Apply(args, source.components, source.configBlocks, source.declares, options) + diags := f.loader.Apply(args, source.blocks, source.configBlocks, source.declares, options) if !f.loadedOnce.Load() && diags.HasErrors() { // The first call to Load should not run any components if there were // errors in the configuration file. diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 503a724113f6..338f21ab4f10 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -119,7 +119,7 @@ func NewLoader(opts LoaderOptions) *Loader { // The provided parentContext can be used to provide global variables and // functions to components. A child context will be constructed from the parent // to expose values of other components. -func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare, options config.LoaderConfigOptions) diag.Diagnostics { +func (l *Loader) Apply(args map[string]any, blocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare, options config.LoaderConfigOptions) diag.Diagnostics { start := time.Now() l.mut.Lock() defer l.mut.Unlock() @@ -133,14 +133,14 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co // Reload the component node manager when a new config is applied. l.componentNodeManager.OnReload(options.AdditionalDeclareContents) - newGraph, diags := l.loadNewGraph(args, componentBlocks, configBlocks, declares) + newGraph, diags := l.loadNewGraph(args, blocks, configBlocks, declares) if diags.HasErrors() { return diags } var ( components = make([]ComponentNode, 0) - componentIDs = make([]ComponentID, 0, len(componentBlocks)) + componentIDs = make([]ComponentID, 0) services = make([]*ServiceNode, 0, len(l.services)) ) @@ -233,7 +233,6 @@ func (l *Loader) Apply(args map[string]any, componentBlocks []*ast.BlockStmt, co l.serviceNodes = services l.graph = &newGraph l.cache.SyncIDs(componentIDs) - l.blocks = componentBlocks if l.globals.OnExportsChange != nil && l.cache.ExportChangeIndex() != l.moduleExportIndex { l.moduleExportIndex = l.cache.ExportChangeIndex() l.globals.OnExportsChange(l.cache.CreateModuleExports()) @@ -254,11 +253,11 @@ func (l *Loader) Cleanup(stopWorkerPool bool) { } // loadNewGraph creates a new graph from the provided blocks and validates it. -func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare) (dag.Graph, diag.Diagnostics) { +func (l *Loader) loadNewGraph(args map[string]any, blocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare) (dag.Graph, diag.Diagnostics) { var g dag.Graph - // Split component blocks into blocks for components and services. - componentBlocks, serviceBlocks := l.splitComponentBlocks(componentBlocks) + // Split blocks into blocks for components and services. + componentBlocks, serviceBlocks := l.categorizeBlocks(blocks) // Fill our graph with service blocks, which must be added before any other // block. @@ -296,7 +295,7 @@ func (l *Loader) loadNewGraph(args map[string]any, componentBlocks []*ast.BlockS return g, diags } -func (l *Loader) splitComponentBlocks(blocks []*ast.BlockStmt) (componentBlocks, serviceBlocks []*ast.BlockStmt) { +func (l *Loader) categorizeBlocks(blocks []*ast.BlockStmt) (componentBlocks, serviceBlocks []*ast.BlockStmt) { componentBlocks = make([]*ast.BlockStmt, 0, len(blocks)) serviceBlocks = make([]*ast.BlockStmt, 0, len(l.services)) diff --git a/pkg/flow/internal/controller/loader_test.go b/pkg/flow/internal/controller/loader_test.go index 537870eb86fa..bfee8c70a877 100644 --- a/pkg/flow/internal/controller/loader_test.go +++ b/pkg/flow/internal/controller/loader_test.go @@ -228,13 +228,13 @@ func applyFromContent(t *testing.T, l *controller.Loader, componentBytes []byte, t.Helper() var ( - diags diag.Diagnostics - componentBlocks []*ast.BlockStmt - configBlocks []*ast.BlockStmt = nil - declares []*controller.Declare + diags diag.Diagnostics + blocks []*ast.BlockStmt + configBlocks []*ast.BlockStmt = nil + declares []*controller.Declare ) - componentBlocks, diags = fileToBlock(t, componentBytes) + blocks, diags = fileToBlock(t, componentBytes) if diags.HasErrors() { return diags } @@ -246,7 +246,7 @@ func applyFromContent(t *testing.T, l *controller.Loader, componentBytes []byte, } } - applyDiags := l.Apply(nil, componentBlocks, configBlocks, declares, config.DefaultLoaderConfigOptions()) + applyDiags := l.Apply(nil, blocks, configBlocks, declares, config.DefaultLoaderConfigOptions()) diags = append(diags, applyDiags...) return diags diff --git a/pkg/flow/source.go b/pkg/flow/source.go index 312a56f61410..7196339380c8 100644 --- a/pkg/flow/source.go +++ b/pkg/flow/source.go @@ -18,9 +18,9 @@ type Source struct { sourceMap map[string][]byte // Map that links parsed Flow source's name with its content. hash [sha256.Size]byte // Hash of all files in sourceMap sorted by name. - // Components holds the list of raw River AST blocks describing components. + // Components holds the list of raw River AST blocks describing components and services. // The Flow controller can interpret them. - components []*ast.BlockStmt + blocks []*ast.BlockStmt configBlocks []*ast.BlockStmt declares []*controller.Declare } @@ -45,9 +45,9 @@ func ParseSource(name string, bb []byte) (*Source, error) { // TODO(rfratto): should this code be brought into a helper somewhere? Maybe // in ast? var ( - components []*ast.BlockStmt - configs []*ast.BlockStmt - declares []*controller.Declare + blocks []*ast.BlockStmt + configs []*ast.BlockStmt + declares []*controller.Declare ) for _, stmt := range node.Body { @@ -68,7 +68,7 @@ func ParseSource(name string, bb []byte) (*Source, error) { case "logging", "tracing", "argument", "export", "import.file", "import.git", "import.http": configs = append(configs, stmt) default: - components = append(components, stmt) + blocks = append(blocks, stmt) } default: @@ -82,7 +82,7 @@ func ParseSource(name string, bb []byte) (*Source, error) { } return &Source{ - components: components, + blocks: blocks, configBlocks: configs, declares: declares, sourceMap: map[string][]byte{name: bb}, @@ -124,7 +124,7 @@ func ParseSources(sources map[string][]byte) (*Source, error) { return nil, err } - mergedSource.components = append(mergedSource.components, sourceFragment.components...) + mergedSource.blocks = append(mergedSource.blocks, sourceFragment.blocks...) mergedSource.configBlocks = append(mergedSource.configBlocks, sourceFragment.configBlocks...) mergedSource.declares = append(mergedSource.declares, sourceFragment.declares...) } diff --git a/pkg/flow/source_test.go b/pkg/flow/source_test.go index fa79c8c1e9e1..9daa935f52fa 100644 --- a/pkg/flow/source_test.go +++ b/pkg/flow/source_test.go @@ -26,9 +26,9 @@ func TestParseSource(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - require.Len(t, f.components, 2) - require.Equal(t, "testcomponents.tick.ticker_a", getBlockID(f.components[0])) - require.Equal(t, "testcomponents.passthrough.static", getBlockID(f.components[1])) + require.Len(t, f.blocks, 2) + require.Equal(t, "testcomponents.tick.ticker_a", getBlockID(f.blocks[0])) + require.Equal(t, "testcomponents.passthrough.static", getBlockID(f.blocks[1])) } func TestParseSourceWithConfigBlock(t *testing.T) { @@ -46,8 +46,8 @@ func TestParseSourceWithConfigBlock(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - require.Len(t, f.components, 1) - require.Equal(t, "testcomponents.tick.ticker_with_config_block", getBlockID(f.components[0])) + require.Len(t, f.blocks, 1) + require.Equal(t, "testcomponents.tick.ticker_with_config_block", getBlockID(f.blocks[0])) require.Len(t, f.configBlocks, 1) require.Equal(t, "logging", getBlockID(f.configBlocks[0])) } @@ -57,7 +57,7 @@ func TestParseSource_Defaults(t *testing.T) { require.NotNil(t, f) require.NoError(t, err) - require.Len(t, f.components, 0) + require.Len(t, f.blocks, 0) } func TestParseSources_DuplicateComponent(t *testing.T) { From 28d8f5f451482d2342c928ec28a310c991a66c85 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 15:36:58 +0100 Subject: [PATCH 19/36] remove unused field in loader --- pkg/flow/internal/controller/loader.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 338f21ab4f10..5384185ce428 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -45,7 +45,6 @@ type Loader struct { componentNodeManager *ComponentNodeManager cache *valueCache - blocks []*ast.BlockStmt // Most recently loaded blocks, used for writing cm *controllerMetrics cc *controllerCollector moduleExportIndex int From 3a9d20d00b45989e72b94bfb8f4f570da95ad931 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 19:07:59 +0100 Subject: [PATCH 20/36] fix mutex and flaky test --- .../internal/controller/node_config_import.go | 50 +++++++++++-------- pkg/flow/module_import_test.go | 14 +++--- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index f9b9de656e66..9b047ad05899 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -22,24 +22,27 @@ import ( ) type ImportConfigNode struct { - id ComponentID - label string - nodeID string - componentName string - globalID string - globals ComponentGlobals // Need a copy of the globals to create other import nodes. - source importsource.ImportSource + id ComponentID + label string + nodeID string + componentName string + globalID string + globals ComponentGlobals // Need a copy of the globals to create other import nodes. + registry *prometheus.Registry - importedDeclares map[string]*Declare - importConfigNodesChildren map[string]*ImportConfigNode OnNodeWithDependantsUpdate func(cn NodeWithDependants) logger log.Logger - inContentUpdate bool - mut sync.RWMutex - importedContentMut sync.RWMutex - block *ast.BlockStmt // Current River blocks to derive config from - lastUpdateTime atomic.Time + mut sync.RWMutex + block *ast.BlockStmt // Current River blocks to derive config from + source importsource.ImportSource + importConfigNodesChildren map[string]*ImportConfigNode + + contentMut sync.RWMutex + importedDeclares map[string]*Declare + inContentUpdate bool + + lastUpdateTime atomic.Time healthMut sync.RWMutex evalHealth component.Health // Health of the last evaluate @@ -161,6 +164,8 @@ func (cn *ImportConfigNode) processNodeBody(node *ast.File, content string) { // processDeclareBlock processes a declare block. func (cn *ImportConfigNode) processDeclareBlock(stmt *ast.BlockStmt, content string) { + cn.contentMut.Lock() + defer cn.contentMut.Unlock() if _, ok := cn.importedDeclares[stmt.Label]; ok { level.Error(cn.logger).Log("msg", "declare block redefined", "name", stmt.Label) return @@ -183,13 +188,16 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str // onContentUpdate is triggered every time the managed import component has new content. func (cn *ImportConfigNode) onContentUpdate(content string) { - cn.importedContentMut.Lock() - defer cn.importedContentMut.Unlock() + cn.contentMut.Lock() cn.inContentUpdate = true defer func() { + cn.contentMut.Lock() cn.inContentUpdate = false + cn.contentMut.Unlock() }() cn.importedDeclares = make(map[string]*Declare) + cn.contentMut.Unlock() + // We recreate the nodes when the content changes. Can we copy instead for optimization? cn.importConfigNodesChildren = make(map[string]*ImportConfigNode) node, err := parser.ParseFile(cn.label, []byte(content)) @@ -247,6 +255,8 @@ func (cn *ImportConfigNode) runChildren(ctx context.Context) error { // OnChildrenContentUpdate passes their imported content to their parents. // To avoid collisions, the content is scoped via namespaces. func (cn *ImportConfigNode) OnChildrenContentUpdate(child NodeWithDependants) { + cn.contentMut.Lock() + defer cn.contentMut.Unlock() switch child := child.(type) { case *ImportConfigNode: for importedDeclareLabel, importedDeclare := range child.importedDeclares { @@ -262,8 +272,8 @@ func (cn *ImportConfigNode) OnChildrenContentUpdate(child NodeWithDependants) { // GetImportedDeclareByLabel returns a declare block imported by the node. func (cn *ImportConfigNode) GetImportedDeclareByLabel(declareLabel string) (*Declare, error) { - cn.importedContentMut.Lock() - defer cn.importedContentMut.Unlock() + cn.contentMut.Lock() + defer cn.contentMut.Unlock() if declare, ok := cn.importedDeclares[declareLabel]; ok { return declare, nil } @@ -368,8 +378,8 @@ func (cn *ImportConfigNode) Component() component.Component { // ImportedDeclares returns all declare blocks that it imported. func (cn *ImportConfigNode) ImportedDeclares() map[string]*Declare { - cn.mut.RLock() - defer cn.mut.RUnlock() + cn.contentMut.RLock() + defer cn.contentMut.RUnlock() return cn.importedDeclares } diff --git a/pkg/flow/module_import_test.go b/pkg/flow/module_import_test.go index 0228c151ff21..7392e13bb6ec 100644 --- a/pkg/flow/module_import_test.go +++ b/pkg/flow/module_import_test.go @@ -181,13 +181,8 @@ func TestImportModule(t *testing.T) { optional = false } - testcomponents.passthrough "pt" { - input = argument.input.value - lag = "1ms" - } - export "output" { - value = testcomponents.passthrough.pt.output + value = argument.input.value } } `, @@ -207,9 +202,14 @@ func TestImportModule(t *testing.T) { frequency = "10ms" max = 10 } + + testcomponents.passthrough "pt" { + input = testcomponents.count.inc.count + lag = "1ms" + } testImport.test "myModule" { - input = testcomponents.count.inc.count + input = testcomponents.passthrough.pt.output } export "output" { From 32e78111e1e33e4e1e7fc4649ada41ce0145c74b Mon Sep 17 00:00:00 2001 From: William Dumont Date: Wed, 17 Jan 2024 19:18:18 +0100 Subject: [PATCH 21/36] remove unecessary mutex in declare --- pkg/flow/internal/controller/node_declare.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/flow/internal/controller/node_declare.go b/pkg/flow/internal/controller/node_declare.go index fc0f4a5c2747..dde1e79189e7 100644 --- a/pkg/flow/internal/controller/node_declare.go +++ b/pkg/flow/internal/controller/node_declare.go @@ -1,8 +1,6 @@ package controller import ( - "sync" - "github.com/grafana/river/ast" "github.com/grafana/river/vm" ) @@ -11,8 +9,8 @@ type DeclareNode struct { label string nodeID string componentName string - declare *Declare - mut sync.RWMutex + // A declare content is static, it does not change during the lifetime of the node. + declare *Declare } var _ BlockNode = (*DeclareNode)(nil) @@ -28,8 +26,6 @@ func NewDeclareNode(declare *Declare) *DeclareNode { } func (cn *DeclareNode) Declare() *Declare { - cn.mut.Lock() - defer cn.mut.Unlock() return cn.declare } @@ -42,8 +38,6 @@ func (cn *DeclareNode) Label() string { return cn.label } // Block implements BlockNode and returns the current block of the managed config node. func (cn *DeclareNode) Block() *ast.BlockStmt { - cn.mut.RLock() - defer cn.mut.RUnlock() return cn.declare.block } From fc0770e363537ff3a302522343161fc0786754f0 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 18 Jan 2024 10:09:22 +0100 Subject: [PATCH 22/36] add more doc for the Apply function --- pkg/flow/internal/controller/loader.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 5384185ce428..7ef5b07462fa 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -118,6 +118,10 @@ func NewLoader(opts LoaderOptions) *Loader { // The provided parentContext can be used to provide global variables and // functions to components. A child context will be constructed from the parent // to expose values of other components. +// +// Declares are pieces of config that can be used as a blueprints to instantiate custom components. +// The declares argument corresponds to declares written directly in the river config. +// Other declares may come via the import config blocks (provided by the configBlocks argument) and via the LoaderConfigOptions. func (l *Loader) Apply(args map[string]any, blocks []*ast.BlockStmt, configBlocks []*ast.BlockStmt, declares []*Declare, options config.LoaderConfigOptions) diag.Diagnostics { start := time.Now() l.mut.Lock() From e0e018bb2a63d0b85df08834df6260b31c35b778 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 18 Jan 2024 10:46:44 +0100 Subject: [PATCH 23/36] rename and document interfaces --- pkg/flow/flow_components.go | 8 ++++---- .../{node_with_component.go => component_info.go} | 7 ++++--- pkg/flow/internal/controller/component_node.go | 6 ++++-- pkg/flow/internal/controller/node_builtin_component.go | 1 + pkg/flow/internal/controller/node_config_import.go | 2 +- pkg/flow/internal/controller/node_with_dependants.go | 2 +- 6 files changed, 15 insertions(+), 11 deletions(-) rename pkg/flow/internal/controller/{node_with_component.go => component_info.go} (84%) diff --git a/pkg/flow/flow_components.go b/pkg/flow/flow_components.go index 94389946adc0..a57661f1bcdb 100644 --- a/pkg/flow/flow_components.go +++ b/pkg/flow/flow_components.go @@ -29,7 +29,7 @@ func (f *Flow) GetComponent(id component.ID, opts component.InfoOptions) (*compo return nil, component.ErrComponentNotFound } - cn, ok := node.(controller.NodeWithComponent) + cn, ok := node.(controller.ComponentInfo) if !ok { return nil, fmt.Errorf("%q is not a ComponentNode", id) } @@ -67,7 +67,7 @@ func (f *Flow) ListComponents(moduleID string, opts component.InfoOptions) ([]*c return detail, nil } -func (f *Flow) getComponentDetail(cn controller.NodeWithComponent, graph *dag.Graph, opts component.InfoOptions) *component.Info { +func (f *Flow) getComponentDetail(cn controller.ComponentInfo, graph *dag.Graph, opts component.InfoOptions) *component.Info { var references, referencedBy []string // Skip over any edge which isn't between two component nodes. This is a @@ -79,12 +79,12 @@ func (f *Flow) getComponentDetail(cn controller.NodeWithComponent, graph *dag.Gr // // TODO(rfratto): add support for config block nodes in the API and UI. for _, dep := range graph.Dependencies(cn) { - if _, ok := dep.(controller.NodeWithComponent); ok { + if _, ok := dep.(controller.ComponentInfo); ok { references = append(references, dep.NodeID()) } } for _, dep := range graph.Dependants(cn) { - if _, ok := dep.(controller.NodeWithComponent); ok { + if _, ok := dep.(controller.ComponentInfo); ok { referencedBy = append(referencedBy, dep.NodeID()) } } diff --git a/pkg/flow/internal/controller/node_with_component.go b/pkg/flow/internal/controller/component_info.go similarity index 84% rename from pkg/flow/internal/controller/node_with_component.go rename to pkg/flow/internal/controller/component_info.go index aef514f0e248..b6ea9f1c9947 100644 --- a/pkg/flow/internal/controller/node_with_component.go +++ b/pkg/flow/internal/controller/component_info.go @@ -7,9 +7,10 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// NodeWithComponent is ? -// TODO we need a better name -type NodeWithComponent interface { +// ComponentInfo is an interface that encapsulates a comprehensive suite of methods +// for managing and retrieving detailed information about a specific component +// within a dag.Node. +type ComponentInfo interface { BlockNode // CurrentHealth returns the current health of the node. diff --git a/pkg/flow/internal/controller/component_node.go b/pkg/flow/internal/controller/component_node.go index f42e6430378d..e38be0728ead 100644 --- a/pkg/flow/internal/controller/component_node.go +++ b/pkg/flow/internal/controller/component_node.go @@ -2,9 +2,11 @@ package controller import "github.com/grafana/river/ast" -// ComponentNode is a dag.Node that manages a component. +// ComponentNode is a generic representation of a Flow component. +// This is an extension of the ComponentInfo interface because although +// a dag.Node might be running a component, it might not necessarily be its direct representation. type ComponentNode interface { - NodeWithComponent + ComponentInfo // UpdateBlock updates the River block used to construct arguments for the managed component. UpdateBlock(b *ast.BlockStmt) diff --git a/pkg/flow/internal/controller/node_builtin_component.go b/pkg/flow/internal/controller/node_builtin_component.go index dead07d03711..2b873da74c56 100644 --- a/pkg/flow/internal/controller/node_builtin_component.go +++ b/pkg/flow/internal/controller/node_builtin_component.go @@ -112,6 +112,7 @@ type BuiltinComponentNode struct { } var _ NodeWithDependants = (*BuiltinComponentNode)(nil) +var _ RunnableNode = (*BuiltinComponentNode)(nil) var _ ComponentNode = (*BuiltinComponentNode)(nil) // BuiltinComponentNode creates a new BuiltinComponentNode from an initial ast.BlockStmt. diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 9b047ad05899..6d5958cde0f8 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -51,7 +51,7 @@ type ImportConfigNode struct { var _ NodeWithDependants = (*ImportConfigNode)(nil) var _ RunnableNode = (*ImportConfigNode)(nil) -var _ NodeWithComponent = (*ImportConfigNode)(nil) +var _ ComponentInfo = (*ImportConfigNode)(nil) // NewImportConfigNode creates a new ImportConfigNode from an initial ast.BlockStmt. // The underlying config isn't applied until Evaluate is called. diff --git a/pkg/flow/internal/controller/node_with_dependants.go b/pkg/flow/internal/controller/node_with_dependants.go index a7c47360c7ff..72b51fab8834 100644 --- a/pkg/flow/internal/controller/node_with_dependants.go +++ b/pkg/flow/internal/controller/node_with_dependants.go @@ -6,7 +6,7 @@ import ( "github.com/grafana/agent/component" ) -// NodeWithDependants must be implemented by nodes which can trigger other nodes to be evaluated. +// NodeWithDependants must be implemented by dag.Node that can trigger other nodes to be evaluated. type NodeWithDependants interface { BlockNode From 3ca36929a0dce41f64c9c81a41f4d1fcd791c077 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 18 Jan 2024 12:32:58 +0100 Subject: [PATCH 24/36] cleanup on comments, namings and mutex --- pkg/flow/flow_components.go | 2 +- .../internal/controller/component_info.go | 4 +- .../controller/component_node_manager.go | 11 +++-- pkg/flow/internal/controller/loader.go | 6 ++- .../internal/controller/node_config_import.go | 45 +++++++++---------- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/pkg/flow/flow_components.go b/pkg/flow/flow_components.go index a57661f1bcdb..943cf45ad151 100644 --- a/pkg/flow/flow_components.go +++ b/pkg/flow/flow_components.go @@ -31,7 +31,7 @@ func (f *Flow) GetComponent(id component.ID, opts component.InfoOptions) (*compo cn, ok := node.(controller.ComponentInfo) if !ok { - return nil, fmt.Errorf("%q is not a ComponentNode", id) + return nil, fmt.Errorf("%q does not implement ComponentInfo", id) } return f.getComponentDetail(cn, graph, opts), nil diff --git a/pkg/flow/internal/controller/component_info.go b/pkg/flow/internal/controller/component_info.go index b6ea9f1c9947..7575e9fbc8f2 100644 --- a/pkg/flow/internal/controller/component_info.go +++ b/pkg/flow/internal/controller/component_info.go @@ -7,9 +7,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// ComponentInfo is an interface that encapsulates a comprehensive suite of methods -// for managing and retrieving detailed information about a specific component -// within a dag.Node. +// ComponentInfo is an interface that encapsulates methods for managing and retrieving detailed information about a component within a dag.Node. type ComponentInfo interface { BlockNode diff --git a/pkg/flow/internal/controller/component_node_manager.go b/pkg/flow/internal/controller/component_node_manager.go index 555a86002308..cd9eb5f08e92 100644 --- a/pkg/flow/internal/controller/component_node_manager.go +++ b/pkg/flow/internal/controller/component_node_manager.go @@ -7,7 +7,7 @@ import ( "github.com/grafana/river/ast" ) -// ComponentNodeManager is a manager that manages component nodes. +// ComponentNodeManager manages component nodes. type ComponentNodeManager struct { importNodes map[string]*ImportConfigNode declareNodes map[string]*DeclareNode @@ -28,8 +28,8 @@ func NewComponentNodeManager(globals ComponentGlobals, componentReg ComponentReg } } -// OnReload resets the state of the component node manager. -func (m *ComponentNodeManager) OnReload(additionalDeclareContents map[string]string) { +// Reload resets the state of the component node manager and stores the provided additionalDeclareContents. +func (m *ComponentNodeManager) Reload(additionalDeclareContents map[string]string) { m.additionalDeclareContents = additionalDeclareContents m.customComponentDependencies = make(map[string][]CustomComponentDependency) m.importNodes = map[string]*ImportConfigNode{} @@ -52,10 +52,10 @@ func (m *ComponentNodeManager) CreateComponentNode(componentName string, block * // GetCustomComponentDependencies retrieves and caches the dependencies that declare might have to other declares. func (m *ComponentNodeManager) getCustomComponentDependencies(declareNode *DeclareNode) ([]CustomComponentDependency, error) { var dependencies []CustomComponentDependency + var err error if deps, ok := m.customComponentDependencies[declareNode.label]; ok { dependencies = deps } else { - var err error dependencies, err = m.FindCustomComponentDependencies(declareNode.Declare()) if err != nil { return nil, err @@ -75,11 +75,13 @@ func (m *ComponentNodeManager) isCustomComponent(componentName string) bool { return declareExists || importExists || additionalDeclareContentExists } +// GetCorrespondingLocalDeclare returns the declareNode matching the declareLabel of the provided CustomComponentNode if present. func (m *ComponentNodeManager) GetCorrespondingLocalDeclare(cc *CustomComponentNode) (*DeclareNode, bool) { declareNode, exist := m.declareNodes[cc.declareLabel] return declareNode, exist } +// GetCorrespondingImportedDeclare returns the importNode matching the importLabel of the provided CustomComponentNode if present. func (m *ComponentNodeManager) GetCorrespondingImportedDeclare(cc *CustomComponentNode) (*ImportConfigNode, bool) { importNode, exist := m.importNodes[cc.importLabel] return importNode, exist @@ -108,6 +110,7 @@ func (m *ComponentNodeManager) getCustomComponentConfig(cc *CustomComponentNode) } if !found { customComponentConfig, found = m.getCustomComponentConfigFromParent(cc) + // Custom components that receive their config from imported declares in a parent controller can only access the imported declares coming from the same import. customComponentConfig.additionalDeclareContents = filterAdditionalDeclareContents(cc.importLabel, customComponentConfig.additionalDeclareContents) } } diff --git a/pkg/flow/internal/controller/loader.go b/pkg/flow/internal/controller/loader.go index 7ef5b07462fa..b776627981ee 100644 --- a/pkg/flow/internal/controller/loader.go +++ b/pkg/flow/internal/controller/loader.go @@ -135,7 +135,9 @@ func (l *Loader) Apply(args map[string]any, blocks []*ast.BlockStmt, configBlock l.cache.SyncModuleArgs(args) // Reload the component node manager when a new config is applied. - l.componentNodeManager.OnReload(options.AdditionalDeclareContents) + // This step is important to remove configs that might have been removed in the new config. + l.componentNodeManager.Reload(options.AdditionalDeclareContents) + newGraph, diags := l.loadNewGraph(args, blocks, configBlocks, declares) if diags.HasErrors() { return diags @@ -539,7 +541,7 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { if err != nil { diags.Add(diag.Diagnostic{ Severity: diag.SeverityLevelError, - Message: fmt.Sprintf("Error while parsing the declare component %s: %v", n.label, err), + Message: fmt.Sprintf("Error while wiring the custom component %s: %v", n.label, err), StartPos: n.block.NamePos.Position(), EndPos: n.block.NamePos.Add(len(n.componentName) - 1).Position(), }) diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 6d5958cde0f8..19eca0ed0fd4 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -28,15 +28,16 @@ type ImportConfigNode struct { componentName string globalID string globals ComponentGlobals // Need a copy of the globals to create other import nodes. + block *ast.BlockStmt // Current River blocks to derive config from + source importsource.ImportSource registry *prometheus.Registry OnNodeWithDependantsUpdate func(cn NodeWithDependants) logger log.Logger - mut sync.RWMutex - block *ast.BlockStmt // Current River blocks to derive config from - source importsource.ImportSource + importChildrenMut sync.RWMutex importConfigNodesChildren map[string]*ImportConfigNode + importChildrenRunning bool contentMut sync.RWMutex importedDeclares map[string]*Declare @@ -137,8 +138,6 @@ func (cn *ImportConfigNode) setEvalHealth(t component.HealthType, msg string) { } func (cn *ImportConfigNode) evaluate(scope *vm.Scope) error { - cn.mut.Lock() - defer cn.mut.Unlock() return cn.source.Evaluate(scope) } @@ -188,6 +187,10 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str // onContentUpdate is triggered every time the managed import component has new content. func (cn *ImportConfigNode) onContentUpdate(content string) { + cn.importChildrenMut.Lock() + defer cn.importChildrenMut.Unlock() + cn.importConfigNodesChildren = make(map[string]*ImportConfigNode) + cn.contentMut.Lock() cn.inContentUpdate = true defer func() { @@ -198,8 +201,6 @@ func (cn *ImportConfigNode) onContentUpdate(content string) { cn.importedDeclares = make(map[string]*Declare) cn.contentMut.Unlock() - // We recreate the nodes when the content changes. Can we copy instead for optimization? - cn.importConfigNodesChildren = make(map[string]*ImportConfigNode) node, err := parser.ParseFile(cn.label, []byte(content)) if err != nil { level.Error(cn.logger).Log("msg", "failed to parse file on update", "err", err) @@ -208,7 +209,7 @@ func (cn *ImportConfigNode) onContentUpdate(content string) { cn.processNodeBody(node, content) err = cn.evaluateChildren() if err != nil { - level.Error(cn.logger).Log("msg", "failed to update content", "err", err) + level.Error(cn.logger).Log("msg", "failed to evaluate nested import", "err", err) return } cn.lastUpdateTime.Store(time.Now()) @@ -232,8 +233,9 @@ func (cn *ImportConfigNode) evaluateChildren() error { // runChildren run the import nodes managed by this import node. func (cn *ImportConfigNode) runChildren(ctx context.Context) error { var wg sync.WaitGroup - errChildrenChan := make(chan error, len(cn.importConfigNodesChildren)) + cn.importChildrenMut.Lock() + errChildrenChan := make(chan error, len(cn.importConfigNodesChildren)) for _, child := range cn.importConfigNodesChildren { wg.Add(1) go func(child *ImportConfigNode) { @@ -243,6 +245,8 @@ func (cn *ImportConfigNode) runChildren(ctx context.Context) error { } }(child) } + cn.importChildrenRunning = true + cn.importChildrenMut.Unlock() go func() { wg.Wait() @@ -287,11 +291,10 @@ func (cn *ImportConfigNode) GetImportedDeclareByLabel(declareLabel string) (*Dec // Run will immediately return ErrUnevaluated if Evaluate has never been called // successfully. Otherwise, Run will return nil. func (cn *ImportConfigNode) Run(ctx context.Context) error { - cn.mut.RLock() - managed := cn.source - cn.mut.RUnlock() - - if managed == nil { + cn.importChildrenMut.Lock() + importChildren := len(cn.importConfigNodesChildren) + cn.importChildrenMut.Unlock() + if cn.source == nil { return ErrUnevaluated } @@ -300,7 +303,7 @@ func (cn *ImportConfigNode) Run(ctx context.Context) error { errChan := make(chan error, 1) - if len(cn.importConfigNodesChildren) > 0 { + if importChildren > 0 { go func() { errChan <- cn.runChildren(ctx) }() @@ -309,7 +312,7 @@ func (cn *ImportConfigNode) Run(ctx context.Context) error { cn.setRunHealth(component.HealthTypeHealthy, "started component") go func() { - errChan <- managed.Run(ctx) + errChan <- cn.source.Run(ctx) }() err := <-errChan @@ -342,8 +345,6 @@ func (cn *ImportConfigNode) Label() string { return cn.label } // Block implements BlockNode and returns the current block of the managed config node. func (cn *ImportConfigNode) Block() *ast.BlockStmt { - cn.mut.RLock() - defer cn.mut.RUnlock() return cn.block } @@ -363,16 +364,12 @@ func (cn *ImportConfigNode) LastUpdateTime() time.Time { // Arguments returns the current arguments of the managed component. func (cn *ImportConfigNode) Arguments() component.Arguments { - cn.mut.RLock() - defer cn.mut.RUnlock() return cn.source.Arguments() } // Component returns the instance of the managed component. Component may be // nil if the ComponentNode has not been successfully evaluated yet. func (cn *ImportConfigNode) Component() component.Component { - cn.mut.RLock() - defer cn.mut.RUnlock() return cn.source.Component() } @@ -385,8 +382,8 @@ func (cn *ImportConfigNode) ImportedDeclares() map[string]*Declare { // ImportConfigNodesChildren returns the ImportConfigNodesChildren of this ImportConfigNode. func (cn *ImportConfigNode) ImportConfigNodesChildren() map[string]*ImportConfigNode { - cn.mut.RLock() - defer cn.mut.RUnlock() + cn.importChildrenMut.Lock() + defer cn.importChildrenMut.Unlock() return cn.importConfigNodesChildren } From 40f5fd99f6ee04e08f3c25a1e933365a3ec760a7 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 18 Jan 2024 14:35:07 +0100 Subject: [PATCH 25/36] update running children in case of an update of the content in import node --- .../internal/controller/node_config_import.go | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 19eca0ed0fd4..79139bfeb616 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -35,6 +35,8 @@ type ImportConfigNode struct { OnNodeWithDependantsUpdate func(cn NodeWithDependants) logger log.Logger + importChildrenUpdateChan chan struct{} + importChildrenMut sync.RWMutex importConfigNodesChildren map[string]*ImportConfigNode importChildrenRunning bool @@ -82,6 +84,7 @@ func NewImportConfigNode(block *ast.BlockStmt, globals ComponentGlobals, sourceT block: block, evalHealth: initHealth, runHealth: initHealth, + importChildrenUpdateChan: make(chan struct{}), } managedOpts := getImportManagedOptions(globals, cn) cn.logger = managedOpts.Logger @@ -212,6 +215,11 @@ func (cn *ImportConfigNode) onContentUpdate(content string) { level.Error(cn.logger).Log("msg", "failed to evaluate nested import", "err", err) return } + + if cn.importChildrenRunning { + cn.importChildrenUpdateChan <- struct{}{} + } + cn.lastUpdateTime.Store(time.Now()) cn.OnNodeWithDependantsUpdate(cn) } @@ -231,29 +239,59 @@ func (cn *ImportConfigNode) evaluateChildren() error { } // runChildren run the import nodes managed by this import node. -func (cn *ImportConfigNode) runChildren(ctx context.Context) error { +// The children list can be updated onContentUpdate. In this case we need to stop the running children and run the new set of children. +func (cn *ImportConfigNode) runChildren(parentCtx context.Context) error { + errChildrenChan := make(chan error, 1) var wg sync.WaitGroup - - cn.importChildrenMut.Lock() - errChildrenChan := make(chan error, len(cn.importConfigNodesChildren)) - for _, child := range cn.importConfigNodesChildren { - wg.Add(1) - go func(child *ImportConfigNode) { - defer wg.Done() - if err := child.Run(ctx); err != nil { - errChildrenChan <- err - } - }(child) + var ctx context.Context + var cancel context.CancelFunc + + startChildren := func(ctx context.Context, children map[string]*ImportConfigNode, wg *sync.WaitGroup) { + for _, child := range children { + wg.Add(1) + go func(child *ImportConfigNode) { + defer wg.Done() + if err := child.Run(ctx); err != nil { + errChildrenChan <- err + } + }(child) + } } - cn.importChildrenRunning = true - cn.importChildrenMut.Unlock() - go func() { + childrenDone := func() { wg.Wait() close(errChildrenChan) - }() + } - return <-errChildrenChan + ctx, cancel = context.WithCancel(parentCtx) + cn.importChildrenMut.Lock() + startChildren(ctx, cn.importConfigNodesChildren, &wg) // initial start of children + cn.importChildrenRunning = true + cn.importChildrenMut.Unlock() + go childrenDone() + + for { + select { + case <-cn.importChildrenUpdateChan: + errChildrenChan = make(chan error, 1) // we erase the previous channel so that it is not closed by the running childrenDone goroutine + cancel() // we cancel all running children, which will stop the running childrenDone goroutine + wg = sync.WaitGroup{} + ctx, cancel = context.WithCancel(parentCtx) // we create a new context + cn.importChildrenMut.Lock() + startChildren(ctx, cn.importConfigNodesChildren, &wg) // start the new set of children + cn.importChildrenMut.Unlock() + go childrenDone() + case err, ok := <-errChildrenChan: + if !ok { + // All children were cancelled without error. + return nil + } + if err != nil { + // One child stopped because of an error. + return err + } + } + } } // OnChildrenContentUpdate passes their imported content to their parents. @@ -299,7 +337,7 @@ func (cn *ImportConfigNode) Run(ctx context.Context) error { } ctx, cancel := context.WithCancel(ctx) - defer cancel() + defer cancel() // This will stop the children and the managed component. errChan := make(chan error, 1) From 79b98ea720643022e7085d48f83a39f04df22163 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 18 Jan 2024 15:12:26 +0100 Subject: [PATCH 26/36] check for go routines leak --- pkg/flow/module_import_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/flow/module_import_test.go b/pkg/flow/module_import_test.go index 7392e13bb6ec..dc820a693c35 100644 --- a/pkg/flow/module_import_test.go +++ b/pkg/flow/module_import_test.go @@ -355,13 +355,8 @@ func TestImportModule(t *testing.T) { optional = false } - testcomponents.passthrough "pt" { - input = argument.input.value - lag = "1ms" - } - export "output" { - value = testcomponents.passthrough.pt.output + value = argument.input.value } } @@ -614,6 +609,7 @@ func TestImportModule(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + defer verifyNoGoroutineLeaks(t) filename := "module" require.NoError(t, os.WriteFile(filename, []byte(tc.module), 0664)) defer os.Remove(filename) From 65de37413d40c98843dbbbe15e0e9337f60260d1 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 18 Jan 2024 15:20:22 +0100 Subject: [PATCH 27/36] comment cleanup --- pkg/flow/internal/controller/node_declare.go | 2 +- pkg/flow/internal/importsource/import_git.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/flow/internal/controller/node_declare.go b/pkg/flow/internal/controller/node_declare.go index dde1e79189e7..ba097008ab5f 100644 --- a/pkg/flow/internal/controller/node_declare.go +++ b/pkg/flow/internal/controller/node_declare.go @@ -15,7 +15,7 @@ type DeclareNode struct { var _ BlockNode = (*DeclareNode)(nil) -// NewDeclareNode creates a new declare node with a content which will be loaded by declare component nodes. +// NewDeclareNode creates a new declare node with a content which will be loaded by custom components. func NewDeclareNode(declare *Declare) *DeclareNode { return &DeclareNode{ label: declare.block.Label, diff --git a/pkg/flow/internal/importsource/import_git.go b/pkg/flow/internal/importsource/import_git.go index 236089f9ff59..1dc8ad385550 100644 --- a/pkg/flow/internal/importsource/import_git.go +++ b/pkg/flow/internal/importsource/import_git.go @@ -18,7 +18,6 @@ import ( ) // The difference between this import source and the others is that there is no git component. -// The git logic in the internal package is a copy of the one used in the old module. type ImportGit struct { opts component.Options log log.Logger From 219474ae70cd72ef06f2fe769928519bbb91a94c Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 18 Jan 2024 17:01:40 +0100 Subject: [PATCH 28/36] can cancel on all paths for linter --- pkg/flow/internal/controller/node_config_import.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 79139bfeb616..72163b3ced13 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -284,10 +284,12 @@ func (cn *ImportConfigNode) runChildren(parentCtx context.Context) error { case err, ok := <-errChildrenChan: if !ok { // All children were cancelled without error. + cancel() return nil } if err != nil { // One child stopped because of an error. + cancel() return err } } From 4f2d64d30372d17d8bccc38d8fa39842969b66a7 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 18 Jan 2024 19:54:19 +0100 Subject: [PATCH 29/36] improve import children handling --- .../internal/controller/node_config_import.go | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 72163b3ced13..5c1b477e325b 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -241,7 +241,7 @@ func (cn *ImportConfigNode) evaluateChildren() error { // runChildren run the import nodes managed by this import node. // The children list can be updated onContentUpdate. In this case we need to stop the running children and run the new set of children. func (cn *ImportConfigNode) runChildren(parentCtx context.Context) error { - errChildrenChan := make(chan error, 1) + errChildrenChan := make(chan error) var wg sync.WaitGroup var ctx context.Context var cancel context.CancelFunc @@ -258,9 +258,9 @@ func (cn *ImportConfigNode) runChildren(parentCtx context.Context) error { } } - childrenDone := func() { + childrenDone := func(wg *sync.WaitGroup, doneChan chan struct{}) { wg.Wait() - close(errChildrenChan) + close(doneChan) } ctx, cancel = context.WithCancel(parentCtx) @@ -268,30 +268,33 @@ func (cn *ImportConfigNode) runChildren(parentCtx context.Context) error { startChildren(ctx, cn.importConfigNodesChildren, &wg) // initial start of children cn.importChildrenRunning = true cn.importChildrenMut.Unlock() - go childrenDone() + + doneChan := make(chan struct{}) + go childrenDone(&wg, doneChan) // start goroutine to check in case all children finish for { select { case <-cn.importChildrenUpdateChan: - errChildrenChan = make(chan error, 1) // we erase the previous channel so that it is not closed by the running childrenDone goroutine - cancel() // we cancel all running children, which will stop the running childrenDone goroutine + cancel() // cancel all running children + <-doneChan // wait for the children to finish + wg = sync.WaitGroup{} - ctx, cancel = context.WithCancel(parentCtx) // we create a new context + errChildrenChan = make(chan error) + doneChan = make(chan struct{}) + + ctx, cancel = context.WithCancel(parentCtx) // create a new context cn.importChildrenMut.Lock() startChildren(ctx, cn.importConfigNodesChildren, &wg) // start the new set of children cn.importChildrenMut.Unlock() - go childrenDone() - case err, ok := <-errChildrenChan: - if !ok { - // All children were cancelled without error. - cancel() - return nil - } - if err != nil { - // One child stopped because of an error. - cancel() - return err - } + go childrenDone(&wg, doneChan) // start goroutine to check in case all new children finish + case err := <-errChildrenChan: + // One child stopped because of an error. + cancel() + return err + case <-doneChan: + // All children were cancelled without error. + cancel() + return nil } } } From 3fdc3721d53c910df14c5f49f4448d94ef7b3650 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Fri, 19 Jan 2024 14:32:37 +0100 Subject: [PATCH 30/36] renaming --- pkg/flow/internal/controller/node_custom_component.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/flow/internal/controller/node_custom_component.go b/pkg/flow/internal/controller/node_custom_component.go index fea3eb6be89e..424a76f59203 100644 --- a/pkg/flow/internal/controller/node_custom_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -126,12 +126,12 @@ func NewCustomComponentNode(globals ComponentGlobals, b *ast.BlockStmt, GetCusto evalHealth: initHealth, runHealth: initHealth, } - cn.managedOpts = getDeclareManagedOptions(globals, cn) + cn.managedOpts = getCustomManagedOptions(globals, cn) return cn } -func getDeclareManagedOptions(globals ComponentGlobals, cn *CustomComponentNode) component.Options { +func getCustomManagedOptions(globals ComponentGlobals, cn *CustomComponentNode) component.Options { cn.registry = prometheus.NewRegistry() return component.Options{ ID: cn.globalID, @@ -289,6 +289,8 @@ func (cn *CustomComponentNode) setExports(e component.Exports) { // exports. var changed bool + fmt.Println(cn.nodeID, e) + cn.exportsMut.Lock() if !reflect.DeepEqual(cn.exports, e) { changed = true From d53e29ec26190411a95d5dcbf47043687345cd39 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Fri, 19 Jan 2024 14:33:05 +0100 Subject: [PATCH 31/36] optimization to avoid reloading a module if the imported content did not change --- pkg/flow/internal/controller/node_config_import.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/flow/internal/controller/node_config_import.go b/pkg/flow/internal/controller/node_config_import.go index 5c1b477e325b..ec89f8441a02 100644 --- a/pkg/flow/internal/controller/node_config_import.go +++ b/pkg/flow/internal/controller/node_config_import.go @@ -44,6 +44,7 @@ type ImportConfigNode struct { contentMut sync.RWMutex importedDeclares map[string]*Declare inContentUpdate bool + content string lastUpdateTime atomic.Time @@ -192,6 +193,13 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str func (cn *ImportConfigNode) onContentUpdate(content string) { cn.importChildrenMut.Lock() defer cn.importChildrenMut.Unlock() + cn.contentMut.Lock() + // If the source sent the same content, there is no need to reload. + if cn.content == content { + return + } + cn.content = content + cn.contentMut.Unlock() cn.importConfigNodesChildren = make(map[string]*ImportConfigNode) cn.contentMut.Lock() From f384f0b78c96d8b86ffc02b71b1d9fc75ff3b170 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Fri, 19 Jan 2024 14:33:20 +0100 Subject: [PATCH 32/36] improve tests --- pkg/flow/module_import_test.go | 107 +++++++++++++++++---------------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/pkg/flow/module_import_test.go b/pkg/flow/module_import_test.go index dc820a693c35..98ea9b1e57f0 100644 --- a/pkg/flow/module_import_test.go +++ b/pkg/flow/module_import_test.go @@ -25,7 +25,7 @@ func TestImportModule(t *testing.T) { lag = "1ms" } - export "output" { + export "testOutput" { value = -10 } } @@ -51,7 +51,7 @@ func TestImportModule(t *testing.T) { lag = "1ms" } - export "output" { + export "testOutput" { value = testcomponents.passthrough.pt.output } }`, @@ -70,7 +70,7 @@ func TestImportModule(t *testing.T) { } testcomponents.summation "sum" { - input = testImport.test.myModule.output + input = testImport.test.myModule.testOutput } `, updateModule: func(filename string) string { @@ -87,7 +87,7 @@ func TestImportModule(t *testing.T) { lag = "1ms" } - export "output" { + export "testOutput" { value = testcomponents.passthrough.pt.output } }`, @@ -100,7 +100,7 @@ func TestImportModule(t *testing.T) { } testcomponents.summation "sum" { - input = testImport.test.myModule.output + input = testImport.test.myModule.testOutput } `, updateModule: func(filename string) string { @@ -111,7 +111,7 @@ func TestImportModule(t *testing.T) { lag = "1ms" } - export "output" { + export "testOutput" { value = testcomponents.passthrough.pt.output } } @@ -132,7 +132,7 @@ func TestImportModule(t *testing.T) { lag = "1ms" } - export "output" { + export "testOutput" { value = testcomponents.passthrough.pt.output } } @@ -157,15 +157,15 @@ func TestImportModule(t *testing.T) { input = testcomponents.count.inc.count } - export "output" { - value = testImport.test.myModule.output + export "anotherModuleOutput" { + value = testImport.test.myModule.testOutput } } anotherModule "myOtherModule" {} testcomponents.summation "sum" { - input = anotherModule.myOtherModule.output + input = anotherModule.myOtherModule.anotherModuleOutput } `, updateModule: func(filename string) string { @@ -181,7 +181,7 @@ func TestImportModule(t *testing.T) { optional = false } - export "output" { + export "testOutput" { value = argument.input.value } } @@ -212,21 +212,21 @@ func TestImportModule(t *testing.T) { input = testcomponents.passthrough.pt.output } - export "output" { - value = testImport.test.myModule.output + export "anotherModuleOutput" { + value = testImport.test.myModule.testOutput } } anotherModule "myOtherModule" {} - export "output" { - value = anotherModule.myOtherModule.output + export "yetAgainAnotherModuleOutput" { + value = anotherModule.myOtherModule.anotherModuleOutput } } yetAgainAnotherModule "default" {} testcomponents.summation "sum" { - input = yetAgainAnotherModule.default.output + input = yetAgainAnotherModule.default.yetAgainAnotherModuleOutput } `, updateModule: func(filename string) string { @@ -250,8 +250,8 @@ func TestImportModule(t *testing.T) { input = testcomponents.count.inc.count } - export "output" { - value = otherModule.test.default.output + export "anotherModuleOutput" { + value = otherModule.test.default.testOutput } } `, @@ -266,7 +266,7 @@ func TestImportModule(t *testing.T) { lag = "1ms" } - export "output" { + export "testOutput" { value = testcomponents.passthrough.pt.output } } @@ -279,7 +279,7 @@ func TestImportModule(t *testing.T) { testImport.anotherModule "myOtherModule" {} testcomponents.summation "sum" { - input = testImport.anotherModule.myOtherModule.output + input = testImport.anotherModule.myOtherModule.anotherModuleOutput } `, updateModule: func(filename string) string { @@ -305,8 +305,8 @@ func TestImportModule(t *testing.T) { input = argument.input.value } - export "output" { - value = default.test.default.output + export "blablaOutput" { + value = default.test.default.testOutput } } @@ -314,8 +314,8 @@ func TestImportModule(t *testing.T) { input = testcomponents.count.inc.count } - export "output" { - value = blabla.default.output + export "anotherModuleOutput" { + value = blabla.default.blablaOutput } } `, @@ -330,7 +330,7 @@ func TestImportModule(t *testing.T) { lag = "1ms" } - export "output" { + export "testOutput" { value = testcomponents.passthrough.pt.output } } @@ -343,7 +343,7 @@ func TestImportModule(t *testing.T) { testImport.anotherModule "myOtherModule" {} testcomponents.summation "sum" { - input = testImport.anotherModule.myOtherModule.output + input = testImport.anotherModule.myOtherModule.anotherModuleOutput } `, }, @@ -355,8 +355,13 @@ func TestImportModule(t *testing.T) { optional = false } - export "output" { - value = argument.input.value + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "other_testOutput" { + value = testcomponents.passthrough.pt.output } } @@ -369,8 +374,8 @@ func TestImportModule(t *testing.T) { input = argument.input.value } - export "output" { - value = other_test.default.output + export "testOutput" { + value = other_test.default.other_testOutput } } `, @@ -394,15 +399,15 @@ func TestImportModule(t *testing.T) { input = testcomponents.count.inc.count } - export "output" { - value = testImport.test.myModule.output + export "anotherModuleOutput" { + value = testImport.test.myModule.testOutput } } anotherModule "myOtherModule" {} testcomponents.summation "sum" { - input = anotherModule.myOtherModule.output + input = anotherModule.myOtherModule.anotherModuleOutput } `, updateModule: func(filename string) string { @@ -425,7 +430,7 @@ func TestImportModule(t *testing.T) { input = argument.input.value } - export "output" { + export "testOutput" { value = other_test.default.output } } @@ -448,8 +453,8 @@ func TestImportModule(t *testing.T) { input = argument.input.value } - export "output" { - value = importOtherTest.other_test.default.output + export "testOutput" { + value = importOtherTest.other_test.default.other_testOutput } } `, @@ -464,7 +469,7 @@ func TestImportModule(t *testing.T) { lag = "1ms" } - export "output" { + export "other_testOutput" { value = testcomponents.passthrough.pt.output } }`, @@ -488,15 +493,15 @@ func TestImportModule(t *testing.T) { input = testcomponents.count.inc.count } - export "output" { - value = testImport.test.myModule.output + export "anotherModuleOutput" { + value = testImport.test.myModule.testOutput } } anotherModule "myOtherModule" {} testcomponents.summation "sum" { - input = anotherModule.myOtherModule.output + input = anotherModule.myOtherModule.anotherModuleOutput } `, updateModule: func(filename string) string { @@ -505,7 +510,7 @@ func TestImportModule(t *testing.T) { argument "input" { optional = false } - export "output" { + export "other_testOutput" { value = -10 } } @@ -531,8 +536,8 @@ func TestImportModule(t *testing.T) { importOtherTest.other_test "default" { input = argument.input.value } - export "output" { - value = importOtherTest.other_test.default.output + export "anotherOneOutput" { + value = importOtherTest.other_test.default.other_testOutput } } @@ -540,8 +545,8 @@ func TestImportModule(t *testing.T) { input = argument.input.value } - export "output" { - value = anotherOne.default.output + export "testOutput" { + value = anotherOne.default.anotherOneOutput } } `, @@ -553,10 +558,10 @@ func TestImportModule(t *testing.T) { testcomponents.passthrough "pt" { input = argument.input.value - lag = "1ms" + lag = "5ms" } - export "output" { + export "other_testOutput" { value = testcomponents.passthrough.pt.output } }`, @@ -580,15 +585,15 @@ func TestImportModule(t *testing.T) { input = testcomponents.count.inc.count } - export "output" { - value = testImport.test.myModule.output + export "anotherModuleOutput" { + value = testImport.test.myModule.testOutput } } anotherModule "myOtherModule" {} testcomponents.summation "sum" { - input = anotherModule.myOtherModule.output + input = anotherModule.myOtherModule.anotherModuleOutput } `, updateModule: func(filename string) string { @@ -597,7 +602,7 @@ func TestImportModule(t *testing.T) { argument "input" { optional = false } - export "output" { + export "other_testOutput" { value = -10 } } From fdcf18ac66bdf5327a9c5eed94175ad7b3a2c056 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Fri, 19 Jan 2024 16:22:01 +0100 Subject: [PATCH 33/36] remove forgotten print --- pkg/flow/internal/controller/node_custom_component.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/flow/internal/controller/node_custom_component.go b/pkg/flow/internal/controller/node_custom_component.go index 424a76f59203..2e5b9c2065f1 100644 --- a/pkg/flow/internal/controller/node_custom_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -289,8 +289,6 @@ func (cn *CustomComponentNode) setExports(e component.Exports) { // exports. var changed bool - fmt.Println(cn.nodeID, e) - cn.exportsMut.Lock() if !reflect.DeepEqual(cn.exports, e) { changed = true From fea7a8a77380ce7a3a4a6fe7ea91b92311989e67 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 23 Jan 2024 15:54:24 +0100 Subject: [PATCH 34/36] store args in custom component --- pkg/flow/internal/controller/node_custom_component.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/flow/internal/controller/node_custom_component.go b/pkg/flow/internal/controller/node_custom_component.go index 2e5b9c2065f1..c47f2e4825d9 100644 --- a/pkg/flow/internal/controller/node_custom_component.go +++ b/pkg/flow/internal/controller/node_custom_component.go @@ -208,6 +208,8 @@ func (cn *CustomComponentNode) evaluate(scope *vm.Scope) error { return fmt.Errorf("decoding River: %w", err) } + cn.args = args + if cn.managed == nil { // We haven't built the managed custom component successfully yet. managed, err := module.NewModuleComponentV2(cn.managedOpts) From 5ef46702c147dbee365cdc7bebd9cfb8134a5f40 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Tue, 23 Jan 2024 17:39:02 +0100 Subject: [PATCH 35/36] refactor integration tests to remove redundancy --- .../tests/module-file/module_file_test.go | 67 +----------------- .../scrape_prom_metrics_module_git_test.go | 69 +------------------ .../scrape_prom_metrics_module_http_test.go | 69 +------------------ 3 files changed, 3 insertions(+), 202 deletions(-) diff --git a/integration-tests/tests/module-file/module_file_test.go b/integration-tests/tests/module-file/module_file_test.go index bb7db88feb44..1bc6e1a4d722 100644 --- a/integration-tests/tests/module-file/module_file_test.go +++ b/integration-tests/tests/module-file/module_file_test.go @@ -1,67 +1,16 @@ package main import ( - "fmt" - "strconv" "testing" "github.com/grafana/agent/integration-tests/common" "github.com/stretchr/testify/assert" ) -const promURL = "http://localhost:9009/prometheus/api/v1/query?query=" const lokiUrl = "http://localhost:3100/loki/api/v1/query?query={test_name=%22module_file%22}" -func metricQuery(metricName string) string { - return fmt.Sprintf("%s%s{test_name='module_file'}", promURL, metricName) -} - func TestScrapePromMetricsModuleFile(t *testing.T) { - metrics := []string{ - // TODO: better differentiate these metric types? - "golang_counter", - "golang_gauge", - "golang_histogram_bucket", - "golang_summary", - "golang_native_histogram", - } - - for _, metric := range metrics { - metric := metric - t.Run(metric, func(t *testing.T) { - t.Parallel() - if metric == "golang_native_histogram" { - assertHistogramData(t, metricQuery(metric), metric) - } else { - assertMetricData(t, metricQuery(metric), metric) - } - }) - } -} - -func assertHistogramData(t *testing.T, query string, expectedMetric string) { - var metricResponse common.MetricResponse - assert.EventuallyWithT(t, func(c *assert.CollectT) { - err := common.FetchDataFromURL(query, &metricResponse) - assert.NoError(c, err) - if assert.NotEmpty(c, metricResponse.Data.Result) { - assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) - assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "module_file") - if assert.NotNil(c, metricResponse.Data.Result[0].Histogram) { - histogram := metricResponse.Data.Result[0].Histogram - if assert.NotEmpty(c, histogram.Data.Count) { - count, _ := strconv.Atoi(histogram.Data.Count) - assert.Greater(c, count, 10, "Count should be at some point greater than 10.") - } - if assert.NotEmpty(c, histogram.Data.Sum) { - sum, _ := strconv.ParseFloat(histogram.Data.Sum, 64) - assert.Greater(c, sum, 10., "Sum should be at some point greater than 10.") - } - assert.NotEmpty(c, histogram.Data.Buckets) - assert.Nil(c, metricResponse.Data.Result[0].Value) - } - } - }, common.DefaultTimeout, common.DefaultRetryInterval, "Histogram data did not satisfy the conditions within the time limit") + common.MimirMetricsTest(t, common.PromDefaultMetrics, common.PromDefaultHistogramMetric, "module_file") } func TestReadLogFile(t *testing.T) { @@ -79,17 +28,3 @@ func TestReadLogFile(t *testing.T) { } }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") } - -func assertMetricData(t *testing.T, query, expectedMetric string) { - var metricResponse common.MetricResponse - assert.EventuallyWithT(t, func(c *assert.CollectT) { - err := common.FetchDataFromURL(query, &metricResponse) - assert.NoError(c, err) - if assert.NotEmpty(c, metricResponse.Data.Result) { - assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) - assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "module_file") - assert.NotEmpty(c, metricResponse.Data.Result[0].Value.Value) - assert.Nil(c, metricResponse.Data.Result[0].Histogram) - } - }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") -} diff --git a/integration-tests/tests/scrape-prom-metrics-module-git/scrape_prom_metrics_module_git_test.go b/integration-tests/tests/scrape-prom-metrics-module-git/scrape_prom_metrics_module_git_test.go index 1c095f2dd6bf..ad309d54bb92 100644 --- a/integration-tests/tests/scrape-prom-metrics-module-git/scrape_prom_metrics_module_git_test.go +++ b/integration-tests/tests/scrape-prom-metrics-module-git/scrape_prom_metrics_module_git_test.go @@ -1,78 +1,11 @@ package main import ( - "fmt" - "strconv" "testing" "github.com/grafana/agent/integration-tests/common" - "github.com/stretchr/testify/assert" ) -const promURL = "http://localhost:9009/prometheus/api/v1/query?query=" - -func metricQuery(metricName string) string { - return fmt.Sprintf("%s%s{test_name='scrape_prom_metrics_module_git'}", promURL, metricName) -} - func TestScrapePromMetricsModuleGit(t *testing.T) { - metrics := []string{ - // TODO: better differentiate these metric types? - "golang_counter", - "golang_gauge", - "golang_histogram_bucket", - "golang_summary", - "golang_native_histogram", - } - - for _, metric := range metrics { - metric := metric - t.Run(metric, func(t *testing.T) { - t.Parallel() - if metric == "golang_native_histogram" { - assertHistogramData(t, metricQuery(metric), metric) - } else { - assertMetricData(t, metricQuery(metric), metric) - } - }) - } -} - -func assertHistogramData(t *testing.T, query string, expectedMetric string) { - var metricResponse common.MetricResponse - assert.EventuallyWithT(t, func(c *assert.CollectT) { - err := common.FetchDataFromURL(query, &metricResponse) - assert.NoError(c, err) - if assert.NotEmpty(c, metricResponse.Data.Result) { - assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) - assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics_module_git") - if assert.NotNil(c, metricResponse.Data.Result[0].Histogram) { - histogram := metricResponse.Data.Result[0].Histogram - if assert.NotEmpty(c, histogram.Data.Count) { - count, _ := strconv.Atoi(histogram.Data.Count) - assert.Greater(c, count, 10, "Count should be at some point greater than 10.") - } - if assert.NotEmpty(c, histogram.Data.Sum) { - sum, _ := strconv.ParseFloat(histogram.Data.Sum, 64) - assert.Greater(c, sum, 10., "Sum should be at some point greater than 10.") - } - assert.NotEmpty(c, histogram.Data.Buckets) - assert.Nil(c, metricResponse.Data.Result[0].Value) - } - } - }, common.DefaultTimeout, common.DefaultRetryInterval, "Histogram data did not satisfy the conditions within the time limit") -} - -func assertMetricData(t *testing.T, query, expectedMetric string) { - var metricResponse common.MetricResponse - assert.EventuallyWithT(t, func(c *assert.CollectT) { - err := common.FetchDataFromURL(query, &metricResponse) - assert.NoError(c, err) - if assert.NotEmpty(c, metricResponse.Data.Result) { - assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) - assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics_module_git") - assert.NotEmpty(c, metricResponse.Data.Result[0].Value.Value) - assert.Nil(c, metricResponse.Data.Result[0].Histogram) - } - }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") + common.MimirMetricsTest(t, common.PromDefaultMetrics, common.PromDefaultHistogramMetric, "scrape_prom_metrics_module_git") } diff --git a/integration-tests/tests/scrape-prom-metrics-module-http/scrape_prom_metrics_module_http_test.go b/integration-tests/tests/scrape-prom-metrics-module-http/scrape_prom_metrics_module_http_test.go index b847de78d444..d6cb64a2ac4e 100644 --- a/integration-tests/tests/scrape-prom-metrics-module-http/scrape_prom_metrics_module_http_test.go +++ b/integration-tests/tests/scrape-prom-metrics-module-http/scrape_prom_metrics_module_http_test.go @@ -1,78 +1,11 @@ package main import ( - "fmt" - "strconv" "testing" "github.com/grafana/agent/integration-tests/common" - "github.com/stretchr/testify/assert" ) -const promURL = "http://localhost:9009/prometheus/api/v1/query?query=" - -func metricQuery(metricName string) string { - return fmt.Sprintf("%s%s{test_name='scrape_prom_metrics_module_http'}", promURL, metricName) -} - func TestScrapePromMetricsModuleHTTP(t *testing.T) { - metrics := []string{ - // TODO: better differentiate these metric types? - "golang_counter", - "golang_gauge", - "golang_histogram_bucket", - "golang_summary", - "golang_native_histogram", - } - - for _, metric := range metrics { - metric := metric - t.Run(metric, func(t *testing.T) { - t.Parallel() - if metric == "golang_native_histogram" { - assertHistogramData(t, metricQuery(metric), metric) - } else { - assertMetricData(t, metricQuery(metric), metric) - } - }) - } -} - -func assertHistogramData(t *testing.T, query string, expectedMetric string) { - var metricResponse common.MetricResponse - assert.EventuallyWithT(t, func(c *assert.CollectT) { - err := common.FetchDataFromURL(query, &metricResponse) - assert.NoError(c, err) - if assert.NotEmpty(c, metricResponse.Data.Result) { - assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) - assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics_module_http") - if assert.NotNil(c, metricResponse.Data.Result[0].Histogram) { - histogram := metricResponse.Data.Result[0].Histogram - if assert.NotEmpty(c, histogram.Data.Count) { - count, _ := strconv.Atoi(histogram.Data.Count) - assert.Greater(c, count, 10, "Count should be at some point greater than 10.") - } - if assert.NotEmpty(c, histogram.Data.Sum) { - sum, _ := strconv.ParseFloat(histogram.Data.Sum, 64) - assert.Greater(c, sum, 10., "Sum should be at some point greater than 10.") - } - assert.NotEmpty(c, histogram.Data.Buckets) - assert.Nil(c, metricResponse.Data.Result[0].Value) - } - } - }, common.DefaultTimeout, common.DefaultRetryInterval, "Histogram data did not satisfy the conditions within the time limit") -} - -func assertMetricData(t *testing.T, query, expectedMetric string) { - var metricResponse common.MetricResponse - assert.EventuallyWithT(t, func(c *assert.CollectT) { - err := common.FetchDataFromURL(query, &metricResponse) - assert.NoError(c, err) - if assert.NotEmpty(c, metricResponse.Data.Result) { - assert.Equal(c, metricResponse.Data.Result[0].Metric.Name, expectedMetric) - assert.Equal(c, metricResponse.Data.Result[0].Metric.TestName, "scrape_prom_metrics_module_http") - assert.NotEmpty(c, metricResponse.Data.Result[0].Value.Value) - assert.Nil(c, metricResponse.Data.Result[0].Histogram) - } - }, common.DefaultTimeout, common.DefaultRetryInterval, "Data did not satisfy the conditions within the time limit") + common.MimirMetricsTest(t, common.PromDefaultMetrics, common.PromDefaultHistogramMetric, "scrape_prom_metrics_module_http") } From 60d6eca7f46fecf3070411707e59d17035ddcd80 Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 25 Jan 2024 17:25:01 +0100 Subject: [PATCH 36/36] cleanup --- pkg/flow/flow_components.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/flow/flow_components.go b/pkg/flow/flow_components.go index 7201a0507265..a60820c62988 100644 --- a/pkg/flow/flow_components.go +++ b/pkg/flow/flow_components.go @@ -31,7 +31,7 @@ func (f *Flow) GetComponent(id component.ID, opts component.InfoOptions) (*compo cn, ok := node.(controller.ComponentNode) if !ok { - return nil, fmt.Errorf("%q does not implement ComponentInfo", id) + return nil, fmt.Errorf("%q is not a component", id) } return f.getComponentDetail(cn, graph, opts), nil @@ -56,9 +56,9 @@ func (f *Flow) ListComponents(moduleID string, opts component.InfoOptions) ([]*c graph = f.loader.OriginalGraph() ) - detail := make([]*component.Info, 0, len(components)) - for _, component := range components { - detail = append(detail, f.getComponentDetail(component, graph, opts)) + detail := make([]*component.Info, len(components)) + for i, component := range components { + detail[i] = f.getComponentDetail(component, graph, opts) } return detail, nil }