Wire up support for converting integrations-next configs to flow.

CHANGELOG.md

@@ -57,6 +57,10 @@ Main (unreleased)
 
 - Added support for `loki.write` to flush WAL on agent shutdown. (@thepalbi)
 
+- Add support for `integrations-next` static to flow config conversion. (@erikbaranowski)
+
+- Add support for passing extra arguments to the static converter such as `-config.expand-env`. (@erikbaranowski)
+
 - Added 'country' mmdb-type to log pipeline-stage geoip. (@superstes)
 
 ### Bugfixes

cmd/internal/flowmode/cmd_convert.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/grafana/agent/converter"
 	convert_diag "github.com/grafana/agent/converter/diag"
@@ -20,6 +21,7 @@ func convertCommand() *cobra.Command {
 		output:       "",
 		sourceFormat: "",
 		bypassErrors: false,
+		extraArgs:    "",
 	}
 
 	cmd := &cobra.Command{
@@ -41,7 +43,11 @@ The -f flag can be used to specify the format we are converting from.
 
 The -b flag can be used to bypass errors. Errors are defined as 
 non-critical issues identified during the conversion where an
-output can still be generated.`,
+output can still be generated.
+
+The -e flag can be used to pass extra arguments to the converter
+which were used by the original format. Multiple arguments can be passed
+by separating them with a space.`,
 		Args:         cobra.RangeArgs(0, 1),
 		SilenceUsage: true,
 
@@ -71,6 +77,7 @@ output can still be generated.`,
 	cmd.Flags().StringVarP(&f.report, "report", "r", f.report, "The filepath and filename where the report is written.")
 	cmd.Flags().StringVarP(&f.sourceFormat, "source-format", "f", f.sourceFormat, fmt.Sprintf("The format of the source file. Supported formats: %s.", supportedFormatsList()))
 	cmd.Flags().BoolVarP(&f.bypassErrors, "bypass-errors", "b", f.bypassErrors, "Enable bypassing errors when converting")
+	cmd.Flags().StringVarP(&f.extraArgs, "extra-args", "e", f.extraArgs, "Extra arguments from the original format used by the converter. Multiple arguments can be passed by separating them with a space.")
 	return cmd
 }
 
@@ -79,6 +86,7 @@ type flowConvert struct {
 	report       string
 	sourceFormat string
 	bypassErrors bool
+	extraArgs    string
 }
 
 func (fc *flowConvert) Run(configFile string) error {
@@ -112,7 +120,12 @@ func convert(r io.Reader, fc *flowConvert) error {
 		return err
 	}
 
-	riverBytes, diags := converter.Convert(inputBytes, converter.Input(fc.sourceFormat), []string{})
+	ea, err := parseExtraArgs(fc.extraArgs)
+	if err != nil {
+		return err
+	}
+
+	riverBytes, diags := converter.Convert(inputBytes, converter.Input(fc.sourceFormat), ea)
 	err = generateConvertReport(diags, fc)
 	if err != nil {
 		return err
@@ -174,3 +187,35 @@ func supportedFormatsList() string {
 	}
 	return strings.Join(ret, ", ")
 }
+
+func parseExtraArgs(extraArgs string) ([]string, error) {
+	var result []string
+	if extraArgs == "" {
+		return result, nil
+	}
+
+	arguments := strings.Fields(extraArgs)
+	fs := pflag.NewFlagSet("extra-args", pflag.ExitOnError)
+	for i, arg := range arguments {
+		var split []string
+		if arg[1] == '-' {
+			split := strings.SplitN(arg, "=", 2)
+			result = append(result, split[0])
+			if len(split) == 2 && split[1] != "" {
+				result = append(result, "")
+				fs.StringVar(&result[len(result)-1], split[i][2:], result[len(result)-1], "")
+			}
+		} else {
+			split = strings.SplitN(arg, "=", 2)
+			result = append(result, split[0])
+			if len(split) == 2 && split[1] != "" {
+				result = append(result, "")
+				fs.StringVarP(&result[len(result)-1], "", split[i][1:], result[len(result)-1], "")
+			}
+		}
+	}
+
+	fs.ParseErrorsWhitelist.UnknownFlags = true
+	err := fs.Parse(arguments)
+	return result, err
+}

cmd/internal/flowmode/cmd_convert_test.go

@@ -0,0 +1,46 @@
+package flowmode
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseExtraArgs(t *testing.T) {
+	type testCase struct {
+		name      string
+		extraArgs string
+		expected  []string
+	}
+
+	var testCases = []testCase{
+		{
+			name:      "full",
+			extraArgs: "--key=value",
+			expected:  []string{"--key", "value"},
+		},
+		{
+			name:      "shorthand",
+			extraArgs: "-k=value",
+			expected:  []string{"-k", "value"},
+		},
+		{
+			name:      "bool",
+			extraArgs: "--boolVariable",
+			expected:  []string{"--boolVariable"},
+		},
+		{
+			name:      "spaced",
+			extraArgs: "--key value",
+			expected:  []string{"--key", "value"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			res, err := parseExtraArgs(tc.extraArgs)
+			require.NoError(t, err)
+			require.Equal(t, tc.expected, res)
+		})
+	}
+}

cmd/internal/flowmode/cmd_run.go

@@ -123,6 +123,7 @@ depending on the nature of the reload error.
 		BoolVar(&r.disableReporting, "disable-reporting", r.disableReporting, "Disable reporting of enabled components to Grafana.")
 	cmd.Flags().StringVar(&r.configFormat, "config.format", r.configFormat, fmt.Sprintf("The format of the source file. Supported formats: %s.", supportedFormatsList()))
 	cmd.Flags().BoolVar(&r.configBypassConversionErrors, "config.bypass-conversion-errors", r.configBypassConversionErrors, "Enable bypassing errors when converting")
+	cmd.Flags().StringVar(&r.configExtraArgs, "config.extra-args", r.configExtraArgs, "Extra arguments from the original format used by the converter. Multiple arguments can be passed by separating them with a space.")
 	return cmd
 }
 
@@ -144,6 +145,7 @@ type flowRun struct {
 	clusterName                  string
 	configFormat                 string
 	configBypassConversionErrors bool
+	configExtraArgs              string
 }
 
 func (fr *flowRun) Run(configPath string) error {
@@ -263,7 +265,7 @@ func (fr *flowRun) Run(configPath string) error {
 
 	ready = f.Ready
 	reload = func() (*flow.Source, error) {
-		flowSource, err := loadFlowSource(configPath, fr.configFormat, fr.configBypassConversionErrors)
+		flowSource, err := loadFlowSource(configPath, fr.configFormat, fr.configBypassConversionErrors, fr.configExtraArgs)
 		defer instrumentation.InstrumentSHA256(flowSource.SHA256())
 		defer instrumentation.InstrumentLoad(err == nil)
 
@@ -362,7 +364,7 @@ func getEnabledComponentsFunc(f *flow.Flow) func() map[string]interface{} {
 	}
 }
 
-func loadFlowSource(path string, converterSourceFormat string, converterBypassErrors bool) (*flow.Source, error) {
+func loadFlowSource(path string, converterSourceFormat string, converterBypassErrors bool, configExtraArgs string) (*flow.Source, error) {
 	fi, err := os.Stat(path)
 	if err != nil {
 		return nil, err
@@ -403,7 +405,12 @@ func loadFlowSource(path string, converterSourceFormat string, converterBypassEr
 	}
 	if converterSourceFormat != "flow" {
 		var diags convert_diag.Diagnostics
-		bb, diags = converter.Convert(bb, converter.Input(converterSourceFormat), []string{})
+		ea, err := parseExtraArgs(configExtraArgs)
+		if err != nil {
+			return nil, err
+		}
+
+		bb, diags = converter.Convert(bb, converter.Input(converterSourceFormat), ea)
 		hasError := hasErrorLevel(diags, convert_diag.SeverityLevelError)
 		hasCritical := hasErrorLevel(diags, convert_diag.SeverityLevelCritical)
 		if hasCritical || (!converterBypassErrors && hasError) {

converter/internal/staticconvert/staticconvert.go

@@ -31,7 +31,7 @@ func Convert(in []byte, extraArgs []string) ([]byte, diag.Diagnostics) {
 	var diags diag.Diagnostics
 
 	fs := flag.NewFlagSet("convert", flag.ContinueOnError)
-	args := []string{"-config.file", "convert", "-config.expand-env"}
+	args := []string{"-config.file", "convert"}
 	args = append(args, extraArgs...)
 	staticConfig, err := config.LoadFromFunc(fs, args, func(_, _ string, expandEnvVars bool, c *config.Config) error {
 		return config.LoadBytes(in, expandEnvVars, c)

converter/internal/staticconvert/staticconvert_test.go

@@ -10,8 +10,8 @@ import (
 )
 
 func TestConvert(t *testing.T) {
-	test_common.TestDirectory(t, "testdata", ".yaml", true, []string{}, staticconvert.Convert)
-	test_common.TestDirectory(t, "testdata-v2", ".yaml", true, []string{"-enable-features", "integrations-next"}, staticconvert.Convert)
+	test_common.TestDirectory(t, "testdata", ".yaml", true, []string{"-config.expand-env"}, staticconvert.Convert)
+	test_common.TestDirectory(t, "testdata-v2", ".yaml", true, []string{"-enable-features", "integrations-next", "-config.expand-env"}, staticconvert.Convert)
 
 	if runtime.GOOS == "windows" {
 		test_common.TestDirectory(t, "testdata_windows", ".yaml", true, []string{}, staticconvert.Convert)

converter/internal/staticconvert/testdata-v2/integrations_v2.river

@@ -16,6 +16,11 @@ loki.write "logs_log_config" {
 	external_labels = {}
 }
 
+logging {
+	level  = "debug"
+	format = "json"
+}
+
 prometheus.exporter.azure "integrations_azure1" {
 	subscriptions = ["subId"]
 	resource_type = "Microsoft.Dashboard/grafana"

converter/internal/staticconvert/testdata-v2/integrations_v2.yaml

@@ -1,3 +1,7 @@
+server:
+  log_level: ${SOME_ENVIRONMENT_VARIABLE:='debug'}
+  log_format: json
+
 metrics:
   global:
     remote_write:

docs/sources/flow/getting-started/migrating-from-prometheus.md

@@ -225,7 +225,7 @@ Configuration conversion is done on a best-effort basis. {{< param "PRODUCT_ROOT
 
 After the configuration is converted, review the {{< param "PRODUCT_NAME" >}} configuration file created and verify that it's correct before starting to use it in a production environment.
 
-Review the following checklist:
+The following list is specific to the convert command and not {{< param "PRODUCT_NAME" >}}:
 
 * The following configurations aren't available for conversion to {{< param "PRODUCT_NAME" >}}: `rule_files`, `alerting`, `remote_read`, `storage`, and `tracing`.
   Any additional unsupported features are returned as errors during conversion.

docs/sources/flow/getting-started/migrating-from-promtail.md

@@ -204,7 +204,7 @@ Configuration conversion is done on a best-effort basis. {{< param "PRODUCT_ROOT
 
 After the configuration is converted, review the {{< param "PRODUCT_NAME" >}} configuration file created and verify that it's correct before starting to use it in a production environment.
 
-Review the following checklist:
+The following list is specific to the convert command and not {{< param "PRODUCT_NAME" >}}:
 
 * Check if you are using any extra command line arguments with Promtail that aren't present in your configuration file. For example, `-max-line-size`.
 * Check if you are setting any environment variables, whether [expanded in the config file][] itself or consumed directly by Promtail, such as `JAEGER_AGENT_HOST`.

docs/sources/flow/getting-started/migrating-from-static.md

@@ -297,15 +297,43 @@ loki.write "logs_varlogs" {
 }
 ```
 
+## Integrations Next
+
+You can convert [integrations next][] configurations by adding the `extra-args` flag for [convert][] or `config.extra-args` for [run][].
+
+{{< code >}}
+
+```static-binary
+AGENT_MODE=flow grafana-agent convert --source-format=static --extra-args="-enable-features=integrations-next" --output=<OUTPUT_CONFIG_PATH> <INPUT_CONFIG_PATH>
+```
+
+```flow-binary
+grafana-agent-flow convert --source-format=static --extra-args="-enable-features=integrations-next" --output=<OUTPUT_CONFIG_PATH> <INPUT_CONFIG_PATH>
+```
+
+{{< /code >}}
+
+ Replace the following:
+   * _`<INPUT_CONFIG_PATH>`_: The full path to the [Static][] configuration.
+   * _`<OUTPUT_CONFIG_PATH>`_: The full path to output the {{< param "PRODUCT_NAME" >}} configuration.
+
+## Environment Vars
+
+You can use the `-config.expand-env` command line flag to interpret environment variables in your Grafana Agent Static configuration.
+You can pass these flags to [convert][] with `--extra-args="-config.expand-env"` or to [run][] with `--config.extra-args="-config.expand-env"`.
+
+> It's possible to combine `integrations-next` with `expand-env`.
+> For [convert][], you can use `--extra-args="-enable-features=integrations-next -config.expand-env"`
+
 ## Limitations
 
 Configuration conversion is done on a best-effort basis. {{< param "PRODUCT_ROOT_NAME" >}} will issue warnings or errors where the conversion can't be performed.
 
 After the configuration is converted, review the {{< param "PRODUCT_NAME" >}} configuration file and verify that it's correct before starting to use it in a production environment.
 
-Review the following checklist:
+The following list is specific to the convert command and not {{< param "PRODUCT_NAME" >}}:
 
-* The following configuration options aren't available for conversion to {{< param "PRODUCT_NAME" >}}: [Integrations next][], [Traces][], and [Agent Management][].
+* The  [Traces][] and [Agent Management][] configuration options can't be converted to {{< param "PRODUCT_NAME" >}}.
   Any additional unsupported features are returned as errors during conversion.
 * There is no gRPC server to configure for {{< param "PRODUCT_NAME" >}}, as any non-default configuration will show as unsupported during the conversion.
 * Check if you are using any extra command line arguments with Static that aren't present in your configuration file. For example, `-server.http.address`.

docs/sources/flow/reference/cli/convert.md

@@ -48,6 +48,8 @@ The following flags are supported:
 
 * `--bypass-errors`, `-b`: Enable bypassing errors when converting.
 
+* `--extra-args`, `e`: Extra arguments from the original format used by the converter.
+
 [prometheus]: #prometheus
 [promtail]: #promtail
 [static]: #static
@@ -101,6 +103,13 @@ Refer to [Migrate from Promtail to {{< param "PRODUCT_NAME" >}}]({{< relref "../
 Using the `--source-format=static` will convert the source configuration from a
 [Grafana Agent Static]({{< relref "../../../static" >}}) configuration to a {{< param "PRODUCT_NAME" >}} configuration.
 
+Include `--extra-args` for passing additional command line flags from the original format.
+For example, `--extra-args="-enable-features=integrations-next"` will convert a Grafana Agent Static
+[integrations-next]({{< relref "../../../static/configuration/integrations/integrations-next/" >}})
+configuration to a {{< param "PRODUCT_NAME" >}} configuration. You can also
+expand environment variables with `--extra-args="-config.expand-env"`. You can combine multiple command line
+flags with a space between each flag, for example `--extra-args="-enable-features=integrations-next -config.expand-env"`.
+
 If you have unsupported features in a Static mode source configuration, you will receive [errors][] when you convert to a Flow mode configuration. The converter will
 also raise warnings for configuration options that may require your attention.
 

docs/sources/flow/reference/cli/run.md

@@ -63,6 +63,7 @@ The following flags are supported:
 * `--cluster.name`: Name to prevent nodes without this identifier from joining the cluster (default `""`).
 * `--config.format`: The format of the source file. Supported formats: `flow`, `prometheus`, `promtail`, `static` (default `"flow"`).
 * `--config.bypass-conversion-errors`: Enable bypassing errors when converting (default `false`).
+* `--config.extra-args`: Extra arguments from the original format used by the converter.
 
 [in-memory HTTP traffic]: {{< relref "../../concepts/component_controller.md#in-memory-traffic" >}}
 [data collection]: {{< relref "../../../data-collection" >}}
@@ -180,11 +181,14 @@ the source format to River and immediately starts running with the new
 configuration. This conversion uses the converter API described in the
 [grafana-agent-flow convert][] docs.
 
-If you also use the `--config.bypass-conversion-errors` command-line argument,
+If you include the `--config.bypass-conversion-errors` command-line argument,
 {{< param "PRODUCT_NAME" >}} will ignore any errors from the converter. Use this argument
 with caution because the resulting conversion may not be equivalent to the
 original configuration.
 
+Include `--config.extra-args` to pass additional command line flags from the original format to the converter.
+Refer to [grafana-agent-flow convert][] for more details on how `extra-args` work.
+
 [grafana-agent-flow convert]: {{< relref "./convert.md" >}}
 [clustering]:  {{< relref "../../concepts/clustering.md" >}}
 [go-discover]: https://github.com/hashicorp/go-discover