From c905116ade5260c2c839406d23dc2acb2be83387 Mon Sep 17 00:00:00 2001 From: David Bennett <71459415+Jagularr@users.noreply.github.com> Date: Thu, 11 Mar 2021 21:53:32 -0500 Subject: [PATCH] Add 'field name' config option to customize field name when using Value parser. (#8979) --- config/config.go | 4 +- plugins/inputs/exec/exec_test.go | 6 +- .../kafka_consumer/kafka_consumer_test.go | 6 +- plugins/parsers/registry.go | 12 +- plugins/parsers/value/parser.go | 16 ++- plugins/parsers/value/parser_test.go | 111 +++++------------- 6 files changed, 61 insertions(+), 94 deletions(-) diff --git a/config/config.go b/config/config.go index 58483428adb36..be64fba4bcf48 100644 --- a/config/config.go +++ b/config/config.go @@ -1342,6 +1342,8 @@ func (c *Config) getParserConfig(name string, tbl *ast.Table) (*parsers.Config, c.getFieldStringSlice(tbl, "form_urlencoded_tag_keys", &pc.FormUrlencodedTagKeys) + c.getFieldString(tbl, "value_field_name", &pc.ValueFieldName) + //for XML parser if node, ok := tbl.Fields["xml"]; ok { if subtbls, ok := node.([]*ast.Table); ok { @@ -1468,7 +1470,7 @@ func (c *Config) missingTomlField(typ reflect.Type, key string) error { "prefix", "prometheus_export_timestamp", "prometheus_sort_metrics", "prometheus_string_as_label", "separator", "splunkmetric_hec_routing", "splunkmetric_multimetric", "tag_keys", "tagdrop", "tagexclude", "taginclude", "tagpass", "tags", "template", "templates", - "wavefront_source_override", "wavefront_use_strict", "xml": + "value_field_name", "wavefront_source_override", "wavefront_use_strict", "xml": // ignore fields that are common to all plugins. default: diff --git a/plugins/inputs/exec/exec_test.go b/plugins/inputs/exec/exec_test.go index 5e614f61dd9ec..ba1bc2078c9f7 100644 --- a/plugins/inputs/exec/exec_test.go +++ b/plugins/inputs/exec/exec_test.go @@ -160,7 +160,7 @@ func TestCommandError(t *testing.T) { } func TestExecCommandWithGlob(t *testing.T) { - parser, _ := parsers.NewValueParser("metric", "string", nil) + parser, _ := parsers.NewValueParser("metric", "string", "", nil) e := NewExec() e.Commands = []string{"/bin/ech* metric_value"} e.SetParser(parser) @@ -176,7 +176,7 @@ func TestExecCommandWithGlob(t *testing.T) { } func TestExecCommandWithoutGlob(t *testing.T) { - parser, _ := parsers.NewValueParser("metric", "string", nil) + parser, _ := parsers.NewValueParser("metric", "string", "", nil) e := NewExec() e.Commands = []string{"/bin/echo metric_value"} e.SetParser(parser) @@ -192,7 +192,7 @@ func TestExecCommandWithoutGlob(t *testing.T) { } func TestExecCommandWithoutGlobAndPath(t *testing.T) { - parser, _ := parsers.NewValueParser("metric", "string", nil) + parser, _ := parsers.NewValueParser("metric", "string", "", nil) e := NewExec() e.Commands = []string{"echo metric_value"} e.SetParser(parser) diff --git a/plugins/inputs/kafka_consumer/kafka_consumer_test.go b/plugins/inputs/kafka_consumer/kafka_consumer_test.go index d7804a01b87e1..90c362b9c01ea 100644 --- a/plugins/inputs/kafka_consumer/kafka_consumer_test.go +++ b/plugins/inputs/kafka_consumer/kafka_consumer_test.go @@ -257,7 +257,7 @@ func (c *FakeConsumerGroupClaim) Messages() <-chan *sarama.ConsumerMessage { func TestConsumerGroupHandler_Lifecycle(t *testing.T) { acc := &testutil.Accumulator{} - parser := &value.ValueParser{MetricName: "cpu", DataType: "int"} + parser := value.NewValueParser("cpu", "int", "", nil) cg := NewConsumerGroupHandler(acc, 1, parser) ctx, cancel := context.WithCancel(context.Background()) @@ -282,7 +282,7 @@ func TestConsumerGroupHandler_Lifecycle(t *testing.T) { func TestConsumerGroupHandler_ConsumeClaim(t *testing.T) { acc := &testutil.Accumulator{} - parser := &value.ValueParser{MetricName: "cpu", DataType: "int"} + parser := value.NewValueParser("cpu", "int", "", nil) cg := NewConsumerGroupHandler(acc, 1, parser) ctx, cancel := context.WithCancel(context.Background()) @@ -392,7 +392,7 @@ func TestConsumerGroupHandler_Handle(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { acc := &testutil.Accumulator{} - parser := &value.ValueParser{MetricName: "cpu", DataType: "int"} + parser := value.NewValueParser("cpu", "int", "", nil) cg := NewConsumerGroupHandler(acc, 1, parser) cg.MaxMessageLen = tt.maxMessageLen cg.TopicTag = tt.topicTag diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index 44cde6c85b7ba..d01b0ee676565 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -152,6 +152,9 @@ type Config struct { // FormData configuration FormUrlencodedTagKeys []string `toml:"form_urlencoded_tag_keys"` + // Value configuration + ValueFieldName string `toml:"value_field_name"` + // XML configuration XMLConfig []XMLConfig `toml:"xml"` } @@ -182,7 +185,7 @@ func NewParser(config *Config) (Parser, error) { ) case "value": parser, err = NewValueParser(config.MetricName, - config.DataType, config.DefaultTags) + config.DataType, config.ValueFieldName, config.DefaultTags) case "influx": parser, err = NewInfluxParser() case "nagios": @@ -291,13 +294,10 @@ func NewGraphiteParser( func NewValueParser( metricName string, dataType string, + fieldName string, defaultTags map[string]string, ) (Parser, error) { - return &value.ValueParser{ - MetricName: metricName, - DataType: dataType, - DefaultTags: defaultTags, - }, nil + return value.NewValueParser(metricName, dataType, fieldName, defaultTags), nil } func NewCollectdParser( diff --git a/plugins/parsers/value/parser.go b/plugins/parsers/value/parser.go index a495033c47b28..95c87a2eae982 100644 --- a/plugins/parsers/value/parser.go +++ b/plugins/parsers/value/parser.go @@ -15,6 +15,7 @@ type ValueParser struct { MetricName string DataType string DefaultTags map[string]string + FieldName string } func (v *ValueParser) Parse(buf []byte) ([]telegraf.Metric, error) { @@ -46,7 +47,7 @@ func (v *ValueParser) Parse(buf []byte) ([]telegraf.Metric, error) { return nil, err } - fields := map[string]interface{}{"value": value} + fields := map[string]interface{}{v.FieldName: value} metric, err := metric.New(v.MetricName, v.DefaultTags, fields, time.Now().UTC()) if err != nil { @@ -73,3 +74,16 @@ func (v *ValueParser) ParseLine(line string) (telegraf.Metric, error) { func (v *ValueParser) SetDefaultTags(tags map[string]string) { v.DefaultTags = tags } + +func NewValueParser(metricName, dataType, fieldName string, defaultTags map[string]string) *ValueParser { + if fieldName == "" { + fieldName = "value" + } + + return &ValueParser{ + MetricName: metricName, + DataType: dataType, + DefaultTags: defaultTags, + FieldName: fieldName, + } +} diff --git a/plugins/parsers/value/parser_test.go b/plugins/parsers/value/parser_test.go index 667fb108cfbfb..5a74085d82980 100644 --- a/plugins/parsers/value/parser_test.go +++ b/plugins/parsers/value/parser_test.go @@ -7,10 +7,7 @@ import ( ) func TestParseValidValues(t *testing.T) { - parser := ValueParser{ - MetricName: "value_test", - DataType: "integer", - } + parser := NewValueParser("value_test", "integer", "", nil) metrics, err := parser.Parse([]byte("55")) assert.NoError(t, err) assert.Len(t, metrics, 1) @@ -20,10 +17,7 @@ func TestParseValidValues(t *testing.T) { }, metrics[0].Fields()) assert.Equal(t, map[string]string{}, metrics[0].Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "float", - } + parser = NewValueParser("value_test", "float", "", nil) metrics, err = parser.Parse([]byte("64")) assert.NoError(t, err) assert.Len(t, metrics, 1) @@ -33,10 +27,7 @@ func TestParseValidValues(t *testing.T) { }, metrics[0].Fields()) assert.Equal(t, map[string]string{}, metrics[0].Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "string", - } + parser = NewValueParser("value_test", "string", "", nil) metrics, err = parser.Parse([]byte("foobar")) assert.NoError(t, err) assert.Len(t, metrics, 1) @@ -46,10 +37,7 @@ func TestParseValidValues(t *testing.T) { }, metrics[0].Fields()) assert.Equal(t, map[string]string{}, metrics[0].Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "boolean", - } + parser = NewValueParser("value_test", "boolean", "", nil) metrics, err = parser.Parse([]byte("true")) assert.NoError(t, err) assert.Len(t, metrics, 1) @@ -61,10 +49,7 @@ func TestParseValidValues(t *testing.T) { } func TestParseMultipleValues(t *testing.T) { - parser := ValueParser{ - MetricName: "value_test", - DataType: "integer", - } + parser := NewValueParser("value_test", "integer", "", nil) metrics, err := parser.Parse([]byte(`55 45 223 @@ -80,11 +65,19 @@ func TestParseMultipleValues(t *testing.T) { assert.Equal(t, map[string]string{}, metrics[0].Tags()) } +func TestParseCustomFieldName(t *testing.T) { + parser := NewValueParser("value_test", "integer", "", nil) + parser.FieldName = "penguin" + metrics, err := parser.Parse([]byte(`55`)) + + assert.NoError(t, err) + assert.Equal(t, map[string]interface{}{ + "penguin": int64(55), + }, metrics[0].Fields()) +} + func TestParseLineValidValues(t *testing.T) { - parser := ValueParser{ - MetricName: "value_test", - DataType: "integer", - } + parser := NewValueParser("value_test", "integer", "", nil) metric, err := parser.ParseLine("55") assert.NoError(t, err) assert.Equal(t, "value_test", metric.Name()) @@ -93,10 +86,7 @@ func TestParseLineValidValues(t *testing.T) { }, metric.Fields()) assert.Equal(t, map[string]string{}, metric.Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "float", - } + parser = NewValueParser("value_test", "float", "", nil) metric, err = parser.ParseLine("64") assert.NoError(t, err) assert.Equal(t, "value_test", metric.Name()) @@ -105,10 +95,7 @@ func TestParseLineValidValues(t *testing.T) { }, metric.Fields()) assert.Equal(t, map[string]string{}, metric.Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "string", - } + parser = NewValueParser("value_test", "string", "", nil) metric, err = parser.ParseLine("foobar") assert.NoError(t, err) assert.Equal(t, "value_test", metric.Name()) @@ -117,10 +104,7 @@ func TestParseLineValidValues(t *testing.T) { }, metric.Fields()) assert.Equal(t, map[string]string{}, metric.Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "boolean", - } + parser = NewValueParser("value_test", "boolean", "", nil) metric, err = parser.ParseLine("true") assert.NoError(t, err) assert.Equal(t, "value_test", metric.Name()) @@ -131,59 +115,38 @@ func TestParseLineValidValues(t *testing.T) { } func TestParseInvalidValues(t *testing.T) { - parser := ValueParser{ - MetricName: "value_test", - DataType: "integer", - } + parser := NewValueParser("value_test", "integer", "", nil) metrics, err := parser.Parse([]byte("55.0")) assert.Error(t, err) assert.Len(t, metrics, 0) - parser = ValueParser{ - MetricName: "value_test", - DataType: "float", - } + parser = NewValueParser("value_test", "float", "", nil) metrics, err = parser.Parse([]byte("foobar")) assert.Error(t, err) assert.Len(t, metrics, 0) - parser = ValueParser{ - MetricName: "value_test", - DataType: "boolean", - } + parser = NewValueParser("value_test", "boolean", "", nil) metrics, err = parser.Parse([]byte("213")) assert.Error(t, err) assert.Len(t, metrics, 0) } func TestParseLineInvalidValues(t *testing.T) { - parser := ValueParser{ - MetricName: "value_test", - DataType: "integer", - } + parser := NewValueParser("value_test", "integer", "", nil) _, err := parser.ParseLine("55.0") assert.Error(t, err) - parser = ValueParser{ - MetricName: "value_test", - DataType: "float", - } + parser = NewValueParser("value_test", "float", "", nil) _, err = parser.ParseLine("foobar") assert.Error(t, err) - parser = ValueParser{ - MetricName: "value_test", - DataType: "boolean", - } + parser = NewValueParser("value_test", "boolean", "", nil) _, err = parser.ParseLine("213") assert.Error(t, err) } func TestParseValidValuesDefaultTags(t *testing.T) { - parser := ValueParser{ - MetricName: "value_test", - DataType: "integer", - } + parser := NewValueParser("value_test", "integer", "", nil) parser.SetDefaultTags(map[string]string{"test": "tag"}) metrics, err := parser.Parse([]byte("55")) assert.NoError(t, err) @@ -194,10 +157,7 @@ func TestParseValidValuesDefaultTags(t *testing.T) { }, metrics[0].Fields()) assert.Equal(t, map[string]string{"test": "tag"}, metrics[0].Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "float", - } + parser = NewValueParser("value_test", "float", "", nil) parser.SetDefaultTags(map[string]string{"test": "tag"}) metrics, err = parser.Parse([]byte("64")) assert.NoError(t, err) @@ -208,10 +168,7 @@ func TestParseValidValuesDefaultTags(t *testing.T) { }, metrics[0].Fields()) assert.Equal(t, map[string]string{"test": "tag"}, metrics[0].Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "string", - } + parser = NewValueParser("value_test", "string", "", nil) parser.SetDefaultTags(map[string]string{"test": "tag"}) metrics, err = parser.Parse([]byte("foobar")) assert.NoError(t, err) @@ -222,10 +179,7 @@ func TestParseValidValuesDefaultTags(t *testing.T) { }, metrics[0].Fields()) assert.Equal(t, map[string]string{"test": "tag"}, metrics[0].Tags()) - parser = ValueParser{ - MetricName: "value_test", - DataType: "boolean", - } + parser = NewValueParser("value_test", "boolean", "", nil) parser.SetDefaultTags(map[string]string{"test": "tag"}) metrics, err = parser.Parse([]byte("true")) assert.NoError(t, err) @@ -238,10 +192,7 @@ func TestParseValidValuesDefaultTags(t *testing.T) { } func TestParseValuesWithNullCharacter(t *testing.T) { - parser := ValueParser{ - MetricName: "value_test", - DataType: "integer", - } + parser := NewValueParser("value_test", "integer", "", nil) metrics, err := parser.Parse([]byte("55\x00")) assert.NoError(t, err) assert.Len(t, metrics, 1)