I-16 - Make non-json log message field configurable with msg default.

components/k8s/provider_k8s_test.go

@@ -1,7 +1,6 @@
 package k8s
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -28,7 +27,6 @@ func TestRetrieveServices(t *testing.T) {
 	provider.Retrieve()
 
 	assert.Len(t, provider.services, 3)
-	fmt.Print(provider.services)
 	assert.Equal(t, "test", provider.services["test.2gis.ru"].Name)
 	assert.Equal(t, "test.2gis.ru", provider.services["test.2gis.ru"].Domains[0])
 	assert.Len(t, provider.services["test.2gis.ru"].Domains, 2)

configuration/config.go

@@ -42,6 +42,7 @@ type ParserConfig struct {
 	UserLogFieldsKey string
 	CRIFieldsKey     string
 	ExtendsFieldsKey string
+	RawLogFieldKey   string
 
 	FlattenUserLog bool
 }
@@ -308,6 +309,10 @@ func GetConfig() Config {
 		Default("").
 		Envar("EXTENDS_FIELDS_KEY").
 		StringVar(&config.ParserConfig.ExtendsFieldsKey)
+	kingpin.Flag("raw-log-field-key", "Entry field inside user log fields map. Used for non-json messages.").
+		Default("msg").
+		Envar("RAW_LOG_FIELD_KEY").
+		StringVar(&config.ParserConfig.RawLogFieldKey)
 
 	kingpin.Flag("flatten-user-log", "Whether to flatten user log or not.").
 		Default("true").

parsers/parser_containerd.go

@@ -26,7 +26,8 @@ func CreateParserContainerDFormat(config configuration.ParserConfig) func(line [
 
 		setContainerDFields(outer, config.CRIFieldsKey, output[1], output[2])
 
-		if err := setLogFieldContent(outer, config.UserLogFieldsKey, output[3], config.FlattenUserLog); err != nil {
+		if err := setLogFieldContent(
+			outer, config.UserLogFieldsKey, config.RawLogFieldKey, output[3], config.FlattenUserLog); err != nil {
 			return nil, fmt.Errorf("error setting user log field: %w", err)
 		}
 

parsers/parser_containerd_test.go

@@ -46,7 +46,7 @@ var testCasesParserContainerD = []testCaseParserContainerD{
 		config: configFlattenTopLevel(),
 		input:  "2020-09-10T07:00:03.585507743Z stdout F my message",
 		entryMapExpected: common.EntryMap{
-			"log":    "my message",
+			"msg":    "my message",
 			"stream": "stdout",
 			"time":   "2020-09-10T07:00:03.585507743Z",
 		},

parsers/parser_docker.go

@@ -38,7 +38,7 @@ func CreateParserDockerFormat(config configuration.ParserConfig) func(line []byt
 		setDockerFields(outer, config.CRIFieldsKey)
 
 		if err := setLogFieldContent(
-			outer, config.UserLogFieldsKey, logFieldContentString, config.FlattenUserLog); err != nil {
+			outer, config.UserLogFieldsKey, config.RawLogFieldKey, logFieldContentString, config.FlattenUserLog); err != nil {
 			return nil, fmt.Errorf("error setting user log field: %w", err)
 		}
 
@@ -65,46 +65,33 @@ func setDockerFields(entryMap common.EntryMap, targetField string) {
 	entryMap[targetField] = dockerFields
 }
 
-func setLogFieldContent(entryMap common.EntryMap, targetField, JSONContent string, flatten bool) error {
+func setLogFieldContent(entryMap common.EntryMap, userLogField, rawField, logFieldContent string, flatten bool) error {
 	var inner interface{}
 
-	err := json.Unmarshal([]byte(JSONContent), &inner)
+	baseMap := selectBaseMap(entryMap, userLogField)
+	err := json.Unmarshal([]byte(logFieldContent), &inner)
 	innerMap, ok := inner.(map[string]interface{})
-
 	if err != nil || !ok {
-		if targetField == "" {
-			entryMap[LogKeyLog] = JSONContent
-			return nil
-		}
-
-		entryMap[targetField] = JSONContent
+		baseMap[rawField] = logFieldContent
 		return nil
 	}
 
 	processNginxFields(innerMap)
 
 	if !flatten {
-		// flatten flag is not set and the target field is empty, we still need some field to set content to.
-		if targetField == "" {
-			entryMap[LogKeyLog] = common.EntryMap(innerMap)
-			return nil
-		}
-
-		entryMap[targetField] = common.EntryMap(innerMap)
+		baseMap.Extend(innerMap)
 		return nil
 	}
 
-	// unpack to top level dict, backward compatibility
-	if targetField == "" {
-		return common.Flatten(entryMap, innerMap)
-	}
-
-	innerMapUnpacked := make(common.EntryMap)
+	return common.Flatten(baseMap, innerMap)
+}
 
-	if err := common.Flatten(innerMapUnpacked, innerMap); err != nil {
-		return err
+func selectBaseMap(baseMap common.EntryMap, userLogField string) common.EntryMap {
+	if userLogField == "" {
+		return baseMap
 	}
 
-	entryMap[targetField] = innerMapUnpacked
-	return nil
+	subMap := make(common.EntryMap)
+	baseMap[userLogField] = subMap
+	return subMap
 }

parsers/parser_docker_test.go

@@ -23,7 +23,7 @@ var (
 		{
 			name:             "Positive, flattening to top-level",
 			input:            "{\"log\":\"hello world\", \"key1\":1}",
-			entryMapExpected: common.EntryMap{"log": "hello world", "key1": float64(1)},
+			entryMapExpected: common.EntryMap{"msg": "hello world", "key1": float64(1)},
 		},
 		{
 			name:             "log field is missing",
@@ -121,6 +121,7 @@ var (
 				UserLogFieldsKey: "user_log",
 				CRIFieldsKey:     "docker",
 				FlattenUserLog:   false,
+				RawLogFieldKey:   "msg",
 			},
 			input: `{"time": "2018-01-09T05:08:03.100481875Z", "log": "{\"time\":{\"value\": 1.142}}"}`,
 			entryMapExpected: common.EntryMap{
@@ -155,15 +156,16 @@ var (
 			},
 		},
 		{
-			name: "flattening is off, user log field is empty, user log expected in log field of top level dict",
+			name: "flattening is off, user log field is empty, user log expected in the level dict",
 			config: configuration.ParserConfig{
 				UserLogFieldsKey: "",
 				CRIFieldsKey:     "docker",
 				FlattenUserLog:   false,
+				RawLogFieldKey:   "msg",
 			},
 			input: `{"time": "2018-01-09T05:08:03.100481875Z", "log": "{\"time\":{\"value\": 1.142}}"}`,
 			entryMapExpected: common.EntryMap{
-				"log":    common.EntryMap{"time": map[string]interface{}{"value": 1.142}},
+				"time": map[string]interface{}{"value": 1.142},
 				"docker": common.EntryMap{"time": "2018-01-09T05:08:03.100481875Z"},
 			},
 		},
@@ -185,8 +187,8 @@ var (
 
 	testCasesParserDockerUnpackingControl = []testCaseParserDocker{
 		{
-			name:             "Different keys to unpack",
-			input:            `{"log": "{\"hello\":\"world\",\"a\": 1,\"b\": null}", "a": "else"}`,
+			name:  "Different keys to unpack",
+			input: `{"log": "{\"hello\":\"world\",\"a\": 1,\"b\": null}", "a": "else"}`,
 			entryMapExpected: common.EntryMap{
 				"log": common.EntryMap{"hello": "world", "a": float64(1), "b": nil},
 				"cri": common.EntryMap{"a": "else"},
@@ -273,6 +275,7 @@ func configFlattenTopLevel() configuration.ParserConfig {
 		UserLogFieldsKey: "",
 		CRIFieldsKey:     "",
 		FlattenUserLog:   true,
+		RawLogFieldKey:   "msg",
 	}
 }
 
@@ -281,5 +284,6 @@ func configFlattenSubDict() configuration.ParserConfig {
 		UserLogFieldsKey: LogKeyLog,
 		CRIFieldsKey:     "cri",
 		FlattenUserLog:   true,
+		RawLogFieldKey:   "msg",
 	}
 }