From fe231251601b2b0dfc5ccfb03ed72e8dcafaeda7 Mon Sep 17 00:00:00 2001 From: Jason Woods Date: Sun, 7 Feb 2021 11:49:04 +0000 Subject: [PATCH] Restore backwards compatibility with 2.0.6 by placing ECS fields behind a configuration, to be enabled in 3.0 Resolve go generate calling go run without -mod=vendor resulting in non-vendor access during packaging --- fact-courier/fact-courier.go | 2 +- lc-admin/lc-admin.go | 2 +- lc-lib/codecs/stream.go | 7 +++-- lc-lib/config/generate/platform.go | 2 +- lc-lib/core/version.go | 2 +- lc-lib/event/stream.go | 43 +++++++++++++++++++----------- lc-lib/harvester/harvester.go | 14 ++++++---- log-carver/log-carver.go | 2 +- log-courier.go | 2 +- 9 files changed, 48 insertions(+), 28 deletions(-) diff --git a/fact-courier/fact-courier.go b/fact-courier/fact-courier.go index 70d779dc..58880bd1 100644 --- a/fact-courier/fact-courier.go +++ b/fact-courier/fact-courier.go @@ -29,7 +29,7 @@ import ( ) // Generate platform-specific default configuration values -//go:generate go run ../lc-lib/config/generate/platform.go platform main config.DefaultConfigurationFile DefaultMuninConfigBase +//go:generate go run -mod=vendor ../lc-lib/config/generate/platform.go platform main config.DefaultConfigurationFile DefaultMuninConfigBase var ( log *logging.Logger diff --git a/lc-admin/lc-admin.go b/lc-admin/lc-admin.go index f9a7e4f4..015e97a1 100644 --- a/lc-admin/lc-admin.go +++ b/lc-admin/lc-admin.go @@ -30,7 +30,7 @@ import ( ) // Generate platform-specific default configuration values -//go:generate go run ../lc-lib/config/generate/platform.go platform main config.DefaultConfigurationFile prospector.DefaultGeneralPersistDir admin.DefaultAdminBind +//go:generate go run -mod=vendor ../lc-lib/config/generate/platform.go platform main config.DefaultConfigurationFile prospector.DefaultGeneralPersistDir admin.DefaultAdminBind type commandProcessor interface { ProcessCommand(string) bool diff --git a/lc-lib/codecs/stream.go b/lc-lib/codecs/stream.go index 4fd11f37..9d3858e4 100644 --- a/lc-lib/codecs/stream.go +++ b/lc-lib/codecs/stream.go @@ -123,12 +123,15 @@ func (cs *Stream) ProcessEvent(startOffset int64, endOffset int64, text string) func (cs *Stream) eventCallback(startOffset int64, endOffset int64, text string) { data := map[string]interface{}{ "message": text, - "log": map[string]interface{}{"offset": startOffset}, } // TODO: Deprecate if cs.streamConfig.AddOffsetField { - data["offset"] = startOffset + if cs.streamConfig.EnableECS { + data["log"] = map[string]interface{}{"offset": startOffset} + } else { + data["offset"] = startOffset + } } cs.eventFunc(startOffset, endOffset, data) diff --git a/lc-lib/config/generate/platform.go b/lc-lib/config/generate/platform.go index c05c77b4..324a2d77 100644 --- a/lc-lib/config/generate/platform.go +++ b/lc-lib/config/generate/platform.go @@ -53,7 +53,7 @@ const platformHeader = `// THIS IS A GO GENERATED FILE // Useful for package maintainers func main() { if len(os.Args) < 3 { - log.Fatalf("Usage: go run /config/generate.go -- ...") + log.Fatalf("Usage: go run -mod=vendor /config/generate.go -- ...") } targetFile := os.Args[1] + ".go" diff --git a/lc-lib/core/version.go b/lc-lib/core/version.go index 56c9941e..3a758cdc 100644 --- a/lc-lib/core/version.go +++ b/lc-lib/core/version.go @@ -17,4 +17,4 @@ package core // LogCourierVersion is the library version number -const LogCourierVersion string = "2.5.0" +const LogCourierVersion string = "2.5.1" diff --git a/lc-lib/event/stream.go b/lc-lib/event/stream.go index 4d480bf0..5da70f9c 100644 --- a/lc-lib/event/stream.go +++ b/lc-lib/event/stream.go @@ -26,12 +26,16 @@ import ( ) const ( + defaultStreamAddHostField bool = true defaultStreamAddTimezoneField bool = false + defaultEnableECS bool = false ) // StreamConfig holds the configuration for a log stream type StreamConfig struct { + AddHostField bool `config:"add host field"` AddTimezoneField bool `config:"add timezone field"` + EnableECS bool `config:"enable ecs"` Fields map[string]interface{} `config:"fields"` genConfig *config.General @@ -40,7 +44,9 @@ type StreamConfig struct { // Defaults initialises the default configuration for a log stream func (sc *StreamConfig) Defaults() { + sc.AddHostField = defaultStreamAddHostField sc.AddTimezoneField = defaultStreamAddTimezoneField + sc.EnableECS = defaultEnableECS sc.timezone = time.Now().Format("-0700 MST") } @@ -62,24 +68,31 @@ func (sc *StreamConfig) Validate(p *config.Parser, path string) (err error) { // to the data that will eventually become an event func (sc *StreamConfig) Decorate(data map[string]interface{}) map[string]interface{} { data["@timestamp"] = time.Now() - data["host"] = map[string]interface{}{ - "name": sc.genConfig.Host, - "hostname": sc.genConfig.Host, + if sc.AddHostField { + if sc.EnableECS { + data["host"] = map[string]interface{}{ + "name": sc.genConfig.Host, + "hostname": sc.genConfig.Host, + } + } else { + data["host"] = sc.genConfig.Host + } } - var ( - eventEntry map[string]interface{} - ok bool - ) - if eventEntry, ok = data["event"].(map[string]interface{}); !ok { - eventEntry = map[string]interface{}{} - data["event"] = eventEntry - } - eventEntry["timezone"] = sc.timezone - - // TODO: Deprecate if sc.AddTimezoneField { - data["timezone"] = sc.timezone + if sc.EnableECS { + var ( + eventEntry map[string]interface{} + ok bool + ) + if eventEntry, ok = data["event"].(map[string]interface{}); !ok { + eventEntry = map[string]interface{}{} + data["event"] = eventEntry + } + eventEntry["timezone"] = sc.timezone + } else { + data["timezone"] = sc.timezone + } } for k := range sc.genConfig.GlobalFields { diff --git a/lc-lib/harvester/harvester.go b/lc-lib/harvester/harvester.go index d0379e8d..597c4f7a 100644 --- a/lc-lib/harvester/harvester.go +++ b/lc-lib/harvester/harvester.go @@ -358,12 +358,16 @@ func (h *Harvester) statCheck() error { // eventCallback receives events from the final codec and ships them to the output func (h *Harvester) eventCallback(startOffset int64, endOffset int64, data map[string]interface{}) { - // data["log"] is provided by codecs StreamConfig - data["log"].(map[string]interface{})["file"] = map[string]interface{}{"path": h.path} - - // TODO: Deprecate if h.streamConfig.AddPathField { - data["path"] = h.path + if h.streamConfig.EnableECS { + if !h.streamConfig.AddOffsetField { + + } + // data["log"] is provided by codecs StreamConfig + data["log"].(map[string]interface{})["file"] = map[string]interface{}{"path": h.path} + } else { + data["path"] = h.path + } } // If we split any of the line data, tag it diff --git a/log-carver/log-carver.go b/log-carver/log-carver.go index 12f0ab2b..efb1b7e0 100644 --- a/log-carver/log-carver.go +++ b/log-carver/log-carver.go @@ -33,7 +33,7 @@ import ( ) // Generate platform-specific default configuration values -//go:generate go run ../lc-lib/config/generate/platform.go platform main config.DefaultConfigurationFile admin.DefaultAdminBind processor.DefaultGeoIPActionDatabase +//go:generate go run -mod=vendor ../lc-lib/config/generate/platform.go platform main config.DefaultConfigurationFile admin.DefaultAdminBind processor.DefaultGeoIPActionDatabase var ( app *core.App diff --git a/log-courier.go b/log-courier.go index c3fd673b..3b6567a0 100644 --- a/log-courier.go +++ b/log-courier.go @@ -34,7 +34,7 @@ import ( ) // Generate platform-specific default configuration values -//go:generate go run lc-lib/config/generate/platform.go platform main config.DefaultConfigurationFile prospector.DefaultGeneralPersistDir admin.DefaultAdminBind +//go:generate go run -mod=vendor lc-lib/config/generate/platform.go platform main config.DefaultConfigurationFile prospector.DefaultGeneralPersistDir admin.DefaultAdminBind var ( app *core.App