diff --git a/benchmark_test.go b/benchmark_test.go new file mode 100644 index 00000000..b15248dc --- /dev/null +++ b/benchmark_test.go @@ -0,0 +1,94 @@ +package humanlog + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "io/fs" + "os" + "path/filepath" + "testing" + + typesv1 "github.com/humanlogio/api/go/types/v1" + "github.com/stretchr/testify/require" +) + +type NopSink struct{} + +func (*NopSink) Receive(ctx context.Context, ev *typesv1.LogEvent) error { + return nil +} + +func (*NopSink) Close(ctx context.Context) error { + return nil +} + +func BenchmarkHarness(b *testing.B) { + ctx := context.Background() + root := "test/benchmark" + des, err := os.ReadDir(root) + if err != nil { + b.Fatal(err) + } + + for _, de := range des { + if !de.IsDir() { + continue + } + + dir := filepath.Join(root, de.Name()) + fileName, err := findfirstMatchedFileName(dir, "*.gz") + require.NoError(b, err) + + testCase := dir + b.Run(testCase, func(bb *testing.B) { + + p := filepath.Join(dir, fileName) + f, err := os.Open(p) + require.NoError(bb, err) + defer f.Close() + + gzipReader, err := gzip.NewReader(f) + require.NoError(bb, err) + + src := bytes.NewBuffer(nil) + _, err = io.Copy(src, gzipReader) + require.NoError(bb, err) + + sink := &NopSink{} + opt := DefaultOptions() + + bb.SetBytes(int64(src.Len())) + for range bb.N { + copy := bytes.NewBuffer(src.Bytes()) + bb.StartTimer() + _ = Scan(ctx, copy, sink, opt) + bb.StopTimer() + } + }) + } +} + +func findfirstMatchedFileName(dirPath string, pattern string) (string, error) { + firstMatched := "" + walkError := filepath.Walk(dirPath, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + fileName := filepath.Base(path) + match, err := filepath.Match(pattern, fileName) + if err != nil { + return err + } + if match { + firstMatched = fileName + return filepath.SkipAll + } + return nil + }) + return firstMatched, walkError +} diff --git a/json_handler.go b/json_handler.go index 9f5b471a..74c30aaf 100644 --- a/json_handler.go +++ b/json_handler.go @@ -96,7 +96,7 @@ func deleteJSONKey(key string, jsonData map[string]interface{}) { func getFlattenedFields(v map[string]interface{}) map[string]string { extValues := make(map[string]string) - for key, nestedVal := range v { + for key, nestedVal := range v { switch valTyped := nestedVal.(type) { case float64: if valTyped-math.Floor(valTyped) < 0.000001 && valTyped < 1e9 { @@ -108,13 +108,13 @@ func getFlattenedFields(v map[string]interface{}) map[string]string { extValues[key] = fmt.Sprintf("%q", valTyped) case map[string]interface{}: flattenedFields := getFlattenedFields(valTyped) - for keyNested, valStr := range flattenedFields { - extValues[key + "." + keyNested] = valStr + for keyNested, valStr := range flattenedFields { + extValues[key+"."+keyNested] = valStr } default: extValues[key] = fmt.Sprintf("%v", valTyped) } - } + } return extValues } @@ -174,8 +174,8 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool { h.Fields[key] = fmt.Sprintf("%q", v) case map[string]interface{}: flattenedFields := getFlattenedFields(v) - for keyNested, val := range flattenedFields { - h.Fields[key + "." + keyNested] = fmt.Sprintf("%v", val) + for keyNested, val := range flattenedFields { + h.Fields[key+"."+keyNested] = fmt.Sprintf("%v", val) } default: h.Fields[key] = fmt.Sprintf("%v", v) diff --git a/logs.txt b/logs.txt deleted file mode 100644 index 54466deb..00000000 --- a/logs.txt +++ /dev/null @@ -1,8 +0,0 @@ -{"data":{"short":"service-startup","message":"The login-api service is running on port 8085."},"datacontenttype":"application/json","id":"01FCV6S4M6S8H3VKAQD9SWFWFP","invloglevel":"Info","source":"irn:libraries:github.com/InVisionApp/invlogger","specversion":"1.0","time":"2021-08-11T13:14:50.246861-05:00","type":"simple-log"} -{"data":{"short":"http access","message":"incoming HTTP request was served"},"datacontenttype":"application/json","id":"01FCV6S9YK70GJ5Q6YT0PYKQDA","invapptracingcallingservice":"unset","invapptracingrequestid":"01FCV6S9YJXD2SYG3HTGWXHX0G","invapptracingtequestsource":"unset","invloglevel":"Info","invwebbytes":0,"invwebbytesin":0,"invwebbytesout":0,"invwebcached":false,"invwebdesthost":"","invwebdestip":"::ffff:0.0.0.0","invwebduration":0,"invwebhttpcontenttype":"","invwebhttpmethod":"","invwebhttpuseragent":"","invwebhttpuseragentlength":0,"invwebsrcip":"::ffff:0.0.0.0","invwebstatus":0,"invweburipath":"","invweburiquery":"","invweburl":"","invweburllength":0,"source":"irn:libraries:github.com/InVisionApp/invlogger","specversion":"1.0","time":"2021-08-11T13:14:55.699075-05:00","type":"incoming_http_request"} -{"data":{"short":"http access","message":"incoming HTTP request was served"},"datacontenttype":"application/json","id":"01FCV6SDKRW3XZDA1FAGZ3QVSH","invapptracingcallingservice":"unset","invapptracingrequestid":"01FCV6SDKRHB1RR1Q87Q1SKT5P","invapptracingtequestsource":"unset","invloglevel":"Info","invwebbytes":0,"invwebbytesin":0,"invwebbytesout":0,"invwebcached":false,"invwebdesthost":"","invwebdestip":"::ffff:0.0.0.0","invwebduration":0,"invwebhttpcontenttype":"","invwebhttpmethod":"","invwebhttpuseragent":"","invwebhttpuseragentlength":0,"invwebsrcip":"::ffff:0.0.0.0","invwebstatus":0,"invweburipath":"","invweburiquery":"","invweburl":"","invweburllength":0,"source":"irn:libraries:github.com/InVisionApp/invlogger","specversion":"1.0","time":"2021-08-11T13:14:59.448566-05:00","type":"incoming_http_request"} -{"data":{"short":"http access","message":"incoming HTTP request was served"},"datacontenttype":"application/json","id":"01FCV6SE597EY6RJ762V59PZQA","invapptracingcallingservice":"unset","invapptracingrequestid":"01FCV6SE596ZMASA1D79M16KVV","invapptracingtequestsource":"unset","invloglevel":"Info","invwebbytes":0,"invwebbytesin":0,"invwebbytesout":0,"invwebcached":false,"invwebdesthost":"","invwebdestip":"::ffff:0.0.0.0","invwebduration":0,"invwebhttpcontenttype":"","invwebhttpmethod":"","invwebhttpuseragent":"","invwebhttpuseragentlength":0,"invwebsrcip":"::ffff:0.0.0.0","invwebstatus":0,"invweburipath":"","invweburiquery":"","invweburl":"","invweburllength":0,"source":"irn:libraries:github.com/InVisionApp/invlogger","specversion":"1.0","time":"2021-08-11T13:15:00.009615-05:00","type":"incoming_http_request"} -{"data":{"short":"http access","message":"incoming HTTP request was served"},"datacontenttype":"application/json","id":"01FCV6SEKCC9RG364AJ60J75KW","invapptracingcallingservice":"unset","invapptracingrequestid":"01FCV6SEKCNSQJJ2NDEPQ2TGMP","invapptracingtequestsource":"unset","invloglevel":"Info","invwebbytes":0,"invwebbytesin":0,"invwebbytesout":0,"invwebcached":false,"invwebdesthost":"","invwebdestip":"::ffff:0.0.0.0","invwebduration":0,"invwebhttpcontenttype":"","invwebhttpmethod":"","invwebhttpuseragent":"","invwebhttpuseragentlength":0,"invwebsrcip":"::ffff:0.0.0.0","invwebstatus":0,"invweburipath":"","invweburiquery":"","invweburl":"","invweburllength":0,"source":"irn:libraries:github.com/InVisionApp/invlogger","specversion":"1.0","time":"2021-08-11T13:15:00.46025-05:00","type":"incoming_http_request"} -{"data":{"short":"http access","message":"incoming HTTP request was served"},"datacontenttype":"application/json","id":"01FCV6SF3DXGB8G1DVX19KQZYT","invapptracingcallingservice":"unset","invapptracingrequestid":"01FCV6SF3DJZSXTT1RNR6F1QAV","invapptracingtequestsource":"unset","invloglevel":"Info","invwebbytes":0,"invwebbytesin":0,"invwebbytesout":0,"invwebcached":false,"invwebdesthost":"","invwebdestip":"::ffff:0.0.0.0","invwebduration":0,"invwebhttpcontenttype":"","invwebhttpmethod":"","invwebhttpuseragent":"","invwebhttpuseragentlength":0,"invwebsrcip":"::ffff:0.0.0.0","invwebstatus":0,"invweburipath":"","invweburiquery":"","invweburl":"","invweburllength":0,"source":"irn:libraries:github.com/InVisionApp/invlogger","specversion":"1.0","time":"2021-08-11T13:15:00.973455-05:00","type":"incoming_http_request"} -{"data":{"short":"http access","message":"incoming HTTP request was served"},"datacontenttype":"application/json","id":"01FCV6SKY9MM7D795258XPQGC9","invapptracingcallingservice":"unset","invapptracingrequestid":"01FCV6SKY9M1D725HTV0ZXKF1V","invapptracingtequestsource":"unset","invloglevel":"Info","invwebbytes":0,"invwebbytesin":0,"invwebbytesout":0,"invwebcached":false,"invwebdesthost":"","invwebdestip":"::ffff:0.0.0.0","invwebduration":0,"invwebhttpcontenttype":"","invwebhttpmethod":"","invwebhttpuseragent":"","invwebhttpuseragentlength":0,"invwebsrcip":"::ffff:0.0.0.0","invwebstatus":0,"invweburipath":"","invweburiquery":"","invweburl":"","invweburllength":0,"source":"irn:libraries:github.com/InVisionApp/invlogger","specversion":"1.0","time":"2021-08-11T13:15:05.929423-05:00","type":"incoming_http_request"} -{"data":{"event":"Shutdown"},"datacontenttype":"application/json","id":"01FCV6SR6JZH7JZ6RFDFN9Q99Y","invloglevel":"Info","source":"irn:libraries:github.com/InVisionApp/invlogger","specversion":"1.0","time":"2021-08-11T13:15:10.290034-05:00","type":"service-shutdown"} \ No newline at end of file diff --git a/test/benchmark/csharp-nlog/sample-nlog-csv.log.gz b/test/benchmark/csharp-nlog-csv/sample-nlog-csv.log.gz similarity index 100% rename from test/benchmark/csharp-nlog/sample-nlog-csv.log.gz rename to test/benchmark/csharp-nlog-csv/sample-nlog-csv.log.gz diff --git a/test/benchmark/csharp-nlog-json/sample-nlog.json.gz b/test/benchmark/csharp-nlog-json/sample-nlog.json.gz new file mode 100644 index 00000000..340cf33c Binary files /dev/null and b/test/benchmark/csharp-nlog-json/sample-nlog.json.gz differ diff --git a/test/benchmark/csharp-nlog/sample-nlog-json.log.gz b/test/benchmark/csharp-nlog/sample-nlog-json.log.gz deleted file mode 100644 index c94b7888..00000000 Binary files a/test/benchmark/csharp-nlog/sample-nlog-json.log.gz and /dev/null differ diff --git a/test/benchmark/csharp-serilog/sample-serilog-json.log.gz b/test/benchmark/csharp-serilog-json/sample-serilog-json.log.gz similarity index 100% rename from test/benchmark/csharp-serilog/sample-serilog-json.log.gz rename to test/benchmark/csharp-serilog-json/sample-serilog-json.log.gz diff --git a/test/benchmark/golang-logrus-json/sample-logrus.json.gz b/test/benchmark/golang-logrus-json/sample-logrus.json.gz new file mode 100644 index 00000000..be3bdfbe Binary files /dev/null and b/test/benchmark/golang-logrus-json/sample-logrus.json.gz differ diff --git a/test/benchmark/golang-logrus/sample-logrus-json.log.gz b/test/benchmark/golang-logrus/sample-logrus-json.log.gz deleted file mode 100644 index 9aac960e..00000000 Binary files a/test/benchmark/golang-logrus/sample-logrus-json.log.gz and /dev/null differ diff --git a/test/benchmark/golang-zerolog/sample-zerolog-json.log.gz b/test/benchmark/golang-zerolog-json/sample-zerolog-json.log.gz similarity index 100% rename from test/benchmark/golang-zerolog/sample-zerolog-json.log.gz rename to test/benchmark/golang-zerolog-json/sample-zerolog-json.log.gz diff --git a/test/benchmark/java-log4j2/sample-log4j2-json.log.gz b/test/benchmark/java-log4j2-json/sample-log4j2-json.log.gz similarity index 100% rename from test/benchmark/java-log4j2/sample-log4j2-json.log.gz rename to test/benchmark/java-log4j2-json/sample-log4j2-json.log.gz diff --git a/test/benchmark/java-logback/sample-logback-extended-json.log.gz b/test/benchmark/java-logback-ext-json/sample-logback-extended-json.log.gz similarity index 100% rename from test/benchmark/java-logback/sample-logback-extended-json.log.gz rename to test/benchmark/java-logback-ext-json/sample-logback-extended-json.log.gz diff --git a/test/benchmark/java-logback/sample-logback-extended-text.log.gz b/test/benchmark/java-logback-ext/sample-logback-extended-text.log.gz similarity index 100% rename from test/benchmark/java-logback/sample-logback-extended-text.log.gz rename to test/benchmark/java-logback-ext/sample-logback-extended-text.log.gz diff --git a/test/benchmark/java-logback/sample-logback-json.log.gz b/test/benchmark/java-logback-json/sample-logback-json.log.gz similarity index 100% rename from test/benchmark/java-logback/sample-logback-json.log.gz rename to test/benchmark/java-logback-json/sample-logback-json.log.gz diff --git a/test/benchmark/java-logback/sample-logback-mdc-json.log.gz b/test/benchmark/java-logback-mdc-json/sample-logback-mdc-json.log.gz similarity index 100% rename from test/benchmark/java-logback/sample-logback-mdc-json.log.gz rename to test/benchmark/java-logback-mdc-json/sample-logback-mdc-json.log.gz diff --git a/test/benchmark/java-logback/sample-logback-mdc-text.log.gz b/test/benchmark/java-logback-mdc/sample-logback-mdc-text.log.gz similarity index 100% rename from test/benchmark/java-logback/sample-logback-mdc-text.log.gz rename to test/benchmark/java-logback-mdc/sample-logback-mdc-text.log.gz diff --git a/test/benchmark/opentelemetry/sample-opentelemetry-structured.log.gz b/test/benchmark/opentelemetry-json/sample-opentelemetry-structured.log.gz similarity index 100% rename from test/benchmark/opentelemetry/sample-opentelemetry-structured.log.gz rename to test/benchmark/opentelemetry-json/sample-opentelemetry-structured.log.gz diff --git a/test/benchmark/python-loguru/sample-loguru-json.log.gz b/test/benchmark/python-loguru-json/sample-loguru-json.log.gz similarity index 100% rename from test/benchmark/python-loguru/sample-loguru-json.log.gz rename to test/benchmark/python-loguru-json/sample-loguru-json.log.gz diff --git a/test/benchmark/ruby-semantic-logger/sample-semantic-logger-json.log.gz b/test/benchmark/ruby-semantic-logger-json/sample-semantic-logger-json.log.gz similarity index 100% rename from test/benchmark/ruby-semantic-logger/sample-semantic-logger-json.log.gz rename to test/benchmark/ruby-semantic-logger-json/sample-semantic-logger-json.log.gz