From 98b18c79d53daba1dc9d3c131d620be27d7d02c2 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Sun, 8 Dec 2024 17:32:03 +0900 Subject: [PATCH 01/17] feat!(logevent): switch to typed log events --- go.mod | 2 +- go.sum | 6 ++---- scanner_test.go | 34 ++++++++++------------------------ 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index f392879..d4fe7bb 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 github.com/go-logfmt/logfmt v0.5.1 github.com/google/uuid v1.6.0 - github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd + github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 github.com/kr/logfmt v0.0.0-20210122060352-19f9bcb100e6 github.com/lrstanley/bubblezone v0.0.0-20240914071701-b48c55a5e78e diff --git a/go.sum b/go.sum index ab015c5..9157c84 100644 --- a/go.sum +++ b/go.sum @@ -80,10 +80,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd h1:449C6cnB4W6DblDMPfCfA4xyEkiYMpngGf7TEX9O8ro= -github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I= -github.com/humanlogio/humanlog-pro v0.0.0-20241129104046-2603398b2ef1 h1:eDsZPPr9GBNNh8wy43wPlb/98R1uCHuxML/r99GIkP4= -github.com/humanlogio/humanlog-pro v0.0.0-20241129104046-2603398b2ef1/go.mod h1:q+Kj7V58BQVVzGgcoyhtE2RXXprEi6DrqmhV/cThKkc= +github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 h1:lsiJGrN2E5qW6yvD6TKTAHa8lURmnh6KAuHvRfU8csU= +github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I= github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 h1:tdUCzFh8qvnWNCmxub0KSj1lIiCeWqvRjsMSSIApneE= github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9/go.mod h1:zq05mTZQXvKheFiAGlPx6+VSo29jw2ER8oy8DIQKW2Q= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= diff --git a/scanner_test.go b/scanner_test.go index 2012820..53dc3f6 100644 --- a/scanner_test.go +++ b/scanner_test.go @@ -104,30 +104,16 @@ func TestFlatteningNestedObjects_with_a_big_number(t *testing.T) { Lvl: "DEBUG", Msg: "first match found at index", Kvs: []*typesv1.KV{ - { - Key: "source.function", - Value: "\"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch\"", - }, - { - Key: "source.file", - Value: "\"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go\"", - }, - { - Key: "source.line", - Value: "243", - }, - { - Key: "storage.machine.id", - Value: "5089", - }, - { - Key: "storage.session.id", - Value: "1730187806608637000", - }, - { - Key: "storage.i", - Value: "0", - }, + typesv1.KeyVal("source", typesv1.ValObj( + typesv1.KeyVal("function", typesv1.ValStr("github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch")), + typesv1.KeyVal("file", typesv1.ValStr("/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go")), + typesv1.KeyVal("line", typesv1.ValI64(243)), + )), + typesv1.KeyVal("storage", typesv1.ValObj( + typesv1.KeyVal("machine.id", typesv1.ValI64(5089)), + typesv1.KeyVal("session.id", typesv1.ValI64(1730187806608637000)), + typesv1.KeyVal("i", typesv1.ValI64(0)), + )), }, }, }, From fad0e772d149da3ea1943e0cc1bab4531f98a4ea Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Fri, 13 Dec 2024 16:02:04 +0900 Subject: [PATCH 02/17] chore(localstorage): use `cmp` for test comparison --- go.mod | 1 + pkg/localstorage/test_suite.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index d4fe7bb..a205c1b 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/fatih/color v1.16.0 github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 github.com/go-logfmt/logfmt v0.5.1 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 diff --git a/pkg/localstorage/test_suite.go b/pkg/localstorage/test_suite.go index 888907b..62eaf9f 100644 --- a/pkg/localstorage/test_suite.go +++ b/pkg/localstorage/test_suite.go @@ -5,9 +5,10 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" typesv1 "github.com/humanlogio/api/go/types/v1" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -213,7 +214,8 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) { require.Len(t, got, len(tt.want)) for i := range tt.want { - require.Equal(t, protojson.Format(tt.want[i]), protojson.Format(got[i])) + diff := cmp.Diff(tt.want[i], got[i], protocmp.Transform()) + require.Empty(t, diff) } }) } From 3056ae5461fbf070630479a4a968d0f905ffd662 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Fri, 13 Dec 2024 16:38:46 +0900 Subject: [PATCH 03/17] dependency: update to humanlogio/api/go@a1e1ce8 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a205c1b..0adc065 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/go-logfmt/logfmt v0.5.1 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 - github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 + github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72 github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 github.com/kr/logfmt v0.0.0-20210122060352-19f9bcb100e6 github.com/lrstanley/bubblezone v0.0.0-20240914071701-b48c55a5e78e diff --git a/go.sum b/go.sum index 9157c84..86c4218 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 h1:lsiJGrN2E5qW6yvD6TKTAHa8lURmnh6KAuHvRfU8csU= github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I= +github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72 h1:68dDinP4+R4eaEaVXOCiZEwypLk3aUEw4gnF7QL+oH0= +github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72/go.mod h1:pFt3YKuAVJk5nziOiKXTKyq5fj4aA9azq6xOx/932KQ= github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 h1:tdUCzFh8qvnWNCmxub0KSj1lIiCeWqvRjsMSSIApneE= github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9/go.mod h1:zq05mTZQXvKheFiAGlPx6+VSo29jw2ER8oy8DIQKW2Q= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= From cbe3b131977fab77f789406b496d9eea73db6863 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Fri, 13 Dec 2024 16:42:17 +0900 Subject: [PATCH 04/17] test(typed key-value): resolve and put key-value into map[string]string --- pkg/sink/stdiosink/stdio.go | 66 +++++++++++++++++++++++++--- pkg/sink/stdiosink/stdio_test.go | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 pkg/sink/stdiosink/stdio_test.go diff --git a/pkg/sink/stdiosink/stdio.go b/pkg/sink/stdiosink/stdio.go index f5d6e76..e454ca7 100644 --- a/pkg/sink/stdiosink/stdio.go +++ b/pkg/sink/stdiosink/stdio.go @@ -11,6 +11,7 @@ import ( "time" "github.com/fatih/color" + "github.com/humanlogio/api/go/pkg/logql" typesv1 "github.com/humanlogio/api/go/types/v1" "github.com/humanlogio/humanlog/internal/pkg/config" "github.com/humanlogio/humanlog/pkg/sink" @@ -228,7 +229,12 @@ func (std *Stdio) ReceiveWithPostProcess(ctx context.Context, ev *typesv1.LogEve kvs := make(map[string]string, len(data.Kvs)) for _, kv := range data.Kvs { - kvs[kv.Key] = kv.Value + key := kv.Key + value, err := logql.ResolveVal(kv.Value, logql.MakeFlatGoMap, logql.MakeFlatMapGoSlice) + if err != nil { + return err + } + put(&kvs, key, value) } std.lastRaw = false std.lastLevel = ev.Structured.Lvl @@ -236,6 +242,52 @@ func (std *Stdio) ReceiveWithPostProcess(ctx context.Context, ev *typesv1.LogEve return nil } +func toString(value *typesv1.Val) (string, error) { + v, err := logql.ResolveVal(value, nil, nil) + if err != nil { + return "", err + } + switch t := v.(type) { + case string: + return t, nil + case int64: + return fmt.Sprintf("%d", t), nil + case float64: + return fmt.Sprintf("%g", t), nil + case bool: + return fmt.Sprintf("%t", t), nil + case time.Time: + return t.Format(time.RFC3339Nano), nil + case time.Duration: + return t.String(), nil + default: + return "", fmt.Errorf("unsupported type: %T", t) + } +} + +func put(ref *map[string]string, key string, value any) { + switch t := value.(type) { + case string: + (*ref)[key] = t + case int64: + (*ref)[key] = fmt.Sprintf("%d", t) + case float64: + (*ref)[key] = fmt.Sprintf("%g", t) + case bool: + (*ref)[key] = fmt.Sprintf("%t", t) + case time.Time: + (*ref)[key] = t.Format(time.RFC3339Nano) + case time.Duration: + (*ref)[key] = t.String() + case map[string]any: + for k, v := range t { + put(ref, key+"."+k, v) + } + default: + (*ref)[key] = fmt.Sprintf("%v", t) + } +} + func (std *Stdio) joinKVs(data *typesv1.StructuredLogEvent, sep string) []string { wasSameLevel := std.lastLevel == data.Lvl skipUnchanged := !std.lastRaw && std.opts.SkipUnchanged && wasSameLevel @@ -246,19 +298,23 @@ func (std *Stdio) joinKVs(data *typesv1.StructuredLogEvent, sep string) []string if !std.opts.shouldShowKey(k) { continue } + w, err := toString(v) + if err != nil { + continue + } if skipUnchanged { - if lastV, ok := std.lastKVs[k]; ok && lastV == v && !std.opts.shouldShowUnchanged(k) { + if lastV, ok := std.lastKVs[k]; ok && lastV == w && !std.opts.shouldShowUnchanged(k) { continue } } kstr := std.opts.Palette.KeyColor.Sprint(k) var vstr string - if std.opts.Truncates && len(v) > std.opts.TruncateLength { - vstr = v[:std.opts.TruncateLength] + "..." + if std.opts.Truncates && len(w) > std.opts.TruncateLength { + vstr = w[:std.opts.TruncateLength] + "..." } else { - vstr = v + vstr = w } vstr = std.opts.Palette.ValColor.Sprint(vstr) kv = append(kv, kstr+sep+vstr) diff --git a/pkg/sink/stdiosink/stdio_test.go b/pkg/sink/stdiosink/stdio_test.go new file mode 100644 index 0000000..d11ff61 --- /dev/null +++ b/pkg/sink/stdiosink/stdio_test.go @@ -0,0 +1,75 @@ +package stdiosink + +import ( + "testing" + + "github.com/humanlogio/api/go/pkg/logql" + typesv1 "github.com/humanlogio/api/go/types/v1" + "github.com/stretchr/testify/require" +) + +func TestPutKV(t *testing.T) { + tests := []struct { + name string + args *typesv1.KV + want map[string]string + }{ + { + name: "convert into map[string]string", + args: typesv1.KeyVal("root", typesv1.ValObj( + typesv1.KeyVal("a", typesv1.ValStr("lorem")), + typesv1.KeyVal("b", typesv1.ValI64(int64(1))), + typesv1.KeyVal("c", typesv1.ValF64(float64(3.14))), + typesv1.KeyVal("d", typesv1.ValBool(true)), + typesv1.KeyVal("e", typesv1.ValObj( + typesv1.KeyVal("f", typesv1.ValStr("foo")), + )), + typesv1.KeyVal("g", typesv1.ValArr( + typesv1.ValStr("bar"), + typesv1.ValI64(int64(2)), + typesv1.ValF64(float64(4.2)), + typesv1.ValBool(false), + typesv1.ValObj( + typesv1.KeyVal("h", typesv1.ValStr("baz")), + ), + typesv1.ValArr( + typesv1.ValStr("qux"), + typesv1.ValI64(int64(3)), + typesv1.ValF64(float64(5.3)), + typesv1.ValBool(true), + ), + ), + ))), + want: map[string]string{ + "root.a": "lorem", + "root.b": "1", + "root.c": "3.14", + "root.d": "true", + "root.e.f": "foo", + "root.g.0": "bar", + "root.g.1": "2", + "root.g.2": "4.2", + "root.g.3": "false", + "root.g.4.h": "baz", + "root.g.5.0": "qux", + "root.g.5.1": "3", + "root.g.5.2": "5.3", + "root.g.5.3": "true", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + key := tt.args.Key + value, err := logql.ResolveVal(tt.args.Value, logql.MakeFlatGoMap, logql.MakeFlatMapGoSlice) + require.NoError(t, err) + + kvs := make(map[string]string) + put(&kvs, key, value) + + got := kvs + require.Equal(t, tt.want, got) + }) + } +} From 9db52644df79ab8a10f33d4eeee066b01d7f3839 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Fri, 13 Dec 2024 19:04:35 +0900 Subject: [PATCH 05/17] feat(typed key-value): fix all compile errors --- cmd/humanlog/query.go | 7 +++- docker_compose_handler.go | 4 +-- internal/localsvc/svc.go | 2 +- internal/memstorage/memory.go | 2 +- json_handler.go | 44 ++++++++++++------------- logfmt_handler.go | 8 ++--- pkg/tui/components/querybar/querybar.go | 4 +-- zap_development_handler.go | 4 +-- 8 files changed, 40 insertions(+), 35 deletions(-) diff --git a/cmd/humanlog/query.go b/cmd/humanlog/query.go index 1b5fac7..8ccc4d2 100644 --- a/cmd/humanlog/query.go +++ b/cmd/humanlog/query.go @@ -17,6 +17,7 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/charmbracelet/x/term" "github.com/crazy3lf/colorconv" + "github.com/humanlogio/api/go/pkg/logql" "github.com/humanlogio/api/go/svc/environment/v1/environmentv1connect" "github.com/humanlogio/api/go/svc/organization/v1/organizationv1connect" queryv1 "github.com/humanlogio/api/go/svc/query/v1" @@ -330,12 +331,16 @@ func queryApiWatchCmd( if state.CurrentEnvironmentID != nil { environmentID = *state.CurrentEnvironmentID } + lq, err := logql.ParseLogQuery(query) + if err != nil { + return fmt.Errorf("parsing query: %v", err) + } req := &queryv1.WatchQueryRequest{ EnvironmentId: environmentID, Query: &typesv1.LogQuery{ From: from, To: to, - Query: query, + Query: lq.Query, }, } res, err := queryClient.WatchQuery(ctx, connect.NewRequest(req)) diff --git a/docker_compose_handler.go b/docker_compose_handler.go index 46c2b3a..c615d08 100644 --- a/docker_compose_handler.go +++ b/docker_compose_handler.go @@ -24,7 +24,7 @@ func tryDockerComposePrefix(d []byte, ev *typesv1.StructuredLogEvent, nextHandle if matches != nil { if nextHandler.TryHandle(matches[2], ev) { ev.Kvs = append(ev.Kvs, &typesv1.KV{ - Key: "service", Value: string(matches[1]), + Key: "service", Value: typesv1.ValStr(string(matches[1])), }) return true } @@ -34,7 +34,7 @@ func tryDockerComposePrefix(d []byte, ev *typesv1.StructuredLogEvent, nextHandle case *JSONHandler: if tryZapDevDCPrefix(matches[2], ev, h) { ev.Kvs = append(ev.Kvs, &typesv1.KV{ - Key: "service", Value: string(matches[1]), + Key: "service", Value: typesv1.ValStr(string(matches[1])), }) return true } diff --git a/internal/localsvc/svc.go b/internal/localsvc/svc.go index 185dcad..746f9a9 100644 --- a/internal/localsvc/svc.go +++ b/internal/localsvc/svc.go @@ -275,7 +275,7 @@ func (svc *Service) WatchQuery(ctx context.Context, req *connect.Request[qrv1.Wa query := req.Msg.GetQuery() ll := svc.ll.With( - slog.String("query.query", query.Query), + slog.String("query.query", query.Query.String()), ) if query.From != nil { ll = ll.With(slog.String("query.from", query.From.AsTime().Format(time.RFC3339Nano))) diff --git a/internal/memstorage/memory.go b/internal/memstorage/memory.go index e9f1729..17f703c 100644 --- a/internal/memstorage/memory.go +++ b/internal/memstorage/memory.go @@ -223,7 +223,7 @@ func newMemStorageSink(ll *slog.Logger, id SinkID) *MemStorageSink { func (snk *MemStorageSink) queryLogger(q *typesv1.LogQuery) *slog.Logger { ll := snk.ll.With( slog.Bool("sink.closed", snk.closed), - slog.String("query", q.Query), + slog.String("query", q.Query.String()), ) if q.From != nil { ll = ll.With(slog.Time("from", q.From.AsTime())) diff --git a/json_handler.go b/json_handler.go index 650ba42..211b819 100644 --- a/json_handler.go +++ b/json_handler.go @@ -19,7 +19,7 @@ type JSONHandler struct { Level string Time time.Time Message string - Fields map[string]string + Fields map[string]*typesv1.Val } // searchJSON searches a document for a key using the found func to determine if the value is accepted. @@ -55,7 +55,7 @@ func (h *JSONHandler) clear() { h.Level = "" h.Time = time.Time{} h.Message = "" - h.Fields = make(map[string]string) + h.Fields = make(map[string]*typesv1.Val) } // TryHandle tells if this line was handled by this handler. @@ -95,22 +95,22 @@ func deleteJSONKey(key string, jsonData map[string]interface{}) { } } -func getFlattenedFields(v map[string]interface{}) map[string]string { - extValues := make(map[string]string) +func getFlattenedFields(v map[string]interface{}) map[string]*typesv1.Val { + extValues := make(map[string]*typesv1.Val) for key, nestedVal := range v { switch valTyped := nestedVal.(type) { case json.Number: if z, err := valTyped.Int64(); err == nil { - extValues[key] = fmt.Sprintf("%d", z) + extValues[key] = typesv1.ValI64(z) continue } if f, err := valTyped.Float64(); err == nil { - extValues[key] = fmt.Sprintf("%g", f) + extValues[key] = typesv1.ValF64(f) continue } - extValues[key] = valTyped.String() + extValues[key] = typesv1.ValStr(valTyped.String()) case string: - extValues[key] = fmt.Sprintf("%q", valTyped) + extValues[key] = typesv1.ValStr(valTyped) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(valTyped) for k, v := range flattenedArrayFields { @@ -122,26 +122,26 @@ func getFlattenedFields(v map[string]interface{}) map[string]string { extValues[key+"."+keyNested] = valStr } default: - extValues[key] = fmt.Sprintf("%v", valTyped) + extValues[key] = typesv1.ValStr(fmt.Sprintf("%v", valTyped)) } } return extValues } -func getFlattenedArrayFields(data []interface{}) map[string]string { - flattened := make(map[string]string) +func getFlattenedArrayFields(data []interface{}) map[string]*typesv1.Val { + flattened := make(map[string]*typesv1.Val) for i, v := range data { switch vt := v.(type) { case json.Number: if z, err := vt.Int64(); err == nil { - flattened[strconv.Itoa(i)] = fmt.Sprintf("%d", z) + flattened[strconv.Itoa(i)] = typesv1.ValI64(z) } else if f, err := vt.Float64(); err == nil { - flattened[strconv.Itoa(i)] = fmt.Sprintf("%g", f) + flattened[strconv.Itoa(i)] = typesv1.ValF64(f) } else { - flattened[strconv.Itoa(i)] = vt.String() + flattened[strconv.Itoa(i)] = typesv1.ValStr(vt.String()) } case string: - flattened[strconv.Itoa(i)] = vt + flattened[strconv.Itoa(i)] = typesv1.ValStr(vt) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(vt) for k, v := range flattenedArrayFields { @@ -153,7 +153,7 @@ func getFlattenedArrayFields(data []interface{}) map[string]string { flattened[fmt.Sprintf("%d.%s", i, k)] = v } default: - flattened[strconv.Itoa(i)] = fmt.Sprintf("%v", vt) + flattened[strconv.Itoa(i)] = typesv1.ValStr(fmt.Sprintf("%v", vt)) } } return flattened @@ -203,23 +203,23 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool { }) if h.Fields == nil { - h.Fields = make(map[string]string) + h.Fields = make(map[string]*typesv1.Val) } for key, val := range raw { switch v := val.(type) { case json.Number: if z, err := v.Int64(); err == nil { - h.Fields[key] = fmt.Sprintf("%d", z) + h.Fields[key] = typesv1.ValI64(z) continue } if f, err := v.Float64(); err == nil { - h.Fields[key] = fmt.Sprintf("%g", f) + h.Fields[key] = typesv1.ValF64(f) continue } - h.Fields[key] = v.String() + h.Fields[key] = typesv1.ValStr(v.String()) case string: - h.Fields[key] = fmt.Sprintf("%q", v) + h.Fields[key] = typesv1.ValStr(v) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(v) for k, v := range flattenedArrayFields { @@ -231,7 +231,7 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool { h.Fields[key+"."+keyNested] = val } default: - h.Fields[key] = fmt.Sprintf("%v", v) + h.Fields[key] = typesv1.ValStr(fmt.Sprintf("%v", v)) } } diff --git a/logfmt_handler.go b/logfmt_handler.go index f3a9647..75f8efe 100644 --- a/logfmt_handler.go +++ b/logfmt_handler.go @@ -16,14 +16,14 @@ type LogfmtHandler struct { Level string Time time.Time Message string - Fields map[string]string + Fields map[string]*typesv1.Val } func (h *LogfmtHandler) clear() { h.Level = "" h.Time = time.Time{} h.Message = "" - h.Fields = make(map[string]string) + h.Fields = make(map[string]*typesv1.Val) } // CanHandle tells if this line can be handled by this handler. @@ -47,7 +47,7 @@ func (h *LogfmtHandler) TryHandle(d []byte, out *typesv1.StructuredLogEvent) boo // HandleLogfmt sets the fields of the handler. func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) bool { if h.Fields == nil { - h.Fields = make(map[string]string) + h.Fields = make(map[string]*typesv1.Val) } dec := logfmt.NewDecoder(bytes.NewReader(data)) for dec.ScanRecord() { @@ -97,7 +97,7 @@ func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) bool { } } - h.Fields[string(key)] = string(val) + h.Fields[string(key)] = typesv1.ValStr(string(val)) } } return dec.Err() == nil diff --git a/pkg/tui/components/querybar/querybar.go b/pkg/tui/components/querybar/querybar.go index c05e0f9..d4f5fb2 100644 --- a/pkg/tui/components/querybar/querybar.go +++ b/pkg/tui/components/querybar/querybar.go @@ -4,7 +4,7 @@ import ( "github.com/charmbracelet/bubbles/key" "github.com/charmbracelet/bubbles/textarea" tea "github.com/charmbracelet/bubbletea" - "github.com/humanlogio/api/go/pkg/lang" + "github.com/humanlogio/api/go/pkg/logql" typesv1 "github.com/humanlogio/api/go/types/v1" ) @@ -53,7 +53,7 @@ func (m *QueryBar) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case tea.KeyMsg: if key.Matches(msg, m.submitQuery) { q := m.textArea.Value() - qq, err := lang.ParseLogQuery(q) + qq, err := logql.ParseLogQuery(q) if err != nil { m.problems = append(m.problems, err.Error()) } else { diff --git a/zap_development_handler.go b/zap_development_handler.go index f20fb4e..bf702ef 100644 --- a/zap_development_handler.go +++ b/zap_development_handler.go @@ -40,7 +40,7 @@ func tryZapDevPrefix(d []byte, ev *typesv1.StructuredLogEvent, handler *JSONHand ev.Lvl = strings.ToLower(string(matches[2])) ev.Msg = string(matches[4]) ev.Kvs = append(ev.Kvs, &typesv1.KV{ - Key: "caller", Value: string(matches[3]), + Key: "caller", Value: typesv1.ValStr(string(matches[3])), }) return true } @@ -64,7 +64,7 @@ func tryZapDevDCPrefix(d []byte, ev *typesv1.StructuredLogEvent, handler *JSONHa ev.Msg = string(matches[4]) ev.Kvs = append( ev.Kvs, - &typesv1.KV{Key: "caller", Value: string(matches[3])}, + &typesv1.KV{Key: "caller", Value: typesv1.ValStr(string(matches[3]))}, ) return true } From ebf266229389c825b28b63e219a40c8ce5337606 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Fri, 13 Dec 2024 19:05:23 +0900 Subject: [PATCH 06/17] test(typed key-value): rewrite the tests --- json_handler_test.go | 65 ++++++++++--------- scanner_test.go | 109 ++++++++++++++++---------------- zap_development_handler_test.go | 2 +- 3 files changed, 87 insertions(+), 89 deletions(-) diff --git a/json_handler_test.go b/json_handler_test.go index 2eea1f6..ee95aa6 100644 --- a/json_handler_test.go +++ b/json_handler_test.go @@ -1,4 +1,4 @@ -package humanlog_test +package humanlog import ( "fmt" @@ -7,7 +7,6 @@ import ( "github.com/google/go-cmp/cmp" typesv1 "github.com/humanlogio/api/go/types/v1" - "github.com/humanlogio/humanlog" "github.com/stretchr/testify/require" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" @@ -24,9 +23,9 @@ func TestJSONHandler_UnmarshalJSON_ParsesFields(t *testing.T) { raw := []byte(fmt.Sprintf(`{ "message": %q, "level": %q, "time": %q }`, msg, level, tm)) - opts := humanlog.DefaultOptions() + opts := DefaultOptions() - h := humanlog.JSONHandler{Opts: opts} + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(raw, ev) { t.Fatalf("failed to parse log level") @@ -56,12 +55,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomFields(t *testing.T) { raw := []byte(fmt.Sprintf(`{ "mymessage": %q, "mylevel": %q, "mytime": %q }`, msg, level, tm)) - opts := humanlog.DefaultOptions() + opts := DefaultOptions() opts.LevelFields = []string{"mylevel"} opts.MessageFields = []string{"mymessage"} opts.TimeFields = []string{"mytime"} - h := humanlog.JSONHandler{Opts: opts} + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(raw, ev) { @@ -91,12 +90,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomNestedFields(t *testing.T) { raw := []byte(fmt.Sprintf(`{ "data": { "message": %q, "level": %q, "time": %q }}`, msg, level, tm)) - opts := humanlog.DefaultOptions() + opts := DefaultOptions() opts.LevelFields = []string{"data.level"} opts.MessageFields = []string{"data.message"} opts.TimeFields = []string{"data.time"} - h := humanlog.JSONHandler{Opts: opts} + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(raw, ev) { t.Fatalf("failed to handle log") @@ -134,12 +133,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomMultiNestedFields(t *testing.T) { } }`, msg, level, tm)) - opts := humanlog.DefaultOptions() + opts := DefaultOptions() opts.LevelFields = []string{"data.l2.level"} opts.MessageFields = []string{"data.l2.message"} opts.TimeFields = []string{"data.l2.time"} - h := humanlog.JSONHandler{Opts: opts} + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(raw, ev) { t.Fatalf("failed to handle log") @@ -159,48 +158,48 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomMultiNestedFields(t *testing.T) { } func TestJsonHandler_TryHandle_LargeNumbers(t *testing.T) { - h := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()} + h := JSONHandler{Opts: DefaultOptions()} ev := new(typesv1.StructuredLogEvent) raw := []byte(`{"storage":{"session.id":1730187806608637000, "some": {"float": 1.2345}}}`) if !h.TryHandle(raw, ev) { t.Fatalf("failed to handle log") } - require.Equal(t, "1.2345", h.Fields["storage.some.float"]) - require.Equal(t, "1730187806608637000", h.Fields["storage.session.id"]) + require.Equal(t, 1.2345, h.Fields["storage.some.float"].GetF64()) + require.Equal(t, 1730187806608637000, h.Fields["storage.session.id"].GetI64()) } func TestJsonHandler_TryHandle_FlattendArrayFields(t *testing.T) { - handler := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()} + handler := JSONHandler{Opts: DefaultOptions()} ev := new(typesv1.StructuredLogEvent) raw := []byte(`{"peers":[{"ID":"10.244.0.126:8083","URI":"10.244.0.126:8083"},{"ID":"10.244.0.206:8083","URI":"10.244.0.206:8083"},{"ID":"10.244.1.150:8083","URI":"10.244.1.150:8083"}],"storage":{"session.id":1730187806608637000, "some": {"float": 1.2345}}}`) if !handler.TryHandle(raw, ev) { t.Fatalf("failed to handle log") } - require.Equal(t, "\"10.244.0.126:8083\"", handler.Fields["peers.0.ID"]) - require.Equal(t, "\"10.244.0.126:8083\"", handler.Fields["peers.0.URI"]) - require.Equal(t, "\"10.244.0.206:8083\"", handler.Fields["peers.1.ID"]) - require.Equal(t, "\"10.244.0.206:8083\"", handler.Fields["peers.1.URI"]) - require.Equal(t, "\"10.244.1.150:8083\"", handler.Fields["peers.2.ID"]) - require.Equal(t, "\"10.244.1.150:8083\"", handler.Fields["peers.2.URI"]) + require.Equal(t, "10.244.0.126:8083", handler.Fields["peers.0.ID"].GetStr()) + require.Equal(t, "10.244.0.126:8083", handler.Fields["peers.0.URI"].GetStr()) + require.Equal(t, "10.244.0.206:8083", handler.Fields["peers.1.ID"].GetStr()) + require.Equal(t, "10.244.0.206:8083", handler.Fields["peers.1.URI"].GetStr()) + require.Equal(t, "10.244.1.150:8083", handler.Fields["peers.2.ID"].GetStr()) + require.Equal(t, "10.244.1.150:8083", handler.Fields["peers.2.URI"].GetStr()) } func TestJsonHandler_TryHandle_FlattenedArrayFields_NestedArray(t *testing.T) { - handler := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()} + handler := JSONHandler{Opts: DefaultOptions()} ev := new(typesv1.StructuredLogEvent) raw := []byte(`{"peers":[[1,2,3.14],[4,50.55,[6,7]],["hello","world"],{"foo":"bar"}]}`) if !handler.TryHandle(raw, ev) { t.Fatalf("failed to handle log") } - require.Equal(t, "1", handler.Fields["peers.0.0"]) - require.Equal(t, "2", handler.Fields["peers.0.1"]) - require.Equal(t, "3.14", handler.Fields["peers.0.2"]) - require.Equal(t, "4", handler.Fields["peers.1.0"]) - require.Equal(t, "50.55", handler.Fields["peers.1.1"]) - require.Equal(t, "6", handler.Fields["peers.1.2.0"]) - require.Equal(t, "7", handler.Fields["peers.1.2.1"]) - require.Equal(t, "hello", handler.Fields["peers.2.0"]) - require.Equal(t, "world", handler.Fields["peers.2.1"]) - require.Equal(t, "\"bar\"", handler.Fields["peers.3.foo"]) + require.Equal(t, "1", handler.Fields["peers.0.0"].GetI64()) + require.Equal(t, "2", handler.Fields["peers.0.1"].GetI64()) + require.Equal(t, "3.14", handler.Fields["peers.0.2"].GetF64()) + require.Equal(t, "4", handler.Fields["peers.1.0"].GetI64()) + require.Equal(t, "50.55", handler.Fields["peers.1.1"].GetF64()) + require.Equal(t, "6", handler.Fields["peers.1.2.0"].GetI64()) + require.Equal(t, "7", handler.Fields["peers.1.2.1"].GetI64()) + require.Equal(t, "hello", handler.Fields["peers.2.0"].GetStr()) + require.Equal(t, "world", handler.Fields["peers.2.1"].GetStr()) + require.Equal(t, "bar", handler.Fields["peers.3.foo"].GetStr()) } func TestParseAsctimeFields(t *testing.T) { @@ -223,8 +222,8 @@ func TestParseAsctimeFields(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - opts := humanlog.DefaultOptions() - h := humanlog.JSONHandler{Opts: opts} + opts := DefaultOptions() + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(test.raw, ev) { t.Fatalf("failed to handle log") diff --git a/scanner_test.go b/scanner_test.go index 0a44c70..a4dbb8c 100644 --- a/scanner_test.go +++ b/scanner_test.go @@ -6,11 +6,13 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" typesv1 "github.com/humanlogio/api/go/types/v1" "github.com/humanlogio/humanlog/pkg/sink/bufsink" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -100,20 +102,16 @@ func TestFlatteningNestedObjects_with_a_big_number(t *testing.T) { ParsedAt: timestamppb.New(now), Raw: []byte(`{"time":"2024-10-29T16:45:54.384776+09:00","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}`), Structured: &typesv1.StructuredLogEvent{ - Timestamp: timestamppb.New(time.Date(2024, 10, 29, 16, 14, 54, 384776000, time.Local)), + Timestamp: timestamppb.New(time.Date(2024, 10, 29, 16, 45, 54, 384776000, time.Local)), Lvl: "DEBUG", Msg: "first match found at index", Kvs: []*typesv1.KV{ - typesv1.KeyVal("source", typesv1.ValObj( - typesv1.KeyVal("function", typesv1.ValStr("github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch")), - typesv1.KeyVal("file", typesv1.ValStr("/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go")), - typesv1.KeyVal("line", typesv1.ValI64(243)), - )), - typesv1.KeyVal("storage", typesv1.ValObj( - typesv1.KeyVal("machine.id", typesv1.ValI64(5089)), - typesv1.KeyVal("session.id", typesv1.ValI64(1730187806608637000)), - typesv1.KeyVal("i", typesv1.ValI64(0)), - )), + typesv1.KeyVal("source.function", typesv1.ValStr("github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch")), + typesv1.KeyVal("source.file", typesv1.ValStr("/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go")), + typesv1.KeyVal("source.line", typesv1.ValI64(243)), + typesv1.KeyVal("storage.machine.id", typesv1.ValI64(5089)), + typesv1.KeyVal("storage.session.id", typesv1.ValI64(1730187806608637000)), + typesv1.KeyVal("storage.i", typesv1.ValI64(0)), }, }, }, @@ -129,15 +127,24 @@ func TestFlatteningNestedObjects_with_a_big_number(t *testing.T) { err := Scan(ctx, src, sink, opts) require.NoError(t, err) - for i, got := range sink.Buffered { - expectedKvs := make(map[string]string) + got := sink.Buffered + require.Equal(t, len(want), len(got)) // assume that there's no skipped log events + + n := len(want) + for i := 0; i < n; i++ { + actualKvs := make(map[string]*typesv1.Val) + for _, kv := range got[i].Structured.Kvs { + actualKvs[kv.Key] = kv.Value + } + expectedKvs := make(map[string]*typesv1.Val) for _, kv := range want[i].Structured.Kvs { expectedKvs[kv.Key] = kv.Value } - actualKvs := make(map[string]string) - for _, kv := range got.Structured.Kvs { - actualKvs[kv.Key] = kv.Value - } + require.Equal(t, got[i].ParsedAt, want[i].ParsedAt) + require.Equal(t, got[i].Raw, want[i].Raw) + require.Equal(t, got[i].Structured.Timestamp, want[i].Structured.Timestamp) + require.Equal(t, got[i].Structured.Msg, want[i].Structured.Msg) + require.Equal(t, got[i].Structured.Lvl, want[i].Structured.Lvl) require.Equal(t, expectedKvs, actualKvs) } } @@ -156,7 +163,7 @@ func TestFlatteningNestedObjects_simple(t *testing.T) { Kvs: []*typesv1.KV{ { Key: "storage.from", - Value: "\"" + time.Date(2024, 10, 29, 5, 47, 0, 0, time.UTC).Format(time.RFC3339) + "\"", + Value: typesv1.ValTime(time.Date(2024, 10, 29, 5, 47, 0, 0, time.UTC)), }, }, }, @@ -176,17 +183,9 @@ func TestFlatteningNestedObjects_simple(t *testing.T) { got := sink.Buffered require.Equal(t, len(want), len(got)) // assume that there's no skipped log events - n := len(want) - for i := 0; i < n; i++ { - actual := make(map[string]string) - for _, kv := range got[i].Structured.Kvs { - actual[kv.Key] = kv.Value - } - expected := make(map[string]string) - for _, kv := range want[i].Structured.Kvs { - expected[kv.Key] = kv.Value - } - require.Equal(t, expected, actual) + for i, got := range sink.Buffered { + diff := cmp.Diff(want[i], got, protocmp.Transform()) + require.Empty(t, diff) } } @@ -205,43 +204,43 @@ func TestFlatteningNestedObjects_with_arrays(t *testing.T) { Kvs: []*typesv1.KV{ { Key: "selfURI", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "source.function", - Value: "\"main.realMain.func5.1\"", + Value: typesv1.ValStr("main.realMain.func5.1"), }, { Key: "source.file", - Value: "\"github.com/humanlogio/apisvc/cmd/apisvc/server_cmd.go\"", + Value: typesv1.ValStr("github.com/humanlogio/apisvc/cmd/apisvc/server_cmd.go"), }, { Key: "source.line", - Value: "407", + Value: typesv1.ValI64(407), }, { Key: "peers.0.ID", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "peers.0.URI", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "peers.1.ID", - Value: "\"10.244.0.206:8083\"", + Value: typesv1.ValStr("10.244.0.206:8083"), }, { Key: "peers.1.URI", - Value: "\"10.244.0.206:8083\"", + Value: typesv1.ValStr("10.244.0.206:8083"), }, { Key: "peers.2.ID", - Value: "\"10.244.1.150:8083\"", + Value: typesv1.ValStr("10.244.1.150:8083"), }, { Key: "peers.2.URI", - Value: "\"10.244.1.150:8083\"", + Value: typesv1.ValStr("10.244.1.150:8083"), }, }, }, @@ -264,11 +263,11 @@ func TestFlatteningNestedObjects_with_arrays(t *testing.T) { n := len(want) for i := 0; i < n; i++ { - actualKvs := make(map[string]string) + actualKvs := make(map[string]*typesv1.Val) for _, kv := range got[i].Structured.Kvs { actualKvs[kv.Key] = kv.Value } - expectedKvs := make(map[string]string) + expectedKvs := make(map[string]*typesv1.Val) for _, kv := range want[i].Structured.Kvs { expectedKvs[kv.Key] = kv.Value } @@ -296,51 +295,51 @@ func TestFlatteningNestedObjects_with_nested_arrays(t *testing.T) { Kvs: []*typesv1.KV{ { Key: "peers.0.0", - Value: "1", + Value: typesv1.ValI64(1), }, { Key: "peers.0.1", - Value: "2", + Value: typesv1.ValI64(2), }, { Key: "peers.0.2", - Value: "3", + Value: typesv1.ValI64(3), }, { Key: "peers.1.0", - Value: "4", + Value: typesv1.ValI64(4), }, { Key: "peers.1.1", - Value: "5", + Value: typesv1.ValI64(5), }, { Key: "peers.1.2", - Value: "6", + Value: typesv1.ValI64(6), }, { Key: "peers.2.0.ID", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "peers.2.0.URI", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "peers.2.1.ID", - Value: "\"10.244.0.206:8083\"", + Value: typesv1.ValStr("10.244.0.206:8083"), }, { Key: "peers.2.1.URI", - Value: "\"10.244.0.206:8083\"", + Value: typesv1.ValStr("10.244.0.206:8083"), }, { Key: "peers.2.2.ID", - Value: "\"10.244.1.150:8083\"", + Value: typesv1.ValStr("10.244.1.150:8083"), }, { Key: "peers.2.2.URI", - Value: "\"10.244.1.150:8083\"", + Value: typesv1.ValStr("10.244.1.150:8083"), }, }, }, @@ -363,11 +362,11 @@ func TestFlatteningNestedObjects_with_nested_arrays(t *testing.T) { n := len(want) for i := 0; i < n; i++ { - actualKvs := make(map[string]string) + actualKvs := make(map[string]*typesv1.Val) for _, kv := range got[i].Structured.Kvs { actualKvs[kv.Key] = kv.Value } - expectedKvs := make(map[string]string) + expectedKvs := make(map[string]*typesv1.Val) for _, kv := range want[i].Structured.Kvs { expectedKvs[kv.Key] = kv.Value } diff --git a/zap_development_handler_test.go b/zap_development_handler_test.go index ca20d88..35a5299 100644 --- a/zap_development_handler_test.go +++ b/zap_development_handler_test.go @@ -229,7 +229,7 @@ func Test_tryZapDevPrefix(t *testing.T) { func findFieldValue(ev *typesv1.StructuredLogEvent, field string) string { for _, kv := range ev.Kvs { if kv.Key == field { - return kv.Value + return kv.Value.String() } } return "" From 0515aed4fc29e97c7fdd6075308691aca4b119ad Mon Sep 17 00:00:00 2001 From: KevRiver Date: Fri, 13 Dec 2024 19:50:41 +0900 Subject: [PATCH 07/17] wip(typed key-value): failed to regress, need to check Stdio.ReceiveWithPostProcess --- pkg/sink/stdiosink/stdio.go | 21 +++++++++++++++------ pkg/sink/stdiosink/stdio_test.go | 13 ++++++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/pkg/sink/stdiosink/stdio.go b/pkg/sink/stdiosink/stdio.go index e454ca7..cfabdc5 100644 --- a/pkg/sink/stdiosink/stdio.go +++ b/pkg/sink/stdiosink/stdio.go @@ -230,6 +230,15 @@ func (std *Stdio) ReceiveWithPostProcess(ctx context.Context, ev *typesv1.LogEve kvs := make(map[string]string, len(data.Kvs)) for _, kv := range data.Kvs { key := kv.Key + // value could be one of the following: + // - string + // - int64 + // - float64 + // - bool + // - time.Time + // - time.Duration + // - []any + // - map[string]any value, err := logql.ResolveVal(kv.Value, logql.MakeFlatGoMap, logql.MakeFlatMapGoSlice) if err != nil { return err @@ -249,7 +258,7 @@ func toString(value *typesv1.Val) (string, error) { } switch t := v.(type) { case string: - return t, nil + return fmt.Sprintf("%q", t), nil case int64: return fmt.Sprintf("%d", t), nil case float64: @@ -257,9 +266,9 @@ func toString(value *typesv1.Val) (string, error) { case bool: return fmt.Sprintf("%t", t), nil case time.Time: - return t.Format(time.RFC3339Nano), nil + return fmt.Sprintf("%q", t.Format(time.RFC3339Nano)), nil case time.Duration: - return t.String(), nil + return fmt.Sprintf("%q", t.String()), nil default: return "", fmt.Errorf("unsupported type: %T", t) } @@ -268,7 +277,7 @@ func toString(value *typesv1.Val) (string, error) { func put(ref *map[string]string, key string, value any) { switch t := value.(type) { case string: - (*ref)[key] = t + (*ref)[key] = fmt.Sprintf("%q", t) case int64: (*ref)[key] = fmt.Sprintf("%d", t) case float64: @@ -276,9 +285,9 @@ func put(ref *map[string]string, key string, value any) { case bool: (*ref)[key] = fmt.Sprintf("%t", t) case time.Time: - (*ref)[key] = t.Format(time.RFC3339Nano) + (*ref)[key] = fmt.Sprintf("%q", t.Format(time.RFC3339Nano)) case time.Duration: - (*ref)[key] = t.String() + (*ref)[key] = fmt.Sprintf("%q", t.String()) case map[string]any: for k, v := range t { put(ref, key+"."+k, v) diff --git a/pkg/sink/stdiosink/stdio_test.go b/pkg/sink/stdiosink/stdio_test.go index d11ff61..242b112 100644 --- a/pkg/sink/stdiosink/stdio_test.go +++ b/pkg/sink/stdiosink/stdio_test.go @@ -2,6 +2,7 @@ package stdiosink import ( "testing" + "time" "github.com/humanlogio/api/go/pkg/logql" typesv1 "github.com/humanlogio/api/go/types/v1" @@ -38,23 +39,25 @@ func TestPutKV(t *testing.T) { typesv1.ValF64(float64(5.3)), typesv1.ValBool(true), ), + typesv1.ValTime(time.Date(2024, 12, 13, 19, 36, 0, 0, time.UTC)), ), ))), want: map[string]string{ - "root.a": "lorem", + "root.a": "\"lorem\"", "root.b": "1", "root.c": "3.14", "root.d": "true", - "root.e.f": "foo", - "root.g.0": "bar", + "root.e.f": "\"foo\"", + "root.g.0": "\"bar\"", "root.g.1": "2", "root.g.2": "4.2", "root.g.3": "false", - "root.g.4.h": "baz", - "root.g.5.0": "qux", + "root.g.4.h": "\"baz\"", + "root.g.5.0": "\"qux\"", "root.g.5.1": "3", "root.g.5.2": "5.3", "root.g.5.3": "true", + "root.g.6": "\"2024-12-13T19:36:00Z\"", }, }, } From 812a666adff46b497f4aa173d75f14c389611ea5 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Mon, 16 Dec 2024 22:42:41 +0900 Subject: [PATCH 08/17] fix(json handler): add boolean value handling case --- json_handler.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/json_handler.go b/json_handler.go index 211b819..019e4ef 100644 --- a/json_handler.go +++ b/json_handler.go @@ -111,6 +111,8 @@ func getFlattenedFields(v map[string]interface{}) map[string]*typesv1.Val { extValues[key] = typesv1.ValStr(valTyped.String()) case string: extValues[key] = typesv1.ValStr(valTyped) + case bool: + extValues[key] = typesv1.ValBool(valTyped) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(valTyped) for k, v := range flattenedArrayFields { @@ -142,6 +144,8 @@ func getFlattenedArrayFields(data []interface{}) map[string]*typesv1.Val { } case string: flattened[strconv.Itoa(i)] = typesv1.ValStr(vt) + case bool: + flattened[strconv.Itoa(i)] = typesv1.ValBool(vt) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(vt) for k, v := range flattenedArrayFields { @@ -220,6 +224,8 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool { h.Fields[key] = typesv1.ValStr(v.String()) case string: h.Fields[key] = typesv1.ValStr(v) + case bool: + h.Fields[key] = typesv1.ValBool(v) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(v) for k, v := range flattenedArrayFields { From 2433efb9e9809a070010fe8991be520353d46038 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Mon, 16 Dec 2024 22:44:06 +0900 Subject: [PATCH 09/17] test(json handler): change expected value from string to typed value --- json_handler_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/json_handler_test.go b/json_handler_test.go index ee95aa6..71d4905 100644 --- a/json_handler_test.go +++ b/json_handler_test.go @@ -165,7 +165,7 @@ func TestJsonHandler_TryHandle_LargeNumbers(t *testing.T) { t.Fatalf("failed to handle log") } require.Equal(t, 1.2345, h.Fields["storage.some.float"].GetF64()) - require.Equal(t, 1730187806608637000, h.Fields["storage.session.id"].GetI64()) + require.Equal(t, int64(1730187806608637000), h.Fields["storage.session.id"].GetI64()) } func TestJsonHandler_TryHandle_FlattendArrayFields(t *testing.T) { @@ -190,13 +190,13 @@ func TestJsonHandler_TryHandle_FlattenedArrayFields_NestedArray(t *testing.T) { if !handler.TryHandle(raw, ev) { t.Fatalf("failed to handle log") } - require.Equal(t, "1", handler.Fields["peers.0.0"].GetI64()) - require.Equal(t, "2", handler.Fields["peers.0.1"].GetI64()) - require.Equal(t, "3.14", handler.Fields["peers.0.2"].GetF64()) - require.Equal(t, "4", handler.Fields["peers.1.0"].GetI64()) - require.Equal(t, "50.55", handler.Fields["peers.1.1"].GetF64()) - require.Equal(t, "6", handler.Fields["peers.1.2.0"].GetI64()) - require.Equal(t, "7", handler.Fields["peers.1.2.1"].GetI64()) + require.Equal(t, int64(1), handler.Fields["peers.0.0"].GetI64()) + require.Equal(t, int64(2), handler.Fields["peers.0.1"].GetI64()) + require.Equal(t, float64(3.14), handler.Fields["peers.0.2"].GetF64()) + require.Equal(t, int64(4), handler.Fields["peers.1.0"].GetI64()) + require.Equal(t, float64(50.55), handler.Fields["peers.1.1"].GetF64()) + require.Equal(t, int64(6), handler.Fields["peers.1.2.0"].GetI64()) + require.Equal(t, int64(7), handler.Fields["peers.1.2.1"].GetI64()) require.Equal(t, "hello", handler.Fields["peers.2.0"].GetStr()) require.Equal(t, "world", handler.Fields["peers.2.1"].GetStr()) require.Equal(t, "bar", handler.Fields["peers.3.foo"].GetStr()) From 2472a796e80cff1e0eb61517dccd3b44d19fd2de Mon Sep 17 00:00:00 2001 From: KevRiver Date: Mon, 16 Dec 2024 23:30:16 +0900 Subject: [PATCH 10/17] feat(json handler): add time format handling case --- json_handler.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/json_handler.go b/json_handler.go index 019e4ef..8b01f44 100644 --- a/json_handler.go +++ b/json_handler.go @@ -110,7 +110,11 @@ func getFlattenedFields(v map[string]interface{}) map[string]*typesv1.Val { } extValues[key] = typesv1.ValStr(valTyped.String()) case string: - extValues[key] = typesv1.ValStr(valTyped) + if t, ok := tryParseTime(valTyped); ok { + extValues[key] = typesv1.ValTime(t) + } else { + extValues[key] = typesv1.ValStr(valTyped) + } case bool: extValues[key] = typesv1.ValBool(valTyped) case []interface{}: @@ -143,7 +147,11 @@ func getFlattenedArrayFields(data []interface{}) map[string]*typesv1.Val { flattened[strconv.Itoa(i)] = typesv1.ValStr(vt.String()) } case string: - flattened[strconv.Itoa(i)] = typesv1.ValStr(vt) + if t, ok := tryParseTime(vt); ok { + flattened[strconv.Itoa(i)] = typesv1.ValTime(t) + } else { + flattened[strconv.Itoa(i)] = typesv1.ValStr(vt) + } case bool: flattened[strconv.Itoa(i)] = typesv1.ValBool(vt) case []interface{}: @@ -223,7 +231,11 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool { } h.Fields[key] = typesv1.ValStr(v.String()) case string: - h.Fields[key] = typesv1.ValStr(v) + if t, ok := tryParseTime(v); ok { + h.Fields[key] = typesv1.ValTime(t) + } else { + h.Fields[key] = typesv1.ValStr(v) + } case bool: h.Fields[key] = typesv1.ValBool(v) case []interface{}: From b9d3a6128185a892efbbbd2e3f964ac27d2a37c4 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Mon, 16 Dec 2024 23:31:22 +0900 Subject: [PATCH 11/17] test(sinkstdio): change not to wrap string values --- pkg/sink/stdiosink/stdio_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/sink/stdiosink/stdio_test.go b/pkg/sink/stdiosink/stdio_test.go index 242b112..59a2d2c 100644 --- a/pkg/sink/stdiosink/stdio_test.go +++ b/pkg/sink/stdiosink/stdio_test.go @@ -43,21 +43,21 @@ func TestPutKV(t *testing.T) { ), ))), want: map[string]string{ - "root.a": "\"lorem\"", + "root.a": "lorem", "root.b": "1", "root.c": "3.14", "root.d": "true", - "root.e.f": "\"foo\"", - "root.g.0": "\"bar\"", + "root.e.f": "foo", + "root.g.0": "bar", "root.g.1": "2", "root.g.2": "4.2", "root.g.3": "false", - "root.g.4.h": "\"baz\"", - "root.g.5.0": "\"qux\"", + "root.g.4.h": "baz", + "root.g.5.0": "qux", "root.g.5.1": "3", "root.g.5.2": "5.3", "root.g.5.3": "true", - "root.g.6": "\"2024-12-13T19:36:00Z\"", + "root.g.6": "2024-12-13T19:36:00Z", }, }, } From 7882867303e6504e5f7aa1b00b37b8baf4f856b7 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Mon, 16 Dec 2024 23:32:43 +0900 Subject: [PATCH 12/17] feat(sinkstdio): change not to wrap string values with quotations --- pkg/sink/stdiosink/stdio.go | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/pkg/sink/stdiosink/stdio.go b/pkg/sink/stdiosink/stdio.go index cfabdc5..e454ca7 100644 --- a/pkg/sink/stdiosink/stdio.go +++ b/pkg/sink/stdiosink/stdio.go @@ -230,15 +230,6 @@ func (std *Stdio) ReceiveWithPostProcess(ctx context.Context, ev *typesv1.LogEve kvs := make(map[string]string, len(data.Kvs)) for _, kv := range data.Kvs { key := kv.Key - // value could be one of the following: - // - string - // - int64 - // - float64 - // - bool - // - time.Time - // - time.Duration - // - []any - // - map[string]any value, err := logql.ResolveVal(kv.Value, logql.MakeFlatGoMap, logql.MakeFlatMapGoSlice) if err != nil { return err @@ -258,7 +249,7 @@ func toString(value *typesv1.Val) (string, error) { } switch t := v.(type) { case string: - return fmt.Sprintf("%q", t), nil + return t, nil case int64: return fmt.Sprintf("%d", t), nil case float64: @@ -266,9 +257,9 @@ func toString(value *typesv1.Val) (string, error) { case bool: return fmt.Sprintf("%t", t), nil case time.Time: - return fmt.Sprintf("%q", t.Format(time.RFC3339Nano)), nil + return t.Format(time.RFC3339Nano), nil case time.Duration: - return fmt.Sprintf("%q", t.String()), nil + return t.String(), nil default: return "", fmt.Errorf("unsupported type: %T", t) } @@ -277,7 +268,7 @@ func toString(value *typesv1.Val) (string, error) { func put(ref *map[string]string, key string, value any) { switch t := value.(type) { case string: - (*ref)[key] = fmt.Sprintf("%q", t) + (*ref)[key] = t case int64: (*ref)[key] = fmt.Sprintf("%d", t) case float64: @@ -285,9 +276,9 @@ func put(ref *map[string]string, key string, value any) { case bool: (*ref)[key] = fmt.Sprintf("%t", t) case time.Time: - (*ref)[key] = fmt.Sprintf("%q", t.Format(time.RFC3339Nano)) + (*ref)[key] = t.Format(time.RFC3339Nano) case time.Duration: - (*ref)[key] = fmt.Sprintf("%q", t.String()) + (*ref)[key] = t.String() case map[string]any: for k, v := range t { put(ref, key+"."+k, v) From 2a2db4bb755912eb80175fe517a42e9f8602817b Mon Sep 17 00:00:00 2001 From: KevRiver Date: Tue, 17 Dec 2024 15:56:54 +0900 Subject: [PATCH 13/17] feat(json handler): remove the function tryParseTime from string mapping case --- json_handler.go | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/json_handler.go b/json_handler.go index 8b01f44..4027791 100644 --- a/json_handler.go +++ b/json_handler.go @@ -110,11 +110,7 @@ func getFlattenedFields(v map[string]interface{}) map[string]*typesv1.Val { } extValues[key] = typesv1.ValStr(valTyped.String()) case string: - if t, ok := tryParseTime(valTyped); ok { - extValues[key] = typesv1.ValTime(t) - } else { - extValues[key] = typesv1.ValStr(valTyped) - } + extValues[key] = typesv1.ValStr(valTyped) case bool: extValues[key] = typesv1.ValBool(valTyped) case []interface{}: @@ -147,11 +143,7 @@ func getFlattenedArrayFields(data []interface{}) map[string]*typesv1.Val { flattened[strconv.Itoa(i)] = typesv1.ValStr(vt.String()) } case string: - if t, ok := tryParseTime(vt); ok { - flattened[strconv.Itoa(i)] = typesv1.ValTime(t) - } else { - flattened[strconv.Itoa(i)] = typesv1.ValStr(vt) - } + flattened[strconv.Itoa(i)] = typesv1.ValStr(vt) case bool: flattened[strconv.Itoa(i)] = typesv1.ValBool(vt) case []interface{}: @@ -231,11 +223,8 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool { } h.Fields[key] = typesv1.ValStr(v.String()) case string: - if t, ok := tryParseTime(v); ok { - h.Fields[key] = typesv1.ValTime(t) - } else { - h.Fields[key] = typesv1.ValStr(v) - } + + h.Fields[key] = typesv1.ValStr(v) case bool: h.Fields[key] = typesv1.ValBool(v) case []interface{}: From c67459e928908f0314071f2955b1a3ac3b5aa95c Mon Sep 17 00:00:00 2001 From: KevRiver Date: Tue, 17 Dec 2024 16:17:21 +0900 Subject: [PATCH 14/17] test!(e2e test): adjust expected result of the test cases(remove quotation marks from string values, expected truncated string and KV order) --- test/cases/00001-json/want | 18 +++++++++--------- test/cases/00004-mixed/want | 2 +- test/cases/10000-behavior-base/want | 6 +++--- test/cases/10001-behavior-truncates/want | 6 +++--- test/cases/20001-strip-docker-compose/want | 16 ++++++++-------- test/cases/20001-strip-syslog/want | 16 ++++++++-------- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/test/cases/00001-json/want b/test/cases/00001-json/want index 86a96db..1ed1b86 100644 --- a/test/cases/00001-json/want +++ b/test/cases/00001-json/want @@ -1,11 +1,11 @@ -Aug 11 18:14:50 || specversion="1.0" type="simple-log" invloglevel="Info" data.short="service-startup" id="01FCV6S4M6S8H3VKAQD9SWFWFP" datacontenttype="application/json" source="irn:libraries:github.com/InVisionApp/invlogger" data.message="The login-api service is running on port 8085." -Aug 11 18:14:55 || invweburl="" invwebbytes=0 invwebstatus=0 invwebbytesin=0 invwebbytesout=0 invwebduration=0 invweburipath="" invwebdesthost="" invweburiquery="" invweburllength=0 invwebcached=false invwebhttpmethod="" invwebhttpuseragent="" data.short="http access" invwebhttpcontenttype="" invwebhttpuseragentlength=0 invwebsrcip="::ffff:0.0.0.0" type="incoming_http_request" invwebdestip="::ffff:0.0.0.0" id="01FCV6S9YK70GJ5Q6YT0PYKQDA" invapptracingtequestsource="unset" invapptracingcallingservice="unset" data.message="incoming HTTP request was served" invapptracingrequestid="01FCV6S9YJXD2SYG3HTGWXHX0G" -Aug 11 18:14:59 || id="01FCV6SDKRW3XZDA1FAGZ3QVSH" invapptracingrequestid="01FCV6SDKRHB1RR1Q87Q1SKT5P" -Aug 11 18:15:00 || id="01FCV6SE597EY6RJ762V59PZQA" invapptracingrequestid="01FCV6SE596ZMASA1D79M16KVV" -Aug 11 18:15:00 || id="01FCV6SEKCC9RG364AJ60J75KW" invapptracingrequestid="01FCV6SEKCNSQJJ2NDEPQ2TGMP" -Aug 11 18:15:00 || id="01FCV6SF3DXGB8G1DVX19KQZYT" invapptracingrequestid="01FCV6SF3DJZSXTT1RNR6F1QAV" -Aug 11 18:15:05 || id="01FCV6SKY9MM7D795258XPQGC9" invapptracingrequestid="01FCV6SKY9M1D725HTV0ZXKF1V" -Aug 11 18:15:10 || data.event="Shutdown" type="service-shutdown" id="01FCV6SR6JZH7JZ6RFDFN9Q99Y" +Aug 11 18:14:50 || specversion=1.0 type=simple-log invloglevel=Info data.short=service-startup id=01FCV6S4M6S8H3VKAQD9SWFWFP datacontenttype=application/json source=irn:libraries:github.com/InVisionApp/invlogger data.message=The login-api service is running on port 8085. +Aug 11 18:14:55 || invweburl= invwebbytes=0 invwebstatus=0 invweburipath= invwebbytesin=0 invwebdesthost= invweburiquery= invwebbytesout=0 invwebduration=0 invwebhttpmethod= invweburllength=0 invwebcached=false invwebhttpuseragent= data.short=http access invwebhttpcontenttype= invwebsrcip=::ffff:0.0.0.0 type=incoming_http_request invwebdestip=::ffff:0.0.0.0 invwebhttpuseragentlength=0 id=01FCV6S9YK70GJ5Q6YT0PYKQDA invapptracingtequestsource=unset invapptracingcallingservice=unset data.message=incoming HTTP request was served invapptracingrequestid=01FCV6S9YJXD2SYG3HTGWXHX0G +Aug 11 18:14:59 || id=01FCV6SDKRW3XZDA1FAGZ3QVSH invapptracingrequestid=01FCV6SDKRHB1RR1Q87Q1SKT5P +Aug 11 18:15:00 || id=01FCV6SE597EY6RJ762V59PZQA invapptracingrequestid=01FCV6SE596ZMASA1D79M16KVV +Aug 11 18:15:00 || id=01FCV6SEKCC9RG364AJ60J75KW invapptracingrequestid=01FCV6SEKCNSQJJ2NDEPQ2TGMP +Aug 11 18:15:00 || id=01FCV6SF3DXGB8G1DVX19KQZYT invapptracingrequestid=01FCV6SF3DJZSXTT1RNR6F1QAV +Aug 11 18:15:05 || id=01FCV6SKY9MM7D795258XPQGC9 invapptracingrequestid=01FCV6SKY9M1D725HTV0ZXKF1V +Aug 11 18:15:10 || data.event=Shutdown type=service-shutdown id=01FCV6SR6JZH7JZ6RFDFN9Q99Y Aug 11 18:15:10 || hello Aug 11 18:15:10 || hello Aug 11 18:15:10 || hello @@ -23,4 +23,4 @@ Aug 11 18:15:10 || hello |ERRO| |PANI| |SOME| -Oct 29 07:45:54 |DEBU| first match found at index storage.i=0 source.line=243 storage.machine.id=5089 storage.session.id=1730187806608637000 source.function="github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch" source.file="/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go" +Oct 29 07:45:54 |DEBU| first match found at index storage.i=0 source.line=243 storage.machine.id=5089 storage.session.id=1730187806608637000 source.function=github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch source.file=/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go diff --git a/test/cases/00004-mixed/want b/test/cases/00004-mixed/want index 6badb96..c4872e3 100644 --- a/test/cases/00004-mixed/want +++ b/test/cases/00004-mixed/want @@ -1,4 +1,4 @@ Aug 10 03:03:24 |INFO| Started Worker Namespace=runtime TaskQueue=runtimed Aug 10 03:05:19 |ERRO| processing job error=worker error from receive: twirp error internal: failed to do request: Post "http://localhost:18081/twirp/aqueduct.api.v1.JobQueueService/Receive": context deadline exceeded (Client.Timeout exceeded while awaiting headers) Aug 10 03:06:20 |DEBU| inbound request twirp_svc=ManagementsAPI twirp_method=GetEnableStatus twirp_req=*managements.GetEnableStatusRequest -Aug 10 03:03:18 |INFO| temporal-sys-tq-scanner-workflow workflow successfully started service="worker" logging-call-at="scanner.go:202" +Aug 10 03:03:18 |INFO| temporal-sys-tq-scanner-workflow workflow successfully started service=worker logging-call-at=scanner.go:202 diff --git a/test/cases/10000-behavior-base/want b/test/cases/10000-behavior-base/want index 7b57a62..7b3e0a3 100644 --- a/test/cases/10000-behavior-base/want +++ b/test/cases/10000-behavior-base/want @@ -1,4 +1,4 @@ - || k1="short" k2="lonnnnnnnnnnnnnnnnnnng" - || repeated="first time" + || k1=short k2=lonnnnnnnnnnnnnnnnnnng + || repeated=first time || - || repeated="second time" + || repeated=second time diff --git a/test/cases/10001-behavior-truncates/want b/test/cases/10001-behavior-truncates/want index ef55986..23e4c98 100644 --- a/test/cases/10001-behavior-truncates/want +++ b/test/cases/10001-behavior-truncates/want @@ -1,4 +1,4 @@ - || k1="short" k2="lonnnnnnnnnnnn... - || repeated="first time" + || k1=short k2=lonnnnnnnnnnnnn... + || repeated=first time || - || repeated="second time" + || repeated=second time diff --git a/test/cases/20001-strip-docker-compose/want b/test/cases/20001-strip-docker-compose/want index f7e6ebb..43e857e 100644 --- a/test/cases/20001-strip-docker-compose/want +++ b/test/cases/20001-strip-docker-compose/want @@ -1,8 +1,8 @@ -Aug 11 18:14:50 || service=web_1 specversion="1.0" type="simple-log" invloglevel="Info" data.short="service-startup" id="01FCV6S4M6S8H3VKAQD9SWFWFP" datacontenttype="application/json" source="irn:libraries:github.com/InVisionApp/invlogger" data.message="The login-api service is running on port 8085." -Aug 11 18:14:55 || invweburl="" invwebbytes=0 invwebstatus=0 invwebbytesin=0 invwebbytesout=0 invwebduration=0 invweburipath="" invwebdesthost="" invweburiquery="" invweburllength=0 invwebcached=false invwebhttpmethod="" invwebhttpuseragent="" data.short="http access" invwebhttpcontenttype="" invwebhttpuseragentlength=0 invwebsrcip="::ffff:0.0.0.0" type="incoming_http_request" invwebdestip="::ffff:0.0.0.0" id="01FCV6S9YK70GJ5Q6YT0PYKQDA" invapptracingtequestsource="unset" invapptracingcallingservice="unset" data.message="incoming HTTP request was served" invapptracingrequestid="01FCV6S9YJXD2SYG3HTGWXHX0G" -Aug 11 18:14:59 || id="01FCV6SDKRW3XZDA1FAGZ3QVSH" invapptracingrequestid="01FCV6SDKRHB1RR1Q87Q1SKT5P" -Aug 11 18:15:00 || id="01FCV6SE597EY6RJ762V59PZQA" invapptracingrequestid="01FCV6SE596ZMASA1D79M16KVV" -Aug 11 18:15:00 || id="01FCV6SEKCC9RG364AJ60J75KW" invapptracingrequestid="01FCV6SEKCNSQJJ2NDEPQ2TGMP" -Aug 11 18:15:00 || id="01FCV6SF3DXGB8G1DVX19KQZYT" invapptracingrequestid="01FCV6SF3DJZSXTT1RNR6F1QAV" -Aug 11 18:15:05 || id="01FCV6SKY9MM7D795258XPQGC9" invapptracingrequestid="01FCV6SKY9M1D725HTV0ZXKF1V" -Aug 11 18:15:10 || data.event="Shutdown" type="service-shutdown" id="01FCV6SR6JZH7JZ6RFDFN9Q99Y" +Aug 11 18:14:50 || service=web_1 specversion=1.0 type=simple-log invloglevel=Info data.short=service-startup id=01FCV6S4M6S8H3VKAQD9SWFWFP datacontenttype=application/json source=irn:libraries:github.com/InVisionApp/invlogger data.message=The login-api service is running on port 8085. +Aug 11 18:14:55 || invweburl= invwebbytes=0 invwebstatus=0 invweburipath= invwebbytesin=0 invwebdesthost= invweburiquery= invwebbytesout=0 invwebduration=0 invwebhttpmethod= invweburllength=0 invwebcached=false invwebhttpuseragent= data.short=http access invwebhttpcontenttype= invwebsrcip=::ffff:0.0.0.0 type=incoming_http_request invwebdestip=::ffff:0.0.0.0 invwebhttpuseragentlength=0 id=01FCV6S9YK70GJ5Q6YT0PYKQDA invapptracingtequestsource=unset invapptracingcallingservice=unset data.message=incoming HTTP request was served invapptracingrequestid=01FCV6S9YJXD2SYG3HTGWXHX0G +Aug 11 18:14:59 || id=01FCV6SDKRW3XZDA1FAGZ3QVSH invapptracingrequestid=01FCV6SDKRHB1RR1Q87Q1SKT5P +Aug 11 18:15:00 || id=01FCV6SE597EY6RJ762V59PZQA invapptracingrequestid=01FCV6SE596ZMASA1D79M16KVV +Aug 11 18:15:00 || id=01FCV6SEKCC9RG364AJ60J75KW invapptracingrequestid=01FCV6SEKCNSQJJ2NDEPQ2TGMP +Aug 11 18:15:00 || id=01FCV6SF3DXGB8G1DVX19KQZYT invapptracingrequestid=01FCV6SF3DJZSXTT1RNR6F1QAV +Aug 11 18:15:05 || id=01FCV6SKY9MM7D795258XPQGC9 invapptracingrequestid=01FCV6SKY9M1D725HTV0ZXKF1V +Aug 11 18:15:10 || data.event=Shutdown type=service-shutdown id=01FCV6SR6JZH7JZ6RFDFN9Q99Y diff --git a/test/cases/20001-strip-syslog/want b/test/cases/20001-strip-syslog/want index eb7070e..7a174ee 100644 --- a/test/cases/20001-strip-syslog/want +++ b/test/cases/20001-strip-syslog/want @@ -1,8 +1,8 @@ -Aug 11 18:14:50 || specversion="1.0" type="simple-log" invloglevel="Info" data.short="service-startup" id="01FCV6S4M6S8H3VKAQD9SWFWFP" datacontenttype="application/json" source="irn:libraries:github.com/InVisionApp/invlogger" data.message="The login-api service is running on port 8085." -Aug 11 18:14:55 || invweburl="" invwebbytes=0 invwebstatus=0 invwebbytesin=0 invwebbytesout=0 invwebduration=0 invweburipath="" invwebdesthost="" invweburiquery="" invweburllength=0 invwebcached=false invwebhttpmethod="" invwebhttpuseragent="" data.short="http access" invwebhttpcontenttype="" invwebhttpuseragentlength=0 invwebsrcip="::ffff:0.0.0.0" type="incoming_http_request" invwebdestip="::ffff:0.0.0.0" id="01FCV6S9YK70GJ5Q6YT0PYKQDA" invapptracingtequestsource="unset" invapptracingcallingservice="unset" data.message="incoming HTTP request was served" invapptracingrequestid="01FCV6S9YJXD2SYG3HTGWXHX0G" -Aug 11 18:14:59 || id="01FCV6SDKRW3XZDA1FAGZ3QVSH" invapptracingrequestid="01FCV6SDKRHB1RR1Q87Q1SKT5P" -Aug 11 18:15:00 || id="01FCV6SE597EY6RJ762V59PZQA" invapptracingrequestid="01FCV6SE596ZMASA1D79M16KVV" -Aug 11 18:15:00 || id="01FCV6SEKCC9RG364AJ60J75KW" invapptracingrequestid="01FCV6SEKCNSQJJ2NDEPQ2TGMP" -Aug 11 18:15:00 || id="01FCV6SF3DXGB8G1DVX19KQZYT" invapptracingrequestid="01FCV6SF3DJZSXTT1RNR6F1QAV" -Aug 11 18:15:05 || id="01FCV6SKY9MM7D795258XPQGC9" invapptracingrequestid="01FCV6SKY9M1D725HTV0ZXKF1V" -Aug 11 18:15:10 || data.event="Shutdown" type="service-shutdown" id="01FCV6SR6JZH7JZ6RFDFN9Q99Y" +Aug 11 18:14:50 || specversion=1.0 type=simple-log invloglevel=Info data.short=service-startup id=01FCV6S4M6S8H3VKAQD9SWFWFP datacontenttype=application/json source=irn:libraries:github.com/InVisionApp/invlogger data.message=The login-api service is running on port 8085. +Aug 11 18:14:55 || invweburl= invwebbytes=0 invwebstatus=0 invweburipath= invwebbytesin=0 invwebdesthost= invweburiquery= invwebbytesout=0 invwebduration=0 invwebhttpmethod= invweburllength=0 invwebcached=false invwebhttpuseragent= data.short=http access invwebhttpcontenttype= invwebsrcip=::ffff:0.0.0.0 type=incoming_http_request invwebdestip=::ffff:0.0.0.0 invwebhttpuseragentlength=0 id=01FCV6S9YK70GJ5Q6YT0PYKQDA invapptracingtequestsource=unset invapptracingcallingservice=unset data.message=incoming HTTP request was served invapptracingrequestid=01FCV6S9YJXD2SYG3HTGWXHX0G +Aug 11 18:14:59 || id=01FCV6SDKRW3XZDA1FAGZ3QVSH invapptracingrequestid=01FCV6SDKRHB1RR1Q87Q1SKT5P +Aug 11 18:15:00 || id=01FCV6SE597EY6RJ762V59PZQA invapptracingrequestid=01FCV6SE596ZMASA1D79M16KVV +Aug 11 18:15:00 || id=01FCV6SEKCC9RG364AJ60J75KW invapptracingrequestid=01FCV6SEKCNSQJJ2NDEPQ2TGMP +Aug 11 18:15:00 || id=01FCV6SF3DXGB8G1DVX19KQZYT invapptracingrequestid=01FCV6SF3DJZSXTT1RNR6F1QAV +Aug 11 18:15:05 || id=01FCV6SKY9MM7D795258XPQGC9 invapptracingrequestid=01FCV6SKY9M1D725HTV0ZXKF1V +Aug 11 18:15:10 || data.event=Shutdown type=service-shutdown id=01FCV6SR6JZH7JZ6RFDFN9Q99Y From 6c430cf407b38886827157e01d458c69c11a36a7 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Tue, 17 Dec 2024 16:20:19 +0900 Subject: [PATCH 15/17] test(flattened nested objects): adjust expected value --- scanner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanner_test.go b/scanner_test.go index a4dbb8c..69c1d0d 100644 --- a/scanner_test.go +++ b/scanner_test.go @@ -163,7 +163,7 @@ func TestFlatteningNestedObjects_simple(t *testing.T) { Kvs: []*typesv1.KV{ { Key: "storage.from", - Value: typesv1.ValTime(time.Date(2024, 10, 29, 5, 47, 0, 0, time.UTC)), + Value: typesv1.ValStr(time.Date(2024, 10, 29, 5, 47, 0, 0, time.UTC).Format(time.RFC3339)), }, }, }, From 21b2f1f5f75ef7393ad7826b5817b98da4fba034 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Tue, 17 Dec 2024 16:23:28 +0900 Subject: [PATCH 16/17] test(zap dev prefix): fix funcFieldValue to use typev1.Val.GetStr --- zap_development_handler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zap_development_handler_test.go b/zap_development_handler_test.go index 35a5299..46810b2 100644 --- a/zap_development_handler_test.go +++ b/zap_development_handler_test.go @@ -229,7 +229,7 @@ func Test_tryZapDevPrefix(t *testing.T) { func findFieldValue(ev *typesv1.StructuredLogEvent, field string) string { for _, kv := range ev.Kvs { if kv.Key == field { - return kv.Value.String() + return kv.Value.GetStr() } } return "" From bfab6e240b68dedef8692ea63c1c36e9246a2706 Mon Sep 17 00:00:00 2001 From: KevRiver Date: Tue, 17 Dec 2024 17:09:04 +0900 Subject: [PATCH 17/17] test(scanner): fix time.Local to time.UTC --- scanner_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scanner_test.go b/scanner_test.go index 69c1d0d..35dd362 100644 --- a/scanner_test.go +++ b/scanner_test.go @@ -94,15 +94,15 @@ func TestLargePayload(t *testing.T) { func TestFlatteningNestedObjects_with_a_big_number(t *testing.T) { ctx := context.Background() - payload := `{"time":"2024-10-29T16:45:54.384776+09:00","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}` + payload := `{"time":"2024-10-29T16:45:54.384776Z","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}` now := time.Date(2024, 11, 26, 4, 0, 0, 0, time.UTC) want := []*typesv1.LogEvent{ { ParsedAt: timestamppb.New(now), - Raw: []byte(`{"time":"2024-10-29T16:45:54.384776+09:00","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}`), + Raw: []byte(`{"time":"2024-10-29T16:45:54.384776Z","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}`), Structured: &typesv1.StructuredLogEvent{ - Timestamp: timestamppb.New(time.Date(2024, 10, 29, 16, 45, 54, 384776000, time.Local)), + Timestamp: timestamppb.New(time.Date(2024, 10, 29, 16, 45, 54, 384776000, time.UTC)), Lvl: "DEBUG", Msg: "first match found at index", Kvs: []*typesv1.KV{