diff --git a/cmd/humanlog/query.go b/cmd/humanlog/query.go index 1b5fac70..8ccc4d27 100644 --- a/cmd/humanlog/query.go +++ b/cmd/humanlog/query.go @@ -17,6 +17,7 @@ import ( "github.com/charmbracelet/lipgloss" "github.com/charmbracelet/x/term" "github.com/crazy3lf/colorconv" + "github.com/humanlogio/api/go/pkg/logql" "github.com/humanlogio/api/go/svc/environment/v1/environmentv1connect" "github.com/humanlogio/api/go/svc/organization/v1/organizationv1connect" queryv1 "github.com/humanlogio/api/go/svc/query/v1" @@ -330,12 +331,16 @@ func queryApiWatchCmd( if state.CurrentEnvironmentID != nil { environmentID = *state.CurrentEnvironmentID } + lq, err := logql.ParseLogQuery(query) + if err != nil { + return fmt.Errorf("parsing query: %v", err) + } req := &queryv1.WatchQueryRequest{ EnvironmentId: environmentID, Query: &typesv1.LogQuery{ From: from, To: to, - Query: query, + Query: lq.Query, }, } res, err := queryClient.WatchQuery(ctx, connect.NewRequest(req)) diff --git a/docker_compose_handler.go b/docker_compose_handler.go index 46c2b3ab..c615d084 100644 --- a/docker_compose_handler.go +++ b/docker_compose_handler.go @@ -24,7 +24,7 @@ func tryDockerComposePrefix(d []byte, ev *typesv1.StructuredLogEvent, nextHandle if matches != nil { if nextHandler.TryHandle(matches[2], ev) { ev.Kvs = append(ev.Kvs, &typesv1.KV{ - Key: "service", Value: string(matches[1]), + Key: "service", Value: typesv1.ValStr(string(matches[1])), }) return true } @@ -34,7 +34,7 @@ func tryDockerComposePrefix(d []byte, ev *typesv1.StructuredLogEvent, nextHandle case *JSONHandler: if tryZapDevDCPrefix(matches[2], ev, h) { ev.Kvs = append(ev.Kvs, &typesv1.KV{ - Key: "service", Value: string(matches[1]), + Key: "service", Value: typesv1.ValStr(string(matches[1])), }) return true } diff --git a/go.mod b/go.mod index 8832123f..0adc065b 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/go-logfmt/logfmt v0.5.1 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 - github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd + github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72 github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 github.com/kr/logfmt v0.0.0-20210122060352-19f9bcb100e6 github.com/lrstanley/bubblezone v0.0.0-20240914071701-b48c55a5e78e diff --git a/go.sum b/go.sum index 80cce096..86c4218c 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd h1:449C6cnB4W6DblDMPfCfA4xyEkiYMpngGf7TEX9O8ro= -github.com/humanlogio/api/go v0.0.0-20241128170213-590d167300cd/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I= +github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 h1:lsiJGrN2E5qW6yvD6TKTAHa8lURmnh6KAuHvRfU8csU= +github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I= +github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72 h1:68dDinP4+R4eaEaVXOCiZEwypLk3aUEw4gnF7QL+oH0= +github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72/go.mod h1:pFt3YKuAVJk5nziOiKXTKyq5fj4aA9azq6xOx/932KQ= github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 h1:tdUCzFh8qvnWNCmxub0KSj1lIiCeWqvRjsMSSIApneE= github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9/go.mod h1:zq05mTZQXvKheFiAGlPx6+VSo29jw2ER8oy8DIQKW2Q= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= diff --git a/internal/localsvc/svc.go b/internal/localsvc/svc.go index 185dcad8..746f9a93 100644 --- a/internal/localsvc/svc.go +++ b/internal/localsvc/svc.go @@ -275,7 +275,7 @@ func (svc *Service) WatchQuery(ctx context.Context, req *connect.Request[qrv1.Wa query := req.Msg.GetQuery() ll := svc.ll.With( - slog.String("query.query", query.Query), + slog.String("query.query", query.Query.String()), ) if query.From != nil { ll = ll.With(slog.String("query.from", query.From.AsTime().Format(time.RFC3339Nano))) diff --git a/internal/memstorage/memory.go b/internal/memstorage/memory.go index e9f1729c..17f703c2 100644 --- a/internal/memstorage/memory.go +++ b/internal/memstorage/memory.go @@ -223,7 +223,7 @@ func newMemStorageSink(ll *slog.Logger, id SinkID) *MemStorageSink { func (snk *MemStorageSink) queryLogger(q *typesv1.LogQuery) *slog.Logger { ll := snk.ll.With( slog.Bool("sink.closed", snk.closed), - slog.String("query", q.Query), + slog.String("query", q.Query.String()), ) if q.From != nil { ll = ll.With(slog.Time("from", q.From.AsTime())) diff --git a/json_handler.go b/json_handler.go index 650ba424..4027791e 100644 --- a/json_handler.go +++ b/json_handler.go @@ -19,7 +19,7 @@ type JSONHandler struct { Level string Time time.Time Message string - Fields map[string]string + Fields map[string]*typesv1.Val } // searchJSON searches a document for a key using the found func to determine if the value is accepted. @@ -55,7 +55,7 @@ func (h *JSONHandler) clear() { h.Level = "" h.Time = time.Time{} h.Message = "" - h.Fields = make(map[string]string) + h.Fields = make(map[string]*typesv1.Val) } // TryHandle tells if this line was handled by this handler. @@ -95,22 +95,24 @@ func deleteJSONKey(key string, jsonData map[string]interface{}) { } } -func getFlattenedFields(v map[string]interface{}) map[string]string { - extValues := make(map[string]string) +func getFlattenedFields(v map[string]interface{}) map[string]*typesv1.Val { + extValues := make(map[string]*typesv1.Val) for key, nestedVal := range v { switch valTyped := nestedVal.(type) { case json.Number: if z, err := valTyped.Int64(); err == nil { - extValues[key] = fmt.Sprintf("%d", z) + extValues[key] = typesv1.ValI64(z) continue } if f, err := valTyped.Float64(); err == nil { - extValues[key] = fmt.Sprintf("%g", f) + extValues[key] = typesv1.ValF64(f) continue } - extValues[key] = valTyped.String() + extValues[key] = typesv1.ValStr(valTyped.String()) case string: - extValues[key] = fmt.Sprintf("%q", valTyped) + extValues[key] = typesv1.ValStr(valTyped) + case bool: + extValues[key] = typesv1.ValBool(valTyped) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(valTyped) for k, v := range flattenedArrayFields { @@ -122,26 +124,28 @@ func getFlattenedFields(v map[string]interface{}) map[string]string { extValues[key+"."+keyNested] = valStr } default: - extValues[key] = fmt.Sprintf("%v", valTyped) + extValues[key] = typesv1.ValStr(fmt.Sprintf("%v", valTyped)) } } return extValues } -func getFlattenedArrayFields(data []interface{}) map[string]string { - flattened := make(map[string]string) +func getFlattenedArrayFields(data []interface{}) map[string]*typesv1.Val { + flattened := make(map[string]*typesv1.Val) for i, v := range data { switch vt := v.(type) { case json.Number: if z, err := vt.Int64(); err == nil { - flattened[strconv.Itoa(i)] = fmt.Sprintf("%d", z) + flattened[strconv.Itoa(i)] = typesv1.ValI64(z) } else if f, err := vt.Float64(); err == nil { - flattened[strconv.Itoa(i)] = fmt.Sprintf("%g", f) + flattened[strconv.Itoa(i)] = typesv1.ValF64(f) } else { - flattened[strconv.Itoa(i)] = vt.String() + flattened[strconv.Itoa(i)] = typesv1.ValStr(vt.String()) } case string: - flattened[strconv.Itoa(i)] = vt + flattened[strconv.Itoa(i)] = typesv1.ValStr(vt) + case bool: + flattened[strconv.Itoa(i)] = typesv1.ValBool(vt) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(vt) for k, v := range flattenedArrayFields { @@ -153,7 +157,7 @@ func getFlattenedArrayFields(data []interface{}) map[string]string { flattened[fmt.Sprintf("%d.%s", i, k)] = v } default: - flattened[strconv.Itoa(i)] = fmt.Sprintf("%v", vt) + flattened[strconv.Itoa(i)] = typesv1.ValStr(fmt.Sprintf("%v", vt)) } } return flattened @@ -203,23 +207,26 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool { }) if h.Fields == nil { - h.Fields = make(map[string]string) + h.Fields = make(map[string]*typesv1.Val) } for key, val := range raw { switch v := val.(type) { case json.Number: if z, err := v.Int64(); err == nil { - h.Fields[key] = fmt.Sprintf("%d", z) + h.Fields[key] = typesv1.ValI64(z) continue } if f, err := v.Float64(); err == nil { - h.Fields[key] = fmt.Sprintf("%g", f) + h.Fields[key] = typesv1.ValF64(f) continue } - h.Fields[key] = v.String() + h.Fields[key] = typesv1.ValStr(v.String()) case string: - h.Fields[key] = fmt.Sprintf("%q", v) + + h.Fields[key] = typesv1.ValStr(v) + case bool: + h.Fields[key] = typesv1.ValBool(v) case []interface{}: flattenedArrayFields := getFlattenedArrayFields(v) for k, v := range flattenedArrayFields { @@ -231,7 +238,7 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool { h.Fields[key+"."+keyNested] = val } default: - h.Fields[key] = fmt.Sprintf("%v", v) + h.Fields[key] = typesv1.ValStr(fmt.Sprintf("%v", v)) } } diff --git a/json_handler_test.go b/json_handler_test.go index 2eea1f66..71d49050 100644 --- a/json_handler_test.go +++ b/json_handler_test.go @@ -1,4 +1,4 @@ -package humanlog_test +package humanlog import ( "fmt" @@ -7,7 +7,6 @@ import ( "github.com/google/go-cmp/cmp" typesv1 "github.com/humanlogio/api/go/types/v1" - "github.com/humanlogio/humanlog" "github.com/stretchr/testify/require" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" @@ -24,9 +23,9 @@ func TestJSONHandler_UnmarshalJSON_ParsesFields(t *testing.T) { raw := []byte(fmt.Sprintf(`{ "message": %q, "level": %q, "time": %q }`, msg, level, tm)) - opts := humanlog.DefaultOptions() + opts := DefaultOptions() - h := humanlog.JSONHandler{Opts: opts} + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(raw, ev) { t.Fatalf("failed to parse log level") @@ -56,12 +55,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomFields(t *testing.T) { raw := []byte(fmt.Sprintf(`{ "mymessage": %q, "mylevel": %q, "mytime": %q }`, msg, level, tm)) - opts := humanlog.DefaultOptions() + opts := DefaultOptions() opts.LevelFields = []string{"mylevel"} opts.MessageFields = []string{"mymessage"} opts.TimeFields = []string{"mytime"} - h := humanlog.JSONHandler{Opts: opts} + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(raw, ev) { @@ -91,12 +90,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomNestedFields(t *testing.T) { raw := []byte(fmt.Sprintf(`{ "data": { "message": %q, "level": %q, "time": %q }}`, msg, level, tm)) - opts := humanlog.DefaultOptions() + opts := DefaultOptions() opts.LevelFields = []string{"data.level"} opts.MessageFields = []string{"data.message"} opts.TimeFields = []string{"data.time"} - h := humanlog.JSONHandler{Opts: opts} + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(raw, ev) { t.Fatalf("failed to handle log") @@ -134,12 +133,12 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomMultiNestedFields(t *testing.T) { } }`, msg, level, tm)) - opts := humanlog.DefaultOptions() + opts := DefaultOptions() opts.LevelFields = []string{"data.l2.level"} opts.MessageFields = []string{"data.l2.message"} opts.TimeFields = []string{"data.l2.time"} - h := humanlog.JSONHandler{Opts: opts} + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(raw, ev) { t.Fatalf("failed to handle log") @@ -159,48 +158,48 @@ func TestJSONHandler_UnmarshalJSON_ParsesCustomMultiNestedFields(t *testing.T) { } func TestJsonHandler_TryHandle_LargeNumbers(t *testing.T) { - h := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()} + h := JSONHandler{Opts: DefaultOptions()} ev := new(typesv1.StructuredLogEvent) raw := []byte(`{"storage":{"session.id":1730187806608637000, "some": {"float": 1.2345}}}`) if !h.TryHandle(raw, ev) { t.Fatalf("failed to handle log") } - require.Equal(t, "1.2345", h.Fields["storage.some.float"]) - require.Equal(t, "1730187806608637000", h.Fields["storage.session.id"]) + require.Equal(t, 1.2345, h.Fields["storage.some.float"].GetF64()) + require.Equal(t, int64(1730187806608637000), h.Fields["storage.session.id"].GetI64()) } func TestJsonHandler_TryHandle_FlattendArrayFields(t *testing.T) { - handler := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()} + handler := JSONHandler{Opts: DefaultOptions()} ev := new(typesv1.StructuredLogEvent) raw := []byte(`{"peers":[{"ID":"10.244.0.126:8083","URI":"10.244.0.126:8083"},{"ID":"10.244.0.206:8083","URI":"10.244.0.206:8083"},{"ID":"10.244.1.150:8083","URI":"10.244.1.150:8083"}],"storage":{"session.id":1730187806608637000, "some": {"float": 1.2345}}}`) if !handler.TryHandle(raw, ev) { t.Fatalf("failed to handle log") } - require.Equal(t, "\"10.244.0.126:8083\"", handler.Fields["peers.0.ID"]) - require.Equal(t, "\"10.244.0.126:8083\"", handler.Fields["peers.0.URI"]) - require.Equal(t, "\"10.244.0.206:8083\"", handler.Fields["peers.1.ID"]) - require.Equal(t, "\"10.244.0.206:8083\"", handler.Fields["peers.1.URI"]) - require.Equal(t, "\"10.244.1.150:8083\"", handler.Fields["peers.2.ID"]) - require.Equal(t, "\"10.244.1.150:8083\"", handler.Fields["peers.2.URI"]) + require.Equal(t, "10.244.0.126:8083", handler.Fields["peers.0.ID"].GetStr()) + require.Equal(t, "10.244.0.126:8083", handler.Fields["peers.0.URI"].GetStr()) + require.Equal(t, "10.244.0.206:8083", handler.Fields["peers.1.ID"].GetStr()) + require.Equal(t, "10.244.0.206:8083", handler.Fields["peers.1.URI"].GetStr()) + require.Equal(t, "10.244.1.150:8083", handler.Fields["peers.2.ID"].GetStr()) + require.Equal(t, "10.244.1.150:8083", handler.Fields["peers.2.URI"].GetStr()) } func TestJsonHandler_TryHandle_FlattenedArrayFields_NestedArray(t *testing.T) { - handler := humanlog.JSONHandler{Opts: humanlog.DefaultOptions()} + handler := JSONHandler{Opts: DefaultOptions()} ev := new(typesv1.StructuredLogEvent) raw := []byte(`{"peers":[[1,2,3.14],[4,50.55,[6,7]],["hello","world"],{"foo":"bar"}]}`) if !handler.TryHandle(raw, ev) { t.Fatalf("failed to handle log") } - require.Equal(t, "1", handler.Fields["peers.0.0"]) - require.Equal(t, "2", handler.Fields["peers.0.1"]) - require.Equal(t, "3.14", handler.Fields["peers.0.2"]) - require.Equal(t, "4", handler.Fields["peers.1.0"]) - require.Equal(t, "50.55", handler.Fields["peers.1.1"]) - require.Equal(t, "6", handler.Fields["peers.1.2.0"]) - require.Equal(t, "7", handler.Fields["peers.1.2.1"]) - require.Equal(t, "hello", handler.Fields["peers.2.0"]) - require.Equal(t, "world", handler.Fields["peers.2.1"]) - require.Equal(t, "\"bar\"", handler.Fields["peers.3.foo"]) + require.Equal(t, int64(1), handler.Fields["peers.0.0"].GetI64()) + require.Equal(t, int64(2), handler.Fields["peers.0.1"].GetI64()) + require.Equal(t, float64(3.14), handler.Fields["peers.0.2"].GetF64()) + require.Equal(t, int64(4), handler.Fields["peers.1.0"].GetI64()) + require.Equal(t, float64(50.55), handler.Fields["peers.1.1"].GetF64()) + require.Equal(t, int64(6), handler.Fields["peers.1.2.0"].GetI64()) + require.Equal(t, int64(7), handler.Fields["peers.1.2.1"].GetI64()) + require.Equal(t, "hello", handler.Fields["peers.2.0"].GetStr()) + require.Equal(t, "world", handler.Fields["peers.2.1"].GetStr()) + require.Equal(t, "bar", handler.Fields["peers.3.foo"].GetStr()) } func TestParseAsctimeFields(t *testing.T) { @@ -223,8 +222,8 @@ func TestParseAsctimeFields(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - opts := humanlog.DefaultOptions() - h := humanlog.JSONHandler{Opts: opts} + opts := DefaultOptions() + h := JSONHandler{Opts: opts} ev := new(typesv1.StructuredLogEvent) if !h.TryHandle(test.raw, ev) { t.Fatalf("failed to handle log") diff --git a/logfmt_handler.go b/logfmt_handler.go index f3a9647c..75f8efe1 100644 --- a/logfmt_handler.go +++ b/logfmt_handler.go @@ -16,14 +16,14 @@ type LogfmtHandler struct { Level string Time time.Time Message string - Fields map[string]string + Fields map[string]*typesv1.Val } func (h *LogfmtHandler) clear() { h.Level = "" h.Time = time.Time{} h.Message = "" - h.Fields = make(map[string]string) + h.Fields = make(map[string]*typesv1.Val) } // CanHandle tells if this line can be handled by this handler. @@ -47,7 +47,7 @@ func (h *LogfmtHandler) TryHandle(d []byte, out *typesv1.StructuredLogEvent) boo // HandleLogfmt sets the fields of the handler. func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) bool { if h.Fields == nil { - h.Fields = make(map[string]string) + h.Fields = make(map[string]*typesv1.Val) } dec := logfmt.NewDecoder(bytes.NewReader(data)) for dec.ScanRecord() { @@ -97,7 +97,7 @@ func (h *LogfmtHandler) UnmarshalLogfmt(data []byte) bool { } } - h.Fields[string(key)] = string(val) + h.Fields[string(key)] = typesv1.ValStr(string(val)) } } return dec.Err() == nil diff --git a/pkg/localstorage/test_suite.go b/pkg/localstorage/test_suite.go index 888907b2..62eaf9f8 100644 --- a/pkg/localstorage/test_suite.go +++ b/pkg/localstorage/test_suite.go @@ -5,9 +5,10 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" typesv1 "github.com/humanlogio/api/go/types/v1" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -213,7 +214,8 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) { require.Len(t, got, len(tt.want)) for i := range tt.want { - require.Equal(t, protojson.Format(tt.want[i]), protojson.Format(got[i])) + diff := cmp.Diff(tt.want[i], got[i], protocmp.Transform()) + require.Empty(t, diff) } }) } diff --git a/pkg/sink/stdiosink/stdio.go b/pkg/sink/stdiosink/stdio.go index f5d6e766..e454ca7e 100644 --- a/pkg/sink/stdiosink/stdio.go +++ b/pkg/sink/stdiosink/stdio.go @@ -11,6 +11,7 @@ import ( "time" "github.com/fatih/color" + "github.com/humanlogio/api/go/pkg/logql" typesv1 "github.com/humanlogio/api/go/types/v1" "github.com/humanlogio/humanlog/internal/pkg/config" "github.com/humanlogio/humanlog/pkg/sink" @@ -228,7 +229,12 @@ func (std *Stdio) ReceiveWithPostProcess(ctx context.Context, ev *typesv1.LogEve kvs := make(map[string]string, len(data.Kvs)) for _, kv := range data.Kvs { - kvs[kv.Key] = kv.Value + key := kv.Key + value, err := logql.ResolveVal(kv.Value, logql.MakeFlatGoMap, logql.MakeFlatMapGoSlice) + if err != nil { + return err + } + put(&kvs, key, value) } std.lastRaw = false std.lastLevel = ev.Structured.Lvl @@ -236,6 +242,52 @@ func (std *Stdio) ReceiveWithPostProcess(ctx context.Context, ev *typesv1.LogEve return nil } +func toString(value *typesv1.Val) (string, error) { + v, err := logql.ResolveVal(value, nil, nil) + if err != nil { + return "", err + } + switch t := v.(type) { + case string: + return t, nil + case int64: + return fmt.Sprintf("%d", t), nil + case float64: + return fmt.Sprintf("%g", t), nil + case bool: + return fmt.Sprintf("%t", t), nil + case time.Time: + return t.Format(time.RFC3339Nano), nil + case time.Duration: + return t.String(), nil + default: + return "", fmt.Errorf("unsupported type: %T", t) + } +} + +func put(ref *map[string]string, key string, value any) { + switch t := value.(type) { + case string: + (*ref)[key] = t + case int64: + (*ref)[key] = fmt.Sprintf("%d", t) + case float64: + (*ref)[key] = fmt.Sprintf("%g", t) + case bool: + (*ref)[key] = fmt.Sprintf("%t", t) + case time.Time: + (*ref)[key] = t.Format(time.RFC3339Nano) + case time.Duration: + (*ref)[key] = t.String() + case map[string]any: + for k, v := range t { + put(ref, key+"."+k, v) + } + default: + (*ref)[key] = fmt.Sprintf("%v", t) + } +} + func (std *Stdio) joinKVs(data *typesv1.StructuredLogEvent, sep string) []string { wasSameLevel := std.lastLevel == data.Lvl skipUnchanged := !std.lastRaw && std.opts.SkipUnchanged && wasSameLevel @@ -246,19 +298,23 @@ func (std *Stdio) joinKVs(data *typesv1.StructuredLogEvent, sep string) []string if !std.opts.shouldShowKey(k) { continue } + w, err := toString(v) + if err != nil { + continue + } if skipUnchanged { - if lastV, ok := std.lastKVs[k]; ok && lastV == v && !std.opts.shouldShowUnchanged(k) { + if lastV, ok := std.lastKVs[k]; ok && lastV == w && !std.opts.shouldShowUnchanged(k) { continue } } kstr := std.opts.Palette.KeyColor.Sprint(k) var vstr string - if std.opts.Truncates && len(v) > std.opts.TruncateLength { - vstr = v[:std.opts.TruncateLength] + "..." + if std.opts.Truncates && len(w) > std.opts.TruncateLength { + vstr = w[:std.opts.TruncateLength] + "..." } else { - vstr = v + vstr = w } vstr = std.opts.Palette.ValColor.Sprint(vstr) kv = append(kv, kstr+sep+vstr) diff --git a/pkg/sink/stdiosink/stdio_test.go b/pkg/sink/stdiosink/stdio_test.go new file mode 100644 index 00000000..59a2d2c3 --- /dev/null +++ b/pkg/sink/stdiosink/stdio_test.go @@ -0,0 +1,78 @@ +package stdiosink + +import ( + "testing" + "time" + + "github.com/humanlogio/api/go/pkg/logql" + typesv1 "github.com/humanlogio/api/go/types/v1" + "github.com/stretchr/testify/require" +) + +func TestPutKV(t *testing.T) { + tests := []struct { + name string + args *typesv1.KV + want map[string]string + }{ + { + name: "convert into map[string]string", + args: typesv1.KeyVal("root", typesv1.ValObj( + typesv1.KeyVal("a", typesv1.ValStr("lorem")), + typesv1.KeyVal("b", typesv1.ValI64(int64(1))), + typesv1.KeyVal("c", typesv1.ValF64(float64(3.14))), + typesv1.KeyVal("d", typesv1.ValBool(true)), + typesv1.KeyVal("e", typesv1.ValObj( + typesv1.KeyVal("f", typesv1.ValStr("foo")), + )), + typesv1.KeyVal("g", typesv1.ValArr( + typesv1.ValStr("bar"), + typesv1.ValI64(int64(2)), + typesv1.ValF64(float64(4.2)), + typesv1.ValBool(false), + typesv1.ValObj( + typesv1.KeyVal("h", typesv1.ValStr("baz")), + ), + typesv1.ValArr( + typesv1.ValStr("qux"), + typesv1.ValI64(int64(3)), + typesv1.ValF64(float64(5.3)), + typesv1.ValBool(true), + ), + typesv1.ValTime(time.Date(2024, 12, 13, 19, 36, 0, 0, time.UTC)), + ), + ))), + want: map[string]string{ + "root.a": "lorem", + "root.b": "1", + "root.c": "3.14", + "root.d": "true", + "root.e.f": "foo", + "root.g.0": "bar", + "root.g.1": "2", + "root.g.2": "4.2", + "root.g.3": "false", + "root.g.4.h": "baz", + "root.g.5.0": "qux", + "root.g.5.1": "3", + "root.g.5.2": "5.3", + "root.g.5.3": "true", + "root.g.6": "2024-12-13T19:36:00Z", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + key := tt.args.Key + value, err := logql.ResolveVal(tt.args.Value, logql.MakeFlatGoMap, logql.MakeFlatMapGoSlice) + require.NoError(t, err) + + kvs := make(map[string]string) + put(&kvs, key, value) + + got := kvs + require.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/tui/components/querybar/querybar.go b/pkg/tui/components/querybar/querybar.go index c05e0f97..d4f5fb25 100644 --- a/pkg/tui/components/querybar/querybar.go +++ b/pkg/tui/components/querybar/querybar.go @@ -4,7 +4,7 @@ import ( "github.com/charmbracelet/bubbles/key" "github.com/charmbracelet/bubbles/textarea" tea "github.com/charmbracelet/bubbletea" - "github.com/humanlogio/api/go/pkg/lang" + "github.com/humanlogio/api/go/pkg/logql" typesv1 "github.com/humanlogio/api/go/types/v1" ) @@ -53,7 +53,7 @@ func (m *QueryBar) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case tea.KeyMsg: if key.Matches(msg, m.submitQuery) { q := m.textArea.Value() - qq, err := lang.ParseLogQuery(q) + qq, err := logql.ParseLogQuery(q) if err != nil { m.problems = append(m.problems, err.Error()) } else { diff --git a/scanner_test.go b/scanner_test.go index 692c453e..35dd362f 100644 --- a/scanner_test.go +++ b/scanner_test.go @@ -6,11 +6,13 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" typesv1 "github.com/humanlogio/api/go/types/v1" "github.com/humanlogio/humanlog/pkg/sink/bufsink" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -92,42 +94,24 @@ func TestLargePayload(t *testing.T) { func TestFlatteningNestedObjects_with_a_big_number(t *testing.T) { ctx := context.Background() - payload := `{"time":"2024-10-29T16:45:54.384776+09:00","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}` + payload := `{"time":"2024-10-29T16:45:54.384776Z","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}` now := time.Date(2024, 11, 26, 4, 0, 0, 0, time.UTC) want := []*typesv1.LogEvent{ { ParsedAt: timestamppb.New(now), - Raw: []byte(`{"time":"2024-10-29T16:45:54.384776+09:00","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}`), + Raw: []byte(`{"time":"2024-10-29T16:45:54.384776Z","level":"DEBUG","source":{"function":"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch","file":"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go","line":243},"msg":"first match found at index","storage":{"machine.id":5089,"session.id":1730187806608637000,"i":0}}`), Structured: &typesv1.StructuredLogEvent{ - Timestamp: timestamppb.New(time.Date(2024, 10, 29, 16, 14, 54, 384776000, time.Local)), + Timestamp: timestamppb.New(time.Date(2024, 10, 29, 16, 45, 54, 384776000, time.UTC)), Lvl: "DEBUG", Msg: "first match found at index", Kvs: []*typesv1.KV{ - { - Key: "source.function", - Value: "\"github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch\"", - }, - { - Key: "source.file", - Value: "\"/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go\"", - }, - { - Key: "source.line", - Value: "243", - }, - { - Key: "storage.machine.id", - Value: "5089", - }, - { - Key: "storage.session.id", - Value: "1730187806608637000", - }, - { - Key: "storage.i", - Value: "0", - }, + typesv1.KeyVal("source.function", typesv1.ValStr("github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch")), + typesv1.KeyVal("source.file", typesv1.ValStr("/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go")), + typesv1.KeyVal("source.line", typesv1.ValI64(243)), + typesv1.KeyVal("storage.machine.id", typesv1.ValI64(5089)), + typesv1.KeyVal("storage.session.id", typesv1.ValI64(1730187806608637000)), + typesv1.KeyVal("storage.i", typesv1.ValI64(0)), }, }, }, @@ -143,15 +127,24 @@ func TestFlatteningNestedObjects_with_a_big_number(t *testing.T) { err := Scan(ctx, src, sink, opts) require.NoError(t, err) - for i, got := range sink.Buffered { - expectedKvs := make(map[string]string) + got := sink.Buffered + require.Equal(t, len(want), len(got)) // assume that there's no skipped log events + + n := len(want) + for i := 0; i < n; i++ { + actualKvs := make(map[string]*typesv1.Val) + for _, kv := range got[i].Structured.Kvs { + actualKvs[kv.Key] = kv.Value + } + expectedKvs := make(map[string]*typesv1.Val) for _, kv := range want[i].Structured.Kvs { expectedKvs[kv.Key] = kv.Value } - actualKvs := make(map[string]string) - for _, kv := range got.Structured.Kvs { - actualKvs[kv.Key] = kv.Value - } + require.Equal(t, got[i].ParsedAt, want[i].ParsedAt) + require.Equal(t, got[i].Raw, want[i].Raw) + require.Equal(t, got[i].Structured.Timestamp, want[i].Structured.Timestamp) + require.Equal(t, got[i].Structured.Msg, want[i].Structured.Msg) + require.Equal(t, got[i].Structured.Lvl, want[i].Structured.Lvl) require.Equal(t, expectedKvs, actualKvs) } } @@ -170,7 +163,7 @@ func TestFlatteningNestedObjects_simple(t *testing.T) { Kvs: []*typesv1.KV{ { Key: "storage.from", - Value: "\"" + time.Date(2024, 10, 29, 5, 47, 0, 0, time.UTC).Format(time.RFC3339) + "\"", + Value: typesv1.ValStr(time.Date(2024, 10, 29, 5, 47, 0, 0, time.UTC).Format(time.RFC3339)), }, }, }, @@ -190,17 +183,9 @@ func TestFlatteningNestedObjects_simple(t *testing.T) { got := sink.Buffered require.Equal(t, len(want), len(got)) // assume that there's no skipped log events - n := len(want) - for i := 0; i < n; i++ { - actual := make(map[string]string) - for _, kv := range got[i].Structured.Kvs { - actual[kv.Key] = kv.Value - } - expected := make(map[string]string) - for _, kv := range want[i].Structured.Kvs { - expected[kv.Key] = kv.Value - } - require.Equal(t, expected, actual) + for i, got := range sink.Buffered { + diff := cmp.Diff(want[i], got, protocmp.Transform()) + require.Empty(t, diff) } } @@ -219,43 +204,43 @@ func TestFlatteningNestedObjects_with_arrays(t *testing.T) { Kvs: []*typesv1.KV{ { Key: "selfURI", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "source.function", - Value: "\"main.realMain.func5.1\"", + Value: typesv1.ValStr("main.realMain.func5.1"), }, { Key: "source.file", - Value: "\"github.com/humanlogio/apisvc/cmd/apisvc/server_cmd.go\"", + Value: typesv1.ValStr("github.com/humanlogio/apisvc/cmd/apisvc/server_cmd.go"), }, { Key: "source.line", - Value: "407", + Value: typesv1.ValI64(407), }, { Key: "peers.0.ID", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "peers.0.URI", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "peers.1.ID", - Value: "\"10.244.0.206:8083\"", + Value: typesv1.ValStr("10.244.0.206:8083"), }, { Key: "peers.1.URI", - Value: "\"10.244.0.206:8083\"", + Value: typesv1.ValStr("10.244.0.206:8083"), }, { Key: "peers.2.ID", - Value: "\"10.244.1.150:8083\"", + Value: typesv1.ValStr("10.244.1.150:8083"), }, { Key: "peers.2.URI", - Value: "\"10.244.1.150:8083\"", + Value: typesv1.ValStr("10.244.1.150:8083"), }, }, }, @@ -278,11 +263,11 @@ func TestFlatteningNestedObjects_with_arrays(t *testing.T) { n := len(want) for i := 0; i < n; i++ { - actualKvs := make(map[string]string) + actualKvs := make(map[string]*typesv1.Val) for _, kv := range got[i].Structured.Kvs { actualKvs[kv.Key] = kv.Value } - expectedKvs := make(map[string]string) + expectedKvs := make(map[string]*typesv1.Val) for _, kv := range want[i].Structured.Kvs { expectedKvs[kv.Key] = kv.Value } @@ -310,51 +295,51 @@ func TestFlatteningNestedObjects_with_nested_arrays(t *testing.T) { Kvs: []*typesv1.KV{ { Key: "peers.0.0", - Value: "1", + Value: typesv1.ValI64(1), }, { Key: "peers.0.1", - Value: "2", + Value: typesv1.ValI64(2), }, { Key: "peers.0.2", - Value: "3", + Value: typesv1.ValI64(3), }, { Key: "peers.1.0", - Value: "4", + Value: typesv1.ValI64(4), }, { Key: "peers.1.1", - Value: "5", + Value: typesv1.ValI64(5), }, { Key: "peers.1.2", - Value: "6", + Value: typesv1.ValI64(6), }, { Key: "peers.2.0.ID", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "peers.2.0.URI", - Value: "\"10.244.0.126:8083\"", + Value: typesv1.ValStr("10.244.0.126:8083"), }, { Key: "peers.2.1.ID", - Value: "\"10.244.0.206:8083\"", + Value: typesv1.ValStr("10.244.0.206:8083"), }, { Key: "peers.2.1.URI", - Value: "\"10.244.0.206:8083\"", + Value: typesv1.ValStr("10.244.0.206:8083"), }, { Key: "peers.2.2.ID", - Value: "\"10.244.1.150:8083\"", + Value: typesv1.ValStr("10.244.1.150:8083"), }, { Key: "peers.2.2.URI", - Value: "\"10.244.1.150:8083\"", + Value: typesv1.ValStr("10.244.1.150:8083"), }, }, }, @@ -377,11 +362,11 @@ func TestFlatteningNestedObjects_with_nested_arrays(t *testing.T) { n := len(want) for i := 0; i < n; i++ { - actualKvs := make(map[string]string) + actualKvs := make(map[string]*typesv1.Val) for _, kv := range got[i].Structured.Kvs { actualKvs[kv.Key] = kv.Value } - expectedKvs := make(map[string]string) + expectedKvs := make(map[string]*typesv1.Val) for _, kv := range want[i].Structured.Kvs { expectedKvs[kv.Key] = kv.Value } diff --git a/test/cases/00001-json/want b/test/cases/00001-json/want index 86a96db3..1ed1b867 100644 --- a/test/cases/00001-json/want +++ b/test/cases/00001-json/want @@ -1,11 +1,11 @@ -Aug 11 18:14:50 || specversion="1.0" type="simple-log" invloglevel="Info" data.short="service-startup" id="01FCV6S4M6S8H3VKAQD9SWFWFP" datacontenttype="application/json" source="irn:libraries:github.com/InVisionApp/invlogger" data.message="The login-api service is running on port 8085." -Aug 11 18:14:55 || invweburl="" invwebbytes=0 invwebstatus=0 invwebbytesin=0 invwebbytesout=0 invwebduration=0 invweburipath="" invwebdesthost="" invweburiquery="" invweburllength=0 invwebcached=false invwebhttpmethod="" invwebhttpuseragent="" data.short="http access" invwebhttpcontenttype="" invwebhttpuseragentlength=0 invwebsrcip="::ffff:0.0.0.0" type="incoming_http_request" invwebdestip="::ffff:0.0.0.0" id="01FCV6S9YK70GJ5Q6YT0PYKQDA" invapptracingtequestsource="unset" invapptracingcallingservice="unset" data.message="incoming HTTP request was served" invapptracingrequestid="01FCV6S9YJXD2SYG3HTGWXHX0G" -Aug 11 18:14:59 || id="01FCV6SDKRW3XZDA1FAGZ3QVSH" invapptracingrequestid="01FCV6SDKRHB1RR1Q87Q1SKT5P" -Aug 11 18:15:00 || id="01FCV6SE597EY6RJ762V59PZQA" invapptracingrequestid="01FCV6SE596ZMASA1D79M16KVV" -Aug 11 18:15:00 || id="01FCV6SEKCC9RG364AJ60J75KW" invapptracingrequestid="01FCV6SEKCNSQJJ2NDEPQ2TGMP" -Aug 11 18:15:00 || id="01FCV6SF3DXGB8G1DVX19KQZYT" invapptracingrequestid="01FCV6SF3DJZSXTT1RNR6F1QAV" -Aug 11 18:15:05 || id="01FCV6SKY9MM7D795258XPQGC9" invapptracingrequestid="01FCV6SKY9M1D725HTV0ZXKF1V" -Aug 11 18:15:10 || data.event="Shutdown" type="service-shutdown" id="01FCV6SR6JZH7JZ6RFDFN9Q99Y" +Aug 11 18:14:50 || specversion=1.0 type=simple-log invloglevel=Info data.short=service-startup id=01FCV6S4M6S8H3VKAQD9SWFWFP datacontenttype=application/json source=irn:libraries:github.com/InVisionApp/invlogger data.message=The login-api service is running on port 8085. +Aug 11 18:14:55 || invweburl= invwebbytes=0 invwebstatus=0 invweburipath= invwebbytesin=0 invwebdesthost= invweburiquery= invwebbytesout=0 invwebduration=0 invwebhttpmethod= invweburllength=0 invwebcached=false invwebhttpuseragent= data.short=http access invwebhttpcontenttype= invwebsrcip=::ffff:0.0.0.0 type=incoming_http_request invwebdestip=::ffff:0.0.0.0 invwebhttpuseragentlength=0 id=01FCV6S9YK70GJ5Q6YT0PYKQDA invapptracingtequestsource=unset invapptracingcallingservice=unset data.message=incoming HTTP request was served invapptracingrequestid=01FCV6S9YJXD2SYG3HTGWXHX0G +Aug 11 18:14:59 || id=01FCV6SDKRW3XZDA1FAGZ3QVSH invapptracingrequestid=01FCV6SDKRHB1RR1Q87Q1SKT5P +Aug 11 18:15:00 || id=01FCV6SE597EY6RJ762V59PZQA invapptracingrequestid=01FCV6SE596ZMASA1D79M16KVV +Aug 11 18:15:00 || id=01FCV6SEKCC9RG364AJ60J75KW invapptracingrequestid=01FCV6SEKCNSQJJ2NDEPQ2TGMP +Aug 11 18:15:00 || id=01FCV6SF3DXGB8G1DVX19KQZYT invapptracingrequestid=01FCV6SF3DJZSXTT1RNR6F1QAV +Aug 11 18:15:05 || id=01FCV6SKY9MM7D795258XPQGC9 invapptracingrequestid=01FCV6SKY9M1D725HTV0ZXKF1V +Aug 11 18:15:10 || data.event=Shutdown type=service-shutdown id=01FCV6SR6JZH7JZ6RFDFN9Q99Y Aug 11 18:15:10 || hello Aug 11 18:15:10 || hello Aug 11 18:15:10 || hello @@ -23,4 +23,4 @@ Aug 11 18:15:10 || hello |ERRO| |PANI| |SOME| -Oct 29 07:45:54 |DEBU| first match found at index storage.i=0 source.line=243 storage.machine.id=5089 storage.session.id=1730187806608637000 source.function="github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch" source.file="/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go" +Oct 29 07:45:54 |DEBU| first match found at index storage.i=0 source.line=243 storage.machine.id=5089 storage.session.id=1730187806608637000 source.function=github.com/humanlogio/humanlog/internal/memstorage.(*MemStorageSink).firstMatch source.file=/Users/antoine/code/src/github.com/humanlogio/humanlog/internal/memstorage/memory.go diff --git a/test/cases/00004-mixed/want b/test/cases/00004-mixed/want index 6badb969..c4872e33 100644 --- a/test/cases/00004-mixed/want +++ b/test/cases/00004-mixed/want @@ -1,4 +1,4 @@ Aug 10 03:03:24 |INFO| Started Worker Namespace=runtime TaskQueue=runtimed Aug 10 03:05:19 |ERRO| processing job error=worker error from receive: twirp error internal: failed to do request: Post "http://localhost:18081/twirp/aqueduct.api.v1.JobQueueService/Receive": context deadline exceeded (Client.Timeout exceeded while awaiting headers) Aug 10 03:06:20 |DEBU| inbound request twirp_svc=ManagementsAPI twirp_method=GetEnableStatus twirp_req=*managements.GetEnableStatusRequest -Aug 10 03:03:18 |INFO| temporal-sys-tq-scanner-workflow workflow successfully started service="worker" logging-call-at="scanner.go:202" +Aug 10 03:03:18 |INFO| temporal-sys-tq-scanner-workflow workflow successfully started service=worker logging-call-at=scanner.go:202 diff --git a/test/cases/10000-behavior-base/want b/test/cases/10000-behavior-base/want index 7b57a62b..7b3e0a38 100644 --- a/test/cases/10000-behavior-base/want +++ b/test/cases/10000-behavior-base/want @@ -1,4 +1,4 @@ - || k1="short" k2="lonnnnnnnnnnnnnnnnnnng" - || repeated="first time" + || k1=short k2=lonnnnnnnnnnnnnnnnnnng + || repeated=first time || - || repeated="second time" + || repeated=second time diff --git a/test/cases/10001-behavior-truncates/want b/test/cases/10001-behavior-truncates/want index ef559864..23e4c98d 100644 --- a/test/cases/10001-behavior-truncates/want +++ b/test/cases/10001-behavior-truncates/want @@ -1,4 +1,4 @@ - || k1="short" k2="lonnnnnnnnnnnn... - || repeated="first time" + || k1=short k2=lonnnnnnnnnnnnn... + || repeated=first time || - || repeated="second time" + || repeated=second time diff --git a/test/cases/20001-strip-docker-compose/want b/test/cases/20001-strip-docker-compose/want index f7e6ebb0..43e857e4 100644 --- a/test/cases/20001-strip-docker-compose/want +++ b/test/cases/20001-strip-docker-compose/want @@ -1,8 +1,8 @@ -Aug 11 18:14:50 || service=web_1 specversion="1.0" type="simple-log" invloglevel="Info" data.short="service-startup" id="01FCV6S4M6S8H3VKAQD9SWFWFP" datacontenttype="application/json" source="irn:libraries:github.com/InVisionApp/invlogger" data.message="The login-api service is running on port 8085." -Aug 11 18:14:55 || invweburl="" invwebbytes=0 invwebstatus=0 invwebbytesin=0 invwebbytesout=0 invwebduration=0 invweburipath="" invwebdesthost="" invweburiquery="" invweburllength=0 invwebcached=false invwebhttpmethod="" invwebhttpuseragent="" data.short="http access" invwebhttpcontenttype="" invwebhttpuseragentlength=0 invwebsrcip="::ffff:0.0.0.0" type="incoming_http_request" invwebdestip="::ffff:0.0.0.0" id="01FCV6S9YK70GJ5Q6YT0PYKQDA" invapptracingtequestsource="unset" invapptracingcallingservice="unset" data.message="incoming HTTP request was served" invapptracingrequestid="01FCV6S9YJXD2SYG3HTGWXHX0G" -Aug 11 18:14:59 || id="01FCV6SDKRW3XZDA1FAGZ3QVSH" invapptracingrequestid="01FCV6SDKRHB1RR1Q87Q1SKT5P" -Aug 11 18:15:00 || id="01FCV6SE597EY6RJ762V59PZQA" invapptracingrequestid="01FCV6SE596ZMASA1D79M16KVV" -Aug 11 18:15:00 || id="01FCV6SEKCC9RG364AJ60J75KW" invapptracingrequestid="01FCV6SEKCNSQJJ2NDEPQ2TGMP" -Aug 11 18:15:00 || id="01FCV6SF3DXGB8G1DVX19KQZYT" invapptracingrequestid="01FCV6SF3DJZSXTT1RNR6F1QAV" -Aug 11 18:15:05 || id="01FCV6SKY9MM7D795258XPQGC9" invapptracingrequestid="01FCV6SKY9M1D725HTV0ZXKF1V" -Aug 11 18:15:10 || data.event="Shutdown" type="service-shutdown" id="01FCV6SR6JZH7JZ6RFDFN9Q99Y" +Aug 11 18:14:50 || service=web_1 specversion=1.0 type=simple-log invloglevel=Info data.short=service-startup id=01FCV6S4M6S8H3VKAQD9SWFWFP datacontenttype=application/json source=irn:libraries:github.com/InVisionApp/invlogger data.message=The login-api service is running on port 8085. +Aug 11 18:14:55 || invweburl= invwebbytes=0 invwebstatus=0 invweburipath= invwebbytesin=0 invwebdesthost= invweburiquery= invwebbytesout=0 invwebduration=0 invwebhttpmethod= invweburllength=0 invwebcached=false invwebhttpuseragent= data.short=http access invwebhttpcontenttype= invwebsrcip=::ffff:0.0.0.0 type=incoming_http_request invwebdestip=::ffff:0.0.0.0 invwebhttpuseragentlength=0 id=01FCV6S9YK70GJ5Q6YT0PYKQDA invapptracingtequestsource=unset invapptracingcallingservice=unset data.message=incoming HTTP request was served invapptracingrequestid=01FCV6S9YJXD2SYG3HTGWXHX0G +Aug 11 18:14:59 || id=01FCV6SDKRW3XZDA1FAGZ3QVSH invapptracingrequestid=01FCV6SDKRHB1RR1Q87Q1SKT5P +Aug 11 18:15:00 || id=01FCV6SE597EY6RJ762V59PZQA invapptracingrequestid=01FCV6SE596ZMASA1D79M16KVV +Aug 11 18:15:00 || id=01FCV6SEKCC9RG364AJ60J75KW invapptracingrequestid=01FCV6SEKCNSQJJ2NDEPQ2TGMP +Aug 11 18:15:00 || id=01FCV6SF3DXGB8G1DVX19KQZYT invapptracingrequestid=01FCV6SF3DJZSXTT1RNR6F1QAV +Aug 11 18:15:05 || id=01FCV6SKY9MM7D795258XPQGC9 invapptracingrequestid=01FCV6SKY9M1D725HTV0ZXKF1V +Aug 11 18:15:10 || data.event=Shutdown type=service-shutdown id=01FCV6SR6JZH7JZ6RFDFN9Q99Y diff --git a/test/cases/20001-strip-syslog/want b/test/cases/20001-strip-syslog/want index eb7070ea..7a174ee0 100644 --- a/test/cases/20001-strip-syslog/want +++ b/test/cases/20001-strip-syslog/want @@ -1,8 +1,8 @@ -Aug 11 18:14:50 || specversion="1.0" type="simple-log" invloglevel="Info" data.short="service-startup" id="01FCV6S4M6S8H3VKAQD9SWFWFP" datacontenttype="application/json" source="irn:libraries:github.com/InVisionApp/invlogger" data.message="The login-api service is running on port 8085." -Aug 11 18:14:55 || invweburl="" invwebbytes=0 invwebstatus=0 invwebbytesin=0 invwebbytesout=0 invwebduration=0 invweburipath="" invwebdesthost="" invweburiquery="" invweburllength=0 invwebcached=false invwebhttpmethod="" invwebhttpuseragent="" data.short="http access" invwebhttpcontenttype="" invwebhttpuseragentlength=0 invwebsrcip="::ffff:0.0.0.0" type="incoming_http_request" invwebdestip="::ffff:0.0.0.0" id="01FCV6S9YK70GJ5Q6YT0PYKQDA" invapptracingtequestsource="unset" invapptracingcallingservice="unset" data.message="incoming HTTP request was served" invapptracingrequestid="01FCV6S9YJXD2SYG3HTGWXHX0G" -Aug 11 18:14:59 || id="01FCV6SDKRW3XZDA1FAGZ3QVSH" invapptracingrequestid="01FCV6SDKRHB1RR1Q87Q1SKT5P" -Aug 11 18:15:00 || id="01FCV6SE597EY6RJ762V59PZQA" invapptracingrequestid="01FCV6SE596ZMASA1D79M16KVV" -Aug 11 18:15:00 || id="01FCV6SEKCC9RG364AJ60J75KW" invapptracingrequestid="01FCV6SEKCNSQJJ2NDEPQ2TGMP" -Aug 11 18:15:00 || id="01FCV6SF3DXGB8G1DVX19KQZYT" invapptracingrequestid="01FCV6SF3DJZSXTT1RNR6F1QAV" -Aug 11 18:15:05 || id="01FCV6SKY9MM7D795258XPQGC9" invapptracingrequestid="01FCV6SKY9M1D725HTV0ZXKF1V" -Aug 11 18:15:10 || data.event="Shutdown" type="service-shutdown" id="01FCV6SR6JZH7JZ6RFDFN9Q99Y" +Aug 11 18:14:50 || specversion=1.0 type=simple-log invloglevel=Info data.short=service-startup id=01FCV6S4M6S8H3VKAQD9SWFWFP datacontenttype=application/json source=irn:libraries:github.com/InVisionApp/invlogger data.message=The login-api service is running on port 8085. +Aug 11 18:14:55 || invweburl= invwebbytes=0 invwebstatus=0 invweburipath= invwebbytesin=0 invwebdesthost= invweburiquery= invwebbytesout=0 invwebduration=0 invwebhttpmethod= invweburllength=0 invwebcached=false invwebhttpuseragent= data.short=http access invwebhttpcontenttype= invwebsrcip=::ffff:0.0.0.0 type=incoming_http_request invwebdestip=::ffff:0.0.0.0 invwebhttpuseragentlength=0 id=01FCV6S9YK70GJ5Q6YT0PYKQDA invapptracingtequestsource=unset invapptracingcallingservice=unset data.message=incoming HTTP request was served invapptracingrequestid=01FCV6S9YJXD2SYG3HTGWXHX0G +Aug 11 18:14:59 || id=01FCV6SDKRW3XZDA1FAGZ3QVSH invapptracingrequestid=01FCV6SDKRHB1RR1Q87Q1SKT5P +Aug 11 18:15:00 || id=01FCV6SE597EY6RJ762V59PZQA invapptracingrequestid=01FCV6SE596ZMASA1D79M16KVV +Aug 11 18:15:00 || id=01FCV6SEKCC9RG364AJ60J75KW invapptracingrequestid=01FCV6SEKCNSQJJ2NDEPQ2TGMP +Aug 11 18:15:00 || id=01FCV6SF3DXGB8G1DVX19KQZYT invapptracingrequestid=01FCV6SF3DJZSXTT1RNR6F1QAV +Aug 11 18:15:05 || id=01FCV6SKY9MM7D795258XPQGC9 invapptracingrequestid=01FCV6SKY9M1D725HTV0ZXKF1V +Aug 11 18:15:10 || data.event=Shutdown type=service-shutdown id=01FCV6SR6JZH7JZ6RFDFN9Q99Y diff --git a/zap_development_handler.go b/zap_development_handler.go index f20fb4ec..bf702ef1 100644 --- a/zap_development_handler.go +++ b/zap_development_handler.go @@ -40,7 +40,7 @@ func tryZapDevPrefix(d []byte, ev *typesv1.StructuredLogEvent, handler *JSONHand ev.Lvl = strings.ToLower(string(matches[2])) ev.Msg = string(matches[4]) ev.Kvs = append(ev.Kvs, &typesv1.KV{ - Key: "caller", Value: string(matches[3]), + Key: "caller", Value: typesv1.ValStr(string(matches[3])), }) return true } @@ -64,7 +64,7 @@ func tryZapDevDCPrefix(d []byte, ev *typesv1.StructuredLogEvent, handler *JSONHa ev.Msg = string(matches[4]) ev.Kvs = append( ev.Kvs, - &typesv1.KV{Key: "caller", Value: string(matches[3])}, + &typesv1.KV{Key: "caller", Value: typesv1.ValStr(string(matches[3]))}, ) return true } diff --git a/zap_development_handler_test.go b/zap_development_handler_test.go index ca20d88e..46810b26 100644 --- a/zap_development_handler_test.go +++ b/zap_development_handler_test.go @@ -229,7 +229,7 @@ func Test_tryZapDevPrefix(t *testing.T) { func findFieldValue(ev *typesv1.StructuredLogEvent, field string) string { for _, kv := range ev.Kvs { if kv.Key == field { - return kv.Value + return kv.Value.GetStr() } } return ""