From 9db6eae2440a491b430a5f63b0c9af8827b33008 Mon Sep 17 00:00:00 2001 From: Antoine Grondin Date: Wed, 18 Dec 2024 19:47:08 +0900 Subject: [PATCH] storage test suite query (#141) * 30 support asctime field for timestamp (#139) * test(parse asctime field): add a test that fails to parse the asctime field * feat(parse asctime field): add "asctime" into DefalutOptions.TimeFields * feat(parse asctime field): try parse single element array([]interface{}) value into time field * test(parse asctime): merge test cases into the table * test(parse asctime): change into table-driven test * chore(localstorage): test suite now expects timestamp to always be set * chore(localstorage): test for simple query eval --------- Co-authored-by: Kev --- go.mod | 6 +- go.sum | 10 ++- pkg/localstorage/test_suite.go | 128 +++++++++++++++++++++++++++++---- 3 files changed, 121 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 0adc065..d394ae3 100644 --- a/go.mod +++ b/go.mod @@ -21,8 +21,8 @@ require ( github.com/go-logfmt/logfmt v0.5.1 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 - github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72 - github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 + github.com/humanlogio/api/go v0.0.0-20241216091509-0fcc02eed938 + github.com/humanlogio/humanlog-pro v0.0.0-20241217115348-251a98ffde3c github.com/kr/logfmt v0.0.0-20210122060352-19f9bcb100e6 github.com/lrstanley/bubblezone v0.0.0-20240914071701-b48c55a5e78e github.com/matoous/go-nanoid v1.5.0 @@ -87,3 +87,5 @@ require ( golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// replace github.com/humanlogio/api/go => ../api/go diff --git a/go.sum b/go.sum index 86c4218..b158768 100644 --- a/go.sum +++ b/go.sum @@ -80,12 +80,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 h1:lsiJGrN2E5qW6yvD6TKTAHa8lURmnh6KAuHvRfU8csU= -github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I= -github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72 h1:68dDinP4+R4eaEaVXOCiZEwypLk3aUEw4gnF7QL+oH0= -github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72/go.mod h1:pFt3YKuAVJk5nziOiKXTKyq5fj4aA9azq6xOx/932KQ= -github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 h1:tdUCzFh8qvnWNCmxub0KSj1lIiCeWqvRjsMSSIApneE= -github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9/go.mod h1:zq05mTZQXvKheFiAGlPx6+VSo29jw2ER8oy8DIQKW2Q= +github.com/humanlogio/api/go v0.0.0-20241216091509-0fcc02eed938 h1:XXZN889MpHbM8ouhCWar47hbdfh1yZUlSnJoOue7xnU= +github.com/humanlogio/api/go v0.0.0-20241216091509-0fcc02eed938/go.mod h1:pFt3YKuAVJk5nziOiKXTKyq5fj4aA9azq6xOx/932KQ= +github.com/humanlogio/humanlog-pro v0.0.0-20241217115348-251a98ffde3c h1:FPMln3Ff3WTEiX6z69DxxuxLaZZCNgRbKHtDdXohe0c= +github.com/humanlogio/humanlog-pro v0.0.0-20241217115348-251a98ffde3c/go.mod h1:+DAvLjnurssG0w1mZrGDqAA6ZfHcllzvC97nVGvi4cE= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= diff --git a/pkg/localstorage/test_suite.go b/pkg/localstorage/test_suite.go index 62eaf9f..04d5104 100644 --- a/pkg/localstorage/test_suite.go +++ b/pkg/localstorage/test_suite.go @@ -58,10 +58,10 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) { { MachineId: 1, SessionId: 2, Logs: []*typesv1.LogEvent{ - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4")}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.001"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.004"))}*/}, }, }, }, @@ -86,9 +86,9 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) { { MachineId: 1, SessionId: 2, Logs: []*typesv1.LogEvent{ - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.001"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/}, }, }, }, @@ -114,9 +114,9 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) { { MachineId: 1, SessionId: 2, Logs: []*typesv1.LogEvent{ - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4")}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.004"))}*/}, }, }, }, @@ -142,9 +142,9 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) { { MachineId: 1, SessionId: 2, Logs: []*typesv1.LogEvent{ - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4")}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.004"))}*/}, }, }, }, @@ -170,8 +170,106 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) { { MachineId: 1, SessionId: 2, Logs: []*typesv1.LogEvent{ - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")}, - {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/}, + {ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/}, + }, + }, + }, + }, + { + name: "simple query on `lvl`", + q: &typesv1.LogQuery{ + From: timestamppb.New(musttime("2006-01-02T15:04:06.002")), + To: timestamppb.New(musttime("2006-01-02T15:04:06.004")), + Query: typesv1.ExprBinary( + typesv1.ExprIdentifier("lvl"), + typesv1.BinaryOp_CMP_EQ, + typesv1.ExprLiteral(typesv1.ValStr("error")), + ), + }, + input: []*typesv1.LogEventGroup{ + { + MachineId: 1, SessionId: 2, + Logs: []*typesv1.LogEvent{ + { + ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1"), + Structured: &typesv1.StructuredLogEvent{ + Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.001")), + Lvl: "error", + Msg: "some sort of problem", + Kvs: []*typesv1.KV{}, + }, + }, + { + ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1"), + Structured: &typesv1.StructuredLogEvent{ + Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.001")), + Lvl: "info", + Msg: "no problem, all is fine", + Kvs: []*typesv1.KV{}, + }, + }, + { + ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 1"), + Structured: &typesv1.StructuredLogEvent{ + Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003")), + Lvl: "error", + Msg: "some sort of problem a bit later", + Kvs: []*typesv1.KV{}, + }, + }, + { + ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 1"), + Structured: &typesv1.StructuredLogEvent{ + Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003")), + Lvl: "info", + Msg: "no problem, all is fine a bit later", + Kvs: []*typesv1.KV{}, + }, + }, + { + ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.006")), Raw: []byte("hello world 1"), + Structured: &typesv1.StructuredLogEvent{ + Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.006")), + Lvl: "error", + Msg: "some sort of problem too late", + Kvs: []*typesv1.KV{}, + }, + }, + { + ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.006")), Raw: []byte("hello world 1"), + Structured: &typesv1.StructuredLogEvent{ + Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.006")), + Lvl: "info", + Msg: "no problem, all is fine too late", + Kvs: []*typesv1.KV{}, + }, + }, + }, + }, + }, + want: []*typesv1.LogEventGroup{ + { + MachineId: 1, SessionId: 2, + Logs: []*typesv1.LogEvent{ + { + ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 1"), + Structured: &typesv1.StructuredLogEvent{ + Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003")), + Lvl: "error", + Msg: "some sort of problem a bit later", + Kvs: []*typesv1.KV{}, + }, + }, + { + ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 1"), + Structured: &typesv1.StructuredLogEvent{ + Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003")), + Lvl: "info", + Msg: "no problem, all is fine a bit later", + Kvs: []*typesv1.KV{}, + }, + }, }, }, },