Skip to content

Commit

Permalink
storage test suite query (#141)
Browse files Browse the repository at this point in the history 
* 30 support asctime field for timestamp (#139)

* test(parse asctime field): add a test that fails to parse the asctime field

* feat(parse asctime field): add "asctime" into DefalutOptions.TimeFields

* feat(parse asctime field): try parse single element array([]interface{}) value into time field

* test(parse asctime): merge test cases into the table

* test(parse asctime): change into table-driven test

* chore(localstorage): test suite now expects timestamp to always be set

* chore(localstorage): test for simple query eval

---------

Co-authored-by: Kev <[email protected]>
  • Loading branch information
@aybabtme @KevRiver
aybabtme and KevRiver authored Dec 18, 2024
1 parent 046d073 commit 9db6eae
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 23 deletions.
6 changes: 4 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ require (
github.com/go-logfmt/logfmt v0.5.1
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72
github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9
github.com/humanlogio/api/go v0.0.0-20241216091509-0fcc02eed938
github.com/humanlogio/humanlog-pro v0.0.0-20241217115348-251a98ffde3c
github.com/kr/logfmt v0.0.0-20210122060352-19f9bcb100e6
github.com/lrstanley/bubblezone v0.0.0-20240914071701-b48c55a5e78e
github.com/matoous/go-nanoid v1.5.0
Expand Down Expand Up @@ -87,3 +87,5 @@ require (
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

// replace github.com/humanlogio/api/go => ../api/go
10 changes: 4 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7 h1:lsiJGrN2E5qW6yvD6TKTAHa8lURmnh6KAuHvRfU8csU=
github.com/humanlogio/api/go v0.0.0-20241208082433-416862db1fa7/go.mod h1:+hU/MU1g6QvtbeknKOlUI1yEStVqkPJ8jmYIj63OV5I=
github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72 h1:68dDinP4+R4eaEaVXOCiZEwypLk3aUEw4gnF7QL+oH0=
github.com/humanlogio/api/go v0.0.0-20241211090836-a1e1ce8a4f72/go.mod h1:pFt3YKuAVJk5nziOiKXTKyq5fj4aA9azq6xOx/932KQ=
github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9 h1:tdUCzFh8qvnWNCmxub0KSj1lIiCeWqvRjsMSSIApneE=
github.com/humanlogio/humanlog-pro v0.0.0-20241129104809-3580d74828a9/go.mod h1:zq05mTZQXvKheFiAGlPx6+VSo29jw2ER8oy8DIQKW2Q=
github.com/humanlogio/api/go v0.0.0-20241216091509-0fcc02eed938 h1:XXZN889MpHbM8ouhCWar47hbdfh1yZUlSnJoOue7xnU=
github.com/humanlogio/api/go v0.0.0-20241216091509-0fcc02eed938/go.mod h1:pFt3YKuAVJk5nziOiKXTKyq5fj4aA9azq6xOx/932KQ=
github.com/humanlogio/humanlog-pro v0.0.0-20241217115348-251a98ffde3c h1:FPMln3Ff3WTEiX6z69DxxuxLaZZCNgRbKHtDdXohe0c=
github.com/humanlogio/humanlog-pro v0.0.0-20241217115348-251a98ffde3c/go.mod h1:+DAvLjnurssG0w1mZrGDqAA6ZfHcllzvC97nVGvi4cE=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
Expand Down
128 changes: 113 additions & 15 deletions pkg/localstorage/test_suite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,10 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) {
{
MachineId: 1, SessionId: 2,
Logs: []*typesv1.LogEvent{
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.001"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.004"))}*/},
},
},
},
Expand All @@ -86,9 +86,9 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) {
{
MachineId: 1, SessionId: 2,
Logs: []*typesv1.LogEvent{
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.001"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/},
},
},
},
Expand All @@ -114,9 +114,9 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) {
{
MachineId: 1, SessionId: 2,
Logs: []*typesv1.LogEvent{
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.004"))}*/},
},
},
},
Expand All @@ -142,9 +142,9 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) {
{
MachineId: 1, SessionId: 2,
Logs: []*typesv1.LogEvent{
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.004")), Raw: []byte("hello world 4") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.004"))}*/},
},
},
},
Expand All @@ -170,8 +170,106 @@ func RunTest(t *testing.T, constructor func(t *testing.T) Storage) {
{
MachineId: 1, SessionId: 2,
Logs: []*typesv1.LogEvent{
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3")},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.002")), Raw: []byte("hello world 2") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.002"))}*/},
{ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 3") /*, Structured: &typesv1.StructuredLogEvent{Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003"))}*/},
},
},
},
},
{
name: "simple query on `lvl`",
q: &typesv1.LogQuery{
From: timestamppb.New(musttime("2006-01-02T15:04:06.002")),
To: timestamppb.New(musttime("2006-01-02T15:04:06.004")),
Query: typesv1.ExprBinary(
typesv1.ExprIdentifier("lvl"),
typesv1.BinaryOp_CMP_EQ,
typesv1.ExprLiteral(typesv1.ValStr("error")),
),
},
input: []*typesv1.LogEventGroup{
{
MachineId: 1, SessionId: 2,
Logs: []*typesv1.LogEvent{
{
ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1"),
Structured: &typesv1.StructuredLogEvent{
Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.001")),
Lvl: "error",
Msg: "some sort of problem",
Kvs: []*typesv1.KV{},
},
},
{
ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.001")), Raw: []byte("hello world 1"),
Structured: &typesv1.StructuredLogEvent{
Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.001")),
Lvl: "info",
Msg: "no problem, all is fine",
Kvs: []*typesv1.KV{},
},
},
{
ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 1"),
Structured: &typesv1.StructuredLogEvent{
Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003")),
Lvl: "error",
Msg: "some sort of problem a bit later",
Kvs: []*typesv1.KV{},
},
},
{
ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 1"),
Structured: &typesv1.StructuredLogEvent{
Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003")),
Lvl: "info",
Msg: "no problem, all is fine a bit later",
Kvs: []*typesv1.KV{},
},
},
{
ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.006")), Raw: []byte("hello world 1"),
Structured: &typesv1.StructuredLogEvent{
Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.006")),
Lvl: "error",
Msg: "some sort of problem too late",
Kvs: []*typesv1.KV{},
},
},
{
ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.006")), Raw: []byte("hello world 1"),
Structured: &typesv1.StructuredLogEvent{
Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.006")),
Lvl: "info",
Msg: "no problem, all is fine too late",
Kvs: []*typesv1.KV{},
},
},
},
},
},
want: []*typesv1.LogEventGroup{
{
MachineId: 1, SessionId: 2,
Logs: []*typesv1.LogEvent{
{
ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 1"),
Structured: &typesv1.StructuredLogEvent{
Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003")),
Lvl: "error",
Msg: "some sort of problem a bit later",
Kvs: []*typesv1.KV{},
},
},
{
ParsedAt: timestamppb.New(musttime("2006-01-02T15:04:06.003")), Raw: []byte("hello world 1"),
Structured: &typesv1.StructuredLogEvent{
Timestamp: timestamppb.New(musttime("2006-01-02T15:04:06.003")),
Lvl: "info",
Msg: "no problem, all is fine a bit later",
Kvs: []*typesv1.KV{},
},
},
},
},
},
Expand Down

0 comments on commit 9db6eae

Please sign in to comment.