Skip to content

Commit

Permalink
feat(scanner): dynamic reordering of successful matchers
Browse files Browse the repository at this point in the history
  • Loading branch information
aybabtme committed Dec 5, 2024
1 parent d3103da commit 5a34ef0
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 12 deletions.
17 changes: 17 additions & 0 deletions optimize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package humanlog

// moveToFront moves the element at index `i` to the front
// of the slice
func moveToFront[El any](i int, s []El) []El {
if i == 0 {
return s
}
el := s[i]
for j := i; j > 0; j-- {
s[j] = s[j-1]
}
s[0] = el
return s
}

const dynamicReordering = false
47 changes: 38 additions & 9 deletions scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,20 @@ func Scan(ctx context.Context, src io.Reader, sink sink.Sink, opts *HandlerOptio
data := new(typesv1.StructuredLogEvent)
ev.Structured = data

handlers := []func([]byte, *typesv1.StructuredLogEvent) bool{
jsonEntry.TryHandle,
logfmtEntry.TryHandle,
func(lineData []byte, data *typesv1.StructuredLogEvent) bool {
return tryDockerComposePrefix(lineData, data, &jsonEntry)
},
func(lineData []byte, data *typesv1.StructuredLogEvent) bool {
return tryDockerComposePrefix(lineData, data, &logfmtEntry)
},
func(lineData []byte, data *typesv1.StructuredLogEvent) bool {
return tryZapDevPrefix(lineData, data, &jsonEntry)
},
}

skipNextScan := false
for {
if !in.Scan() {
Expand Down Expand Up @@ -65,21 +79,36 @@ func Scan(ctx context.Context, src io.Reader, sink sink.Sink, opts *HandlerOptio

// remove that pesky syslog crap
lineData = bytes.TrimPrefix(lineData, []byte("@cee: "))
switch {

case jsonEntry.TryHandle(lineData, data):
handled := false
handled_line:
for i, tryHandler := range handlers {
if tryHandler(lineData, data) {
if dynamicReordering {
handlers = moveToFront(i, handlers)
}
handled = true
break handled_line
}
}
if !handled {
ev.Structured = nil
}
// switch {

case logfmtEntry.TryHandle(lineData, data):
// case jsonEntry.TryHandle(lineData, data):

case tryDockerComposePrefix(lineData, data, &jsonEntry):
// case logfmtEntry.TryHandle(lineData, data):

case tryDockerComposePrefix(lineData, data, &logfmtEntry):
// case tryDockerComposePrefix(lineData, data, &jsonEntry):

case tryZapDevPrefix(lineData, data, &jsonEntry):
// case tryDockerComposePrefix(lineData, data, &logfmtEntry):

default:
ev.Structured = nil
}
// case tryZapDevPrefix(lineData, data, &jsonEntry):

// default:

// }
if err := sink.Receive(ctx, ev); err != nil {
return err
}
Expand Down
8 changes: 5 additions & 3 deletions time_parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,15 @@ func tryParseTime(value interface{}) (time.Time, bool) {
var t time.Time
switch v := value.(type) {
case string:
for _, layout := range TimeFormats {
t, err := time.Parse(layout, v)
for i, layout := range TimeFormats {
t, err := time.Parse(layout, value.(string))
if err == nil {
if dynamicReordering {
TimeFormats = moveToFront(i, TimeFormats)
}
t = fixTimebeforeUnixZero(t)
return t, true
}

}
// try to parse unix time number from string
floatVal, err := strconv.ParseFloat(v, 64)
Expand Down
47 changes: 47 additions & 0 deletions time_parse_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,55 @@ import (
"fmt"
"testing"
"time"

"github.com/stretchr/testify/require"
)

func TestMoveToFront(t *testing.T) {
t.Run("already front", func(t *testing.T) {
in := []string{
"a",
"b",
"c",
}
want := []string{
"a",
"b",
"c",
}
got := moveToFront(0, in)
require.Equal(t, want, got)
})
t.Run("middle", func(t *testing.T) {
in := []string{
"a",
"b",
"c",
}
want := []string{
"b",
"a",
"c",
}
got := moveToFront(1, in)
require.Equal(t, want, got)
})
t.Run("last", func(t *testing.T) {
in := []string{
"a",
"b",
"c",
}
want := []string{
"c",
"a",
"b",
}
got := moveToFront(2, in)
require.Equal(t, want, got)
})
}

func TestTimeParseFloat64(t *testing.T) {
t.Run("nanoseconds", func(t *testing.T) {
golden := float64(1540369190466951764)
Expand Down

0 comments on commit 5a34ef0

Please sign in to comment.