From 9f4bbea0376e8f52b74c0e556c2ae658f46f8658 Mon Sep 17 00:00:00 2001 From: Arne Claus Date: Tue, 11 Jul 2017 11:25:44 +0200 Subject: [PATCH 1/2] Added parralel capture benchmark --- grok_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/grok_test.go b/grok_test.go index 19637ed..3f2061a 100644 --- a/grok_test.go +++ b/grok_test.go @@ -613,6 +613,18 @@ func BenchmarkCapturesTypedReal(b *testing.B) { } } +func BenchmarkParallelCaptures(b *testing.B) { + g, _ := NewWithConfig(&Config{NamedCapturesOnly: true}) + b.ReportAllocs() + b.ResetTimer() + + b.RunParallel(func(b *testing.PB) { + for b.Next() { + g.Parse(`%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`, `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) + } + }) +} + func TestGrok_AddPatternsFromMap_not_exist(t *testing.T) { defer func() { if r := recover(); r != nil { From e4222b34a2854ed2797cc02b6bb098511db36761 Mon Sep 17 00:00:00 2001 From: Arne Claus Date: Tue, 11 Jul 2017 11:37:16 +0200 Subject: [PATCH 2/2] Replaced global mutex with two rwlocks --- grok.go | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/grok.go b/grok.go index 62729c6..fd9172a 100644 --- a/grok.go +++ b/grok.go @@ -33,7 +33,8 @@ type Grok struct { config *Config compiledPatterns map[string]*gRegexp patterns map[string]*gPattern - serviceMu sync.Mutex + patternsGuard *sync.RWMutex + compiledGuard *sync.RWMutex } type gPattern struct { @@ -61,6 +62,8 @@ func NewWithConfig(config *Config) (*Grok, error) { compiledPatterns: map[string]*gRegexp{}, patterns: map[string]*gPattern{}, rawPattern: map[string]string{}, + patternsGuard: new(sync.RWMutex), + compiledGuard: new(sync.RWMutex), } if !config.SkipDefaultPatterns { @@ -97,17 +100,18 @@ func (g *Grok) addPattern(name, pattern string) error { // AddPattern adds a named pattern to grok func (g *Grok) AddPattern(name, pattern string) error { - g.serviceMu.Lock() + g.patternsGuard.Lock() + defer g.patternsGuard.Unlock() + g.rawPattern[name] = pattern g.buildPatterns() - g.serviceMu.Unlock() return nil } // AddPatternsFromMap loads a map of named patterns func (g *Grok) AddPatternsFromMap(m map[string]string) error { - g.serviceMu.Lock() - defer g.serviceMu.Unlock() + g.patternsGuard.Lock() + defer g.patternsGuard.Unlock() for name, pattern := range m { g.rawPattern[name] = pattern @@ -196,7 +200,6 @@ func (g *Grok) Match(pattern, text string) (bool, error) { // compiledParse parses the specified text and returns a map with the results. func (g *Grok) compiledParse(gr *gRegexp, text string) (map[string]string, error) { captures := make(map[string]string) - g.serviceMu.Lock() if match := gr.regexp.FindStringSubmatch(text); len(match) > 0 { for i, name := range gr.regexp.SubexpNames() { if name != "" { @@ -204,11 +207,9 @@ func (g *Grok) compiledParse(gr *gRegexp, text string) (map[string]string, error continue } captures[name] = match[i] - } } } - g.serviceMu.Unlock() return captures, nil } @@ -287,15 +288,17 @@ func (g *Grok) buildPatterns() error { } func (g *Grok) compile(pattern string) (*gRegexp, error) { - g.serviceMu.Lock() - defer g.serviceMu.Unlock() - if g.compiledPatterns == nil { - g.compiledPatterns = map[string]*gRegexp{} - } - if gr, ok := g.compiledPatterns[pattern]; ok { + g.compiledGuard.RLock() + gr, ok := g.compiledPatterns[pattern] + g.compiledGuard.RUnlock() + + if ok { return gr, nil } + + g.patternsGuard.RLock() newPattern, ti, err := g.denormalizePattern(pattern, g.patterns) + g.patternsGuard.RUnlock() if err != nil { return nil, err } @@ -304,8 +307,12 @@ func (g *Grok) compile(pattern string) (*gRegexp, error) { if err != nil { return nil, err } - gr := &gRegexp{regexp: compiledRegex, typeInfo: ti} + gr = &gRegexp{regexp: compiledRegex, typeInfo: ti} + + g.compiledGuard.Lock() g.compiledPatterns[pattern] = gr + g.compiledGuard.Unlock() + return gr, nil }