diff --git a/compiler.go b/compiler/compiler.go similarity index 51% rename from compiler.go rename to compiler/compiler.go index bc48559..0278b57 100644 --- a/compiler.go +++ b/compiler/compiler.go @@ -1,4 +1,4 @@ -package glob +package compiler // TODO use constructor with all matchers, and to their structs private // TODO glue multiple Text nodes (like after QuoteMeta) @@ -6,11 +6,12 @@ package glob import ( "fmt" "github.com/gobwas/glob/match" - "github.com/gobwas/glob/runes" + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/util/runes" "reflect" ) -func optimize(matcher match.Matcher) match.Matcher { +func optimizeMatcher(matcher match.Matcher) match.Matcher { switch m := matcher.(type) { case match.Any: @@ -33,8 +34,8 @@ func optimize(matcher match.Matcher) match.Matcher { return m case match.BTree: - m.Left = optimize(m.Left) - m.Right = optimize(m.Right) + m.Left = optimizeMatcher(m.Left) + m.Right = optimizeMatcher(m.Right) r, ok := m.Value.(match.Text) if !ok { @@ -80,34 +81,72 @@ func optimize(matcher match.Matcher) match.Matcher { return matcher } -func glueMatchers(matchers []match.Matcher) match.Matcher { - var ( - glued []match.Matcher - winner match.Matcher - ) +func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { + if len(matchers) == 0 { + return nil, fmt.Errorf("compile error: need at least one matcher") + } + if len(matchers) == 1 { + return matchers[0], nil + } + if m := glueMatchers(matchers); m != nil { + return m, nil + } + + idx := -1 maxLen := -1 + var val match.Matcher + for i, matcher := range matchers { + if l := matcher.Len(); l != -1 && l >= maxLen { + maxLen = l + idx = i + val = matcher + } + } - if m := glueAsEvery(matchers); m != nil { - glued = append(glued, m) - return m + if val == nil { // not found matcher with static length + r, err := compileMatchers(matchers[1:]) + if err != nil { + return nil, err + } + return match.NewBTree(matchers[0], nil, r), nil } - if m := glueAsRow(matchers); m != nil { - glued = append(glued, m) - return m + left := matchers[:idx] + var right []match.Matcher + if len(matchers) > idx+1 { + right = matchers[idx+1:] } - for _, g := range glued { - if l := g.Len(); l > maxLen { - maxLen = l - winner = g + var l, r match.Matcher + var err error + if len(left) > 0 { + l, err = compileMatchers(left) + if err != nil { + return nil, err + } + } + + if len(right) > 0 { + r, err = compileMatchers(right) + if err != nil { + return nil, err } } - return winner + return match.NewBTree(val, l, r), nil +} + +func glueMatchers(matchers []match.Matcher) match.Matcher { + if m := glueMatchersAsEvery(matchers); m != nil { + return m + } + if m := glueMatchersAsRow(matchers); m != nil { + return m + } + return nil } -func glueAsRow(matchers []match.Matcher) match.Matcher { +func glueMatchersAsRow(matchers []match.Matcher) match.Matcher { if len(matchers) <= 1 { return nil } @@ -124,11 +163,10 @@ func glueAsRow(matchers []match.Matcher) match.Matcher { l += ml } } - return match.NewRow(l, c...) } -func glueAsEvery(matchers []match.Matcher) match.Matcher { +func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher { if len(matchers) <= 1 { return nil } @@ -254,255 +292,212 @@ func minimizeMatchers(matchers []match.Matcher) []match.Matcher { return minimizeMatchers(next) } -func minimizeAnyOf(children []node) node { - var nodes [][]node - var min int - var idx int - for i, desc := range children { - pat, ok := desc.(*nodePattern) - if !ok { - return nil - } - - n := pat.children() - ln := len(n) - if len(nodes) == 0 || (ln < min) { - min = ln - idx = i - } - - nodes = append(nodes, pat.children()) - } - - minNodes := nodes[idx] - if idx+1 < len(nodes) { - nodes = append(nodes[:idx], nodes[idx+1:]...) - } else { - nodes = nodes[:idx] - } - - var commonLeft []node - var commonLeftCount int - for i, n := range minNodes { - has := true - for _, t := range nodes { - if !reflect.DeepEqual(n, t[i]) { - has = false - break - } - } - - if has { - commonLeft = append(commonLeft, n) - commonLeftCount++ - } else { - break - } +// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree +func minimizeTree(tree *ast.Node) *ast.Node { + switch tree.Kind { + case ast.KindAnyOf: + return minimizeTreeAnyOf(tree) + default: + return nil } +} - var commonRight []node - var commonRightCount int - for i := min - 1; i > commonLeftCount-1; i-- { - n := minNodes[i] - has := true - for _, t := range nodes { - if !reflect.DeepEqual(n, t[len(t)-(min-i)]) { - has = false - break - } - } - - if has { - commonRight = append(commonRight, n) - commonRightCount++ - } else { - break - } +// minimizeAnyOf tries to find common children of given node of AnyOf pattern +// it searches for common children from left and from right +// if any common children are found – then it returns new optimized ast tree +// else it returns nil +func minimizeTreeAnyOf(tree *ast.Node) *ast.Node { + if !areOfSameKind(tree.Children, ast.KindPattern) { + return nil } - if commonLeftCount == 0 && commonRightCount == 0 { + commonLeft, commonRight := commonChildren(tree.Children) + commonLeftCount, commonRightCount := len(commonLeft), len(commonRight) + if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts return nil } - nodes = append(nodes, minNodes) - nodes[len(nodes)-1], nodes[idx] = nodes[idx], nodes[len(nodes)-1] - - var result []node + var result []*ast.Node if commonLeftCount > 0 { - result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: commonLeft}}) + result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...)) } - var anyOf []node - for _, n := range nodes { - if commonLeftCount+commonRightCount == len(n) { - anyOf = append(anyOf, nil) + var anyOf []*ast.Node + for _, child := range tree.Children { + reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount] + var node *ast.Node + if len(reuse) == 0 { + // this pattern is completely reduced by commonLeft and commonRight patterns + // so it become nothing + node = ast.NewNode(ast.KindNothing, nil) } else { - anyOf = append(anyOf, &nodePattern{nodeImpl: nodeImpl{desc: n[commonLeftCount : len(n)-commonRightCount]}}) + node = ast.NewNode(ast.KindPattern, nil, reuse...) } + anyOf = appendIfUnique(anyOf, node) } - - anyOf = uniqueNodes(anyOf) - if len(anyOf) == 1 { - if anyOf[0] != nil { - result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: anyOf}}) - } - } else { - result = append(result, &nodeAnyOf{nodeImpl: nodeImpl{desc: anyOf}}) + switch { + case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing: + result = append(result, anyOf[0]) + case len(anyOf) > 1: + result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...)) } if commonRightCount > 0 { - result = append(result, &nodePattern{nodeImpl: nodeImpl{desc: commonRight}}) + result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...)) } - return &nodePattern{nodeImpl: nodeImpl{desc: result}} + return ast.NewNode(ast.KindPattern, nil, result...) } -func uniqueNodes(nodes []node) (result []node) { -head: - for _, n := range nodes { - for _, e := range result { - if reflect.DeepEqual(e, n) { - continue head - } - } - - result = append(result, n) +func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { + // find node that has least number of children + idx := leastChildren(nodes) + if idx == -1 { + return nil, nil } + tree := nodes[idx] - return -} + var ( + breakLeft bool + breakRight bool + ) + for i, j := 0, len(tree.Children)-1; j >= 0 && !(breakLeft && breakLeft); i, j = i+1, j-1 { + treeLeft := tree.Children[i] + treeRight := tree.Children[j] -func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { - if len(matchers) == 0 { - return nil, fmt.Errorf("compile error: need at least one matcher") - } + fmt.Println(i, j) - if len(matchers) == 1 { - return matchers[0], nil - } + for k := 0; k < len(nodes) && !(breakLeft && breakLeft); k++ { + // skip least children node + if k == idx { + continue + } - if m := glueMatchers(matchers); m != nil { - return m, nil - } + restLeft := nodes[k].Children[i] + restRight := nodes[k].Children[j+len(nodes[k].Children)-len(tree.Children)] - idx := -1 - maxLen := -1 - var val match.Matcher - for i, matcher := range matchers { - if l := matcher.Len(); l != -1 && l >= maxLen { - maxLen = l - idx = i - val = matcher + breakLeft = breakLeft || !treeLeft.Equal(restLeft) + + // disable searching for right common parts, if left part is already overlapping + breakRight = breakRight || (!breakLeft && j <= i) + breakRight = breakRight || !treeRight.Equal(restRight) + } + if !breakLeft { + fmt.Println("left app") + commonLeft = append(commonLeft, treeLeft) + } + if !breakRight { + fmt.Println("right app") + commonRight = append(commonRight, treeRight) } } + return +} - if val == nil { // not found matcher with static length - r, err := compileMatchers(matchers[1:]) - if err != nil { - return nil, err +func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node { + for _, n := range target { + if reflect.DeepEqual(n, val) { + return target } - return match.NewBTree(matchers[0], nil, r), nil } + return append(target, val) +} - left := matchers[:idx] - var right []match.Matcher - if len(matchers) > idx+1 { - right = matchers[idx+1:] +func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool { + for _, n := range nodes { + if n.Kind != kind { + return false + } } + return true +} - var l, r match.Matcher - var err error - if len(left) > 0 { - l, err = compileMatchers(left) - if err != nil { - return nil, err +func leastChildren(nodes []*ast.Node) int { + min := -1 + idx := -1 + for i, n := range nodes { + if idx == -1 || (len(n.Children) < min) { + min = len(n.Children) + idx = i } } + return idx +} - if len(right) > 0 { - r, err = compileMatchers(right) +func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) { + var matchers []match.Matcher + for _, desc := range tree.Children { + m, err := compile(desc, sep) if err != nil { return nil, err } + matchers = append(matchers, optimizeMatcher(m)) } - - return match.NewBTree(val, l, r), nil + return matchers, nil } -func do(leaf node, s []rune) (m match.Matcher, err error) { - switch n := leaf.(type) { - case *nodeAnyOf: - // todo this could be faster on pattern_alternatives_combine_lite - if n := minimizeAnyOf(n.children()); n != nil { - return do(n, s) +func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) { + switch tree.Kind { + case ast.KindAnyOf: + // todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go) + if n := minimizeTree(tree); n != nil { + return compile(n, sep) } - - var matchers []match.Matcher - for _, desc := range n.children() { - if desc == nil { - matchers = append(matchers, match.NewNothing()) - continue - } - - m, err := do(desc, s) - if err != nil { - return nil, err - } - matchers = append(matchers, optimize(m)) + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err } - return match.NewAnyOf(matchers...), nil - case *nodePattern: - nodes := leaf.children() - if len(nodes) == 0 { + case ast.KindPattern: + if len(tree.Children) == 0 { return match.NewNothing(), nil } - - var matchers []match.Matcher - for _, desc := range nodes { - m, err := do(desc, s) - if err != nil { - return nil, err - } - matchers = append(matchers, optimize(m)) + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err } - m, err = compileMatchers(minimizeMatchers(matchers)) if err != nil { return nil, err } - case *nodeList: - m = match.NewList([]rune(n.chars), n.not) + case ast.KindAny: + m = match.NewAny(sep) - case *nodeRange: - m = match.NewRange(n.lo, n.hi, n.not) + case ast.KindSuper: + m = match.NewSuper() - case *nodeAny: - m = match.NewAny(s) + case ast.KindSingle: + m = match.NewSingle(sep) - case *nodeSuper: - m = match.NewSuper() + case ast.KindNothing: + m = match.NewNothing() + + case ast.KindList: + l := tree.Value.(ast.List) + m = match.NewList([]rune(l.Chars), l.Not) - case *nodeSingle: - m = match.NewSingle(s) + case ast.KindRange: + r := tree.Value.(ast.Range) + m = match.NewRange(r.Lo, r.Hi, r.Not) - case *nodeText: - m = match.NewText(n.text) + case ast.KindText: + t := tree.Value.(ast.Text) + m = match.NewText(t.Text) default: return nil, fmt.Errorf("could not compile tree: unknown node type") } - return optimize(m), nil + return optimizeMatcher(m), nil } -func compile(ast *nodePattern, s []rune) (Glob, error) { - g, err := do(ast, s) +func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) { + m, err := compile(tree, sep) if err != nil { return nil, err } - return g, nil + return m, nil } diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go new file mode 100644 index 0000000..a93b918 --- /dev/null +++ b/compiler/compiler_test.go @@ -0,0 +1,573 @@ +package compiler + +import ( + "github.com/gobwas/glob/match" + "github.com/gobwas/glob/match/debug" + "github.com/gobwas/glob/syntax/ast" + "reflect" + "testing" +) + +var separators = []rune{'.'} + +func TestCommonChildren(t *testing.T) { + for i, test := range []struct { + nodes []*ast.Node + left []*ast.Node + right []*ast.Node + }{ + // { + // nodes: []*ast.Node{ + // ast.NewNode(ast.KindNothing, nil, + // ast.NewNode(ast.KindText, ast.Text{"a"}), + // ast.NewNode(ast.KindText, ast.Text{"z"}), + // ast.NewNode(ast.KindText, ast.Text{"c"}), + // ), + // ast.NewNode(ast.KindNothing, nil, + // ast.NewNode(ast.KindText, ast.Text{"a"}), + // ast.NewNode(ast.KindText, ast.Text{"b"}), + // ast.NewNode(ast.KindText, ast.Text{"c"}), + // ), + // }, + // left: []*ast.Node{ + // ast.NewNode(ast.KindText, ast.Text{"a"}), + // }, + // right: []*ast.Node{ + // ast.NewNode(ast.KindText, ast.Text{"c"}), + // }, + // }, + { + nodes: []*ast.Node{ + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + ast.NewNode(ast.KindNothing, nil, + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + ast.NewNode(ast.KindText, ast.Text{"d"}), + ), + }, + left: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"a"}), + ast.NewNode(ast.KindText, ast.Text{"b"}), + ast.NewNode(ast.KindText, ast.Text{"c"}), + }, + right: []*ast.Node{ + ast.NewNode(ast.KindText, ast.Text{"d"}), + }, + }, + } { + left, right := commonChildren(test.nodes) + if !nodesEqual(left, test.left) { + t.Errorf("[%d] left, right := commonChildren(); left = %v; want %v", i, left, test.left) + } + if !nodesEqual(right, test.right) { + t.Errorf("[%d] left, right := commonChildren(); right = %v; want %v", i, right, test.right) + } + } +} + +func nodesEqual(a, b []*ast.Node) bool { + if len(a) != len(b) { + return false + } + for i, av := range a { + if !av.Equal(b[i]) { + return false + } + } + return true +} + +func TestGlueMatchers(t *testing.T) { + for id, test := range []struct { + in []match.Matcher + exp match.Matcher + }{ + { + []match.Matcher{ + match.NewSuper(), + match.NewSingle(nil), + }, + match.NewMin(1), + }, + { + []match.Matcher{ + match.NewAny(separators), + match.NewSingle(separators), + }, + match.EveryOf{match.Matchers{ + match.NewMin(1), + match.NewContains(string(separators), true), + }}, + }, + { + []match.Matcher{ + match.NewSingle(nil), + match.NewSingle(nil), + match.NewSingle(nil), + }, + match.EveryOf{match.Matchers{ + match.NewMin(3), + match.NewMax(3), + }}, + }, + { + []match.Matcher{ + match.NewList([]rune{'a'}, true), + match.NewAny([]rune{'a'}), + }, + match.EveryOf{match.Matchers{ + match.NewMin(1), + match.NewContains("a", true), + }}, + }, + } { + act, err := compileMatchers(test.in) + if err != nil { + t.Errorf("#%d convert matchers error: %s", id, err) + continue + } + + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestCompileMatchers(t *testing.T) { + for id, test := range []struct { + in []match.Matcher + exp match.Matcher + }{ + { + []match.Matcher{ + match.NewSuper(), + match.NewSingle(separators), + match.NewText("c"), + }, + match.NewBTree( + match.NewText("c"), + match.NewBTree( + match.NewSingle(separators), + match.NewSuper(), + nil, + ), + nil, + ), + }, + { + []match.Matcher{ + match.NewAny(nil), + match.NewText("c"), + match.NewAny(nil), + }, + match.NewBTree( + match.NewText("c"), + match.NewAny(nil), + match.NewAny(nil), + ), + }, + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }, + match.NewRow( + 4, + match.Matchers{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }..., + ), + }, + } { + act, err := compileMatchers(test.in) + if err != nil { + t.Errorf("#%d convert matchers error: %s", id, err) + continue + } + + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestConvertMatchers(t *testing.T) { + for id, test := range []struct { + in, exp []match.Matcher + }{ + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + match.NewAny(nil), + }, + []match.Matcher{ + match.NewRow( + 4, + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + }..., + ), + match.NewAny(nil), + }, + }, + { + []match.Matcher{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + match.NewSingle(nil), + match.NewAny(nil), + match.NewSingle(nil), + match.NewSingle(nil), + match.NewAny(nil), + }, + []match.Matcher{ + match.NewRow( + 3, + match.Matchers{ + match.NewRange('a', 'c', true), + match.NewList([]rune{'z', 't', 'e'}, false), + match.NewText("c"), + }..., + ), + match.NewMin(3), + }, + }, + } { + act := minimizeMatchers(test.in) + if !reflect.DeepEqual(act, test.exp) { + t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp) + continue + } + } +} + +func TestCompiler(t *testing.T) { + for id, test := range []struct { + ast *ast.Node + result match.Matcher + sep []rune + }{ + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ), + // result: match.NewText("abc"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ), + // sep: separators, + // result: match.NewAny(separators), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ), + // result: match.NewSuper(), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindSuper, nil), + // ), + // result: match.NewSuper(), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindSingle, nil), + // ), + // sep: separators, + // result: match.NewSingle(separators), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindRange, &ast.Range{ + // Lo: 'a', + // Hi: 'z', + // Not: true, + // }), + // ), + // result: match.NewRange('a', 'z', true), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindList, &ast.List{ + // Chars: "abc", + // Not: true, + // }), + // ), + // result: match.NewList([]rune{'a', 'b', 'c'}, true), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindSingle, nil), + // ), + // sep: separators, + // result: match.EveryOf{Matchers: match.Matchers{ + // match.NewMin(3), + // match.NewContains(string(separators), true), + // }}, + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindSingle, nil), + // ), + // result: match.NewMin(3), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindSingle, nil), + // ), + // sep: separators, + // result: match.NewBTree( + // match.NewRow( + // 4, + // match.Matchers{ + // match.NewText("abc"), + // match.NewSingle(separators), + // }..., + // ), + // match.NewAny(separators), + // nil, + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"/"}), + // ast.NewNode(ast.KindAnyOf, nil, + // ast.NewNode(ast.KindText, &ast.Text{"z"}), + // ast.NewNode(ast.KindText, &ast.Text{"ab"}), + // ), + // ast.NewNode(ast.KindSuper, nil), + // ), + // sep: separators, + // result: match.NewBTree( + // match.NewText("/"), + // nil, + // match.NewBTree( + // match.NewAnyOf(match.NewText("z"), match.NewText("ab")), + // nil, + // match.NewSuper(), + // ), + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindSuper, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindSingle, nil), + // ), + // sep: separators, + // result: match.NewBTree( + // match.NewRow( + // 5, + // match.Matchers{ + // match.NewSingle(separators), + // match.NewText("abc"), + // match.NewSingle(separators), + // }..., + // ), + // match.NewSuper(), + // nil, + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ), + // result: match.NewSuffix("abc"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindAny, nil), + // ), + // result: match.NewPrefix("abc"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"def"}), + // ), + // result: match.NewPrefixSuffix("abc", "def"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ), + // result: match.NewContains("abc", false), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindAny, nil), + // ast.NewNode(ast.KindAny, nil), + // ), + // sep: separators, + // result: match.NewBTree( + // match.NewText("abc"), + // match.NewAny(separators), + // match.NewAny(separators), + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindSuper, nil), + // ast.NewNode(ast.KindSingle, nil), + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindSuper, nil), + // ast.NewNode(ast.KindSingle, nil), + // ), + // result: match.NewBTree( + // match.NewText("abc"), + // match.NewMin(1), + // match.NewMin(1), + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ), + // result: match.NewText("abc"), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAnyOf, nil, + // ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAnyOf, nil, + // ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ), + // ), + // ), + // ), + // ), + // result: match.NewText("abc"), + // }, + { + ast: ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindAnyOf, nil, + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ast.NewNode(ast.KindSingle, nil), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ast.NewNode(ast.KindList, &ast.List{Chars: "def"}), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ), + ast.NewNode(ast.KindPattern, nil, + ast.NewNode(ast.KindText, &ast.Text{"abc"}), + ), + ), + ), + result: match.NewBTree( + match.NewText("abc"), + nil, + match.AnyOf{Matchers: match.Matchers{ + match.NewSingle(nil), + match.NewList([]rune{'d', 'e', 'f'}, false), + match.NewNothing(), + }}, + ), + }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindRange, &ast.Range{Lo: 'a', Hi: 'z'}), + // ast.NewNode(ast.KindRange, &ast.Range{Lo: 'a', Hi: 'x', Not: true}), + // ast.NewNode(ast.KindAny, nil), + // ), + // result: match.NewBTree( + // match.NewRow( + // 2, + // match.Matchers{ + // match.NewRange('a', 'z', false), + // match.NewRange('a', 'x', true), + // }..., + // ), + // nil, + // match.NewSuper(), + // ), + // }, + // { + // ast: ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindAnyOf, nil, + // ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindList, &ast.List{Chars: "abc"}), + // ast.NewNode(ast.KindText, &ast.Text{"ghi"}), + // ), + // ast.NewNode(ast.KindPattern, nil, + // ast.NewNode(ast.KindText, &ast.Text{"abc"}), + // ast.NewNode(ast.KindList, &ast.List{Chars: "def"}), + // ast.NewNode(ast.KindText, &ast.Text{"ghi"}), + // ), + // ), + // ), + // result: match.NewRow( + // 7, + // match.Matchers{ + // match.NewText("abc"), + // match.AnyOf{Matchers: match.Matchers{ + // match.NewList([]rune{'a', 'b', 'c'}, false), + // match.NewList([]rune{'d', 'e', 'f'}, false), + // }}, + // match.NewText("ghi"), + // }..., + // ), + // }, + } { + m, err := Compile(test.ast, test.sep) + if err != nil { + t.Errorf("compilation error: %s", err) + continue + } + + if !reflect.DeepEqual(m, test.result) { + t.Errorf("[%d] Compile():\nexp: %#v\nact: %#v\n\ngraphviz:\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher))) + continue + } + } +} diff --git a/compiler_test.go b/compiler_test.go deleted file mode 100644 index 4c20db2..0000000 --- a/compiler_test.go +++ /dev/null @@ -1,548 +0,0 @@ -package glob - -import ( - "github.com/gobwas/glob/match" - "github.com/gobwas/glob/match/debug" - "reflect" - "testing" -) - -var separators = []rune{'.'} - -func TestGlueMatchers(t *testing.T) { - for id, test := range []struct { - in []match.Matcher - exp match.Matcher - }{ - { - []match.Matcher{ - match.NewSuper(), - match.NewSingle(nil), - }, - match.NewMin(1), - }, - { - []match.Matcher{ - match.NewAny(separators), - match.NewSingle(separators), - }, - match.EveryOf{match.Matchers{ - match.NewMin(1), - match.NewContains(string(separators), true), - }}, - }, - { - []match.Matcher{ - match.NewSingle(nil), - match.NewSingle(nil), - match.NewSingle(nil), - }, - match.EveryOf{match.Matchers{ - match.NewMin(3), - match.NewMax(3), - }}, - }, - { - []match.Matcher{ - match.NewList([]rune{'a'}, true), - match.NewAny([]rune{'a'}), - }, - match.EveryOf{match.Matchers{ - match.NewMin(1), - match.NewContains("a", true), - }}, - }, - } { - act, err := compileMatchers(test.in) - if err != nil { - t.Errorf("#%d convert matchers error: %s", id, err) - continue - } - - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers result:\nact: %#v;\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func TestCompileMatchers(t *testing.T) { - for id, test := range []struct { - in []match.Matcher - exp match.Matcher - }{ - { - []match.Matcher{ - match.NewSuper(), - match.NewSingle(separators), - match.NewText("c"), - }, - match.NewBTree( - match.NewText("c"), - match.NewBTree( - match.NewSingle(separators), - match.NewSuper(), - nil, - ), - nil, - ), - }, - { - []match.Matcher{ - match.NewAny(nil), - match.NewText("c"), - match.NewAny(nil), - }, - match.NewBTree( - match.NewText("c"), - match.NewAny(nil), - match.NewAny(nil), - ), - }, - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }, - match.NewRow( - 4, - match.Matchers{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }..., - ), - }, - } { - act, err := compileMatchers(test.in) - if err != nil { - t.Errorf("#%d convert matchers error: %s", id, err) - continue - } - - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers result:\nact: %#v\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func TestConvertMatchers(t *testing.T) { - for id, test := range []struct { - in, exp []match.Matcher - }{ - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - match.NewAny(nil), - }, - []match.Matcher{ - match.NewRow( - 4, - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - }..., - ), - match.NewAny(nil), - }, - }, - { - []match.Matcher{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - match.NewSingle(nil), - match.NewAny(nil), - match.NewSingle(nil), - match.NewSingle(nil), - match.NewAny(nil), - }, - []match.Matcher{ - match.NewRow( - 3, - match.Matchers{ - match.NewRange('a', 'c', true), - match.NewList([]rune{'z', 't', 'e'}, false), - match.NewText("c"), - }..., - ), - match.NewMin(3), - }, - }, - } { - act := minimizeMatchers(test.in) - if !reflect.DeepEqual(act, test.exp) { - t.Errorf("#%d unexpected convert matchers 2 result:\nact: %#v\nexp: %#v", id, act, test.exp) - continue - } - } -} - -func pattern(nodes ...node) *nodePattern { - return &nodePattern{ - nodeImpl: nodeImpl{ - desc: nodes, - }, - } -} -func anyOf(nodes ...node) *nodeAnyOf { - return &nodeAnyOf{ - nodeImpl: nodeImpl{ - desc: nodes, - }, - } -} -func TestCompiler(t *testing.T) { - for id, test := range []struct { - ast *nodePattern - result Glob - sep []rune - }{ - { - ast: pattern(&nodeText{text: "abc"}), - result: match.NewText("abc"), - }, - { - ast: pattern(&nodeAny{}), - sep: separators, - result: match.NewAny(separators), - }, - { - ast: pattern(&nodeAny{}), - result: match.NewSuper(), - }, - { - ast: pattern(&nodeSuper{}), - result: match.NewSuper(), - }, - { - ast: pattern(&nodeSingle{}), - sep: separators, - result: match.NewSingle(separators), - }, - { - ast: pattern(&nodeRange{ - lo: 'a', - hi: 'z', - not: true, - }), - result: match.NewRange('a', 'z', true), - }, - { - ast: pattern(&nodeList{ - chars: "abc", - not: true, - }), - result: match.NewList([]rune{'a', 'b', 'c'}, true), - }, - { - ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), - sep: separators, - result: match.EveryOf{Matchers: match.Matchers{ - match.NewMin(3), - match.NewContains(string(separators), true), - }}, - }, - { - ast: pattern(&nodeAny{}, &nodeSingle{}, &nodeSingle{}, &nodeSingle{}), - result: match.NewMin(3), - }, - { - ast: pattern(&nodeAny{}, &nodeText{text: "abc"}, &nodeSingle{}), - sep: separators, - result: match.NewBTree( - match.NewRow( - 4, - match.Matchers{ - match.NewText("abc"), - match.NewSingle(separators), - }..., - ), - match.NewAny(separators), - nil, - ), - }, - { - ast: pattern(&nodeText{text: "/"}, anyOf(&nodeText{text: "z"}, &nodeText{text: "ab"}), &nodeSuper{}), - sep: separators, - result: match.NewBTree( - match.NewText("/"), - nil, - match.NewBTree( - match.NewAnyOf(match.NewText("z"), match.NewText("ab")), - nil, - match.NewSuper(), - ), - ), - }, - { - ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSingle{}), - sep: separators, - result: match.NewBTree( - match.NewRow( - 5, - match.Matchers{ - match.NewSingle(separators), - match.NewText("abc"), - match.NewSingle(separators), - }..., - ), - match.NewSuper(), - nil, - ), - }, - { - ast: pattern(&nodeAny{}, &nodeText{text: "abc"}), - result: match.NewSuffix("abc"), - }, - { - ast: pattern(&nodeText{text: "abc"}, &nodeAny{}), - result: match.NewPrefix("abc"), - }, - { - ast: pattern(&nodeText{text: "abc"}, &nodeAny{}, &nodeText{text: "def"}), - result: match.NewPrefixSuffix("abc", "def"), - }, - { - ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), - result: match.NewContains("abc", false), - }, - { - ast: pattern(&nodeAny{}, &nodeAny{}, &nodeAny{}, &nodeText{text: "abc"}, &nodeAny{}, &nodeAny{}), - sep: separators, - result: match.NewBTree( - match.NewText("abc"), - match.NewAny(separators), - match.NewAny(separators), - ), - }, - { - ast: pattern(&nodeSuper{}, &nodeSingle{}, &nodeText{text: "abc"}, &nodeSuper{}, &nodeSingle{}), - result: match.NewBTree( - match.NewText("abc"), - match.NewMin(1), - match.NewMin(1), - ), - }, - { - ast: pattern(anyOf(&nodeText{text: "abc"})), - result: match.NewText("abc"), - }, - { - ast: pattern(anyOf(pattern(anyOf(pattern(&nodeText{text: "abc"}))))), - result: match.NewText("abc"), - }, - { - ast: pattern(anyOf( - pattern( - &nodeText{text: "abc"}, - &nodeSingle{}, - ), - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "def"}, - ), - pattern( - &nodeText{text: "abc"}, - ), - pattern( - &nodeText{text: "abc"}, - ), - )), - result: match.NewBTree( - match.NewText("abc"), - nil, - match.AnyOf{Matchers: match.Matchers{ - match.NewSingle(nil), - match.NewList([]rune{'d', 'e', 'f'}, false), - match.NewNothing(), - }}, - ), - }, - { - ast: pattern( - &nodeRange{lo: 'a', hi: 'z'}, - &nodeRange{lo: 'a', hi: 'x', not: true}, - &nodeAny{}, - ), - result: match.NewBTree( - match.NewRow( - 2, - match.Matchers{ - match.NewRange('a', 'z', false), - match.NewRange('a', 'x', true), - }..., - ), - nil, - match.NewSuper(), - ), - }, - { - ast: pattern(anyOf( - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "abc"}, - &nodeText{text: "ghi"}, - ), - pattern( - &nodeText{text: "abc"}, - &nodeList{chars: "def"}, - &nodeText{text: "ghi"}, - ), - )), - result: match.NewRow( - 7, - match.Matchers{ - match.NewText("abc"), - match.AnyOf{Matchers: match.Matchers{ - match.NewList([]rune{'a', 'b', 'c'}, false), - match.NewList([]rune{'d', 'e', 'f'}, false), - }}, - match.NewText("ghi"), - }..., - ), - }, - } { - m, err := compile(test.ast, test.sep) - if err != nil { - t.Errorf("compilation error: %s", err) - continue - } - - if !reflect.DeepEqual(m, test.result) { - t.Errorf("#%d results are not equal:\nexp: %#v\nact: %#v\nexp:\n%s\nact:\n%s\n", id, test.result, m, debug.Graphviz("", test.result.(match.Matcher)), debug.Graphviz("", m.(match.Matcher))) - continue - } - } -} - -const complexityString = "abcd" - -//func BenchmarkComplexityAny(b *testing.B) { -// m := match.NewAny(nil) -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityContains(b *testing.B) { -// m := match.NewContains() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityList(b *testing.B) { -// m := match.NewList() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityMax(b *testing.B) { -// m := match.NewMax() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityMin(b *testing.B) { -// m := match.NewMin() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityNothing(b *testing.B) { -// m := match.NewNothing() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityPrefix(b *testing.B) { -// m := match.NewPrefix() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityPrefixSuffix(b *testing.B) { -// m := match.NewPrefixSuffix() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityRange(b *testing.B) { -// m := match.NewRange() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityRow(b *testing.B) { -// m := match.NewRow() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexitySingle(b *testing.B) { -// m := match.NewSingle(nil) -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexitySuffix(b *testing.B) { -// m := match.NewSuffix() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexitySuper(b *testing.B) { -// m := match.NewSuper() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityText(b *testing.B) { -// m := match.NewText() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityAnyOf(b *testing.B) { -// m := match.NewAnyOf() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityBTree(b *testing.B) { -// m := match.NewBTree(match.NewText("abc"), match.NewText("d"), match.NewText("e")) -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} -//func BenchmarkComplexityEveryOf(b *testing.B) { -// m := match.NewEveryOf() -// for i := 0; i < b.N; i++ { -// _ = m.Match(complexityString) -// _, _ = m.Index(complexityString) -// } -//} diff --git a/glob.go b/glob.go index 58f45c9..9eb9d58 100644 --- a/glob.go +++ b/glob.go @@ -1,5 +1,10 @@ package glob +import ( + "github.com/gobwas/glob/parser" + "github.com/gobwas/glob/syntax" +) + // Glob represents compiled glob pattern. type Glob interface { Match(string) bool @@ -32,7 +37,7 @@ type Glob interface { // comma-separated (without spaces) patterns // func Compile(pattern string, separators ...rune) (Glob, error) { - ast, err := parse(newLexer(pattern)) + ast, err := syntax.Parse(pattern) if err != nil { return nil, err } @@ -63,7 +68,7 @@ func QuoteMeta(s string) string { // a byte loop is correct because all meta characters are ASCII j := 0 for i := 0; i < len(s); i++ { - if special(s[i]) { + if syntax.Special(s[i]) { b[j] = '\\' j++ } diff --git a/match/any.go b/match/any.go index 1d2d12b..514a9a5 100644 --- a/match/any.go +++ b/match/any.go @@ -2,7 +2,7 @@ package match import ( "fmt" - "github.com/gobwas/glob/strings" + "github.com/gobwas/glob/util/strings" ) type Any struct { diff --git a/match/list.go b/match/list.go index fe0841f..7fd763e 100644 --- a/match/list.go +++ b/match/list.go @@ -2,7 +2,7 @@ package match import ( "fmt" - "github.com/gobwas/glob/runes" + "github.com/gobwas/glob/util/runes" "unicode/utf8" ) diff --git a/match/single.go b/match/single.go index 33e926d..ee6e395 100644 --- a/match/single.go +++ b/match/single.go @@ -2,7 +2,7 @@ package match import ( "fmt" - "github.com/gobwas/glob/runes" + "github.com/gobwas/glob/util/runes" "unicode/utf8" ) diff --git a/parser/ast.go b/parser/ast.go deleted file mode 100644 index 588a45c..0000000 --- a/parser/ast.go +++ /dev/null @@ -1,48 +0,0 @@ -package parser - -type Node interface { - Children() []Node - Parent() Node - append(Node) Node -} - -type node struct { - parent Node - children []Node -} - -func (n *node) Children() []Node { - return n.children -} - -func (n *node) Parent() Node { - return n.parent -} - -func (n *node) append(c Node) Node { - n.children = append(n.children, c) - return c -} - -type ListNode struct { - node - Not bool - Chars string -} - -type RangeNode struct { - node - Not bool - Lo, Hi rune -} - -type TextNode struct { - node - Text string -} - -type PatternNode struct{ node } -type AnyNode struct{ node } -type SuperNode struct{ node } -type SingleNode struct{ node } -type AnyOfNode struct{ node } diff --git a/parser/parser_test.go b/parser/parser_test.go deleted file mode 100644 index 177d7bf..0000000 --- a/parser/parser_test.go +++ /dev/null @@ -1,332 +0,0 @@ -package parser - -import ( - "fmt" - "github.com/gobwas/glob/lexer" - "reflect" - "testing" -) - -type stubLexer struct { - tokens []lexer.Token - pos int -} - -func (s *stubLexer) Next() (ret lexer.Token) { - if s.pos == len(s.tokens) { - return lexer.Token{lexer.EOF, ""} - } - ret = s.tokens[s.pos] - s.pos++ - return -} - -func TestParseString(t *testing.T) { - for id, test := range []struct { - tokens []lexer.Token - tree Node - }{ - { - //pattern: "abc", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "abc"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "abc"}, - }, - }, - }, - }, - { - //pattern: "a*c", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Any, "*"}, - lexer.Token{lexer.Text, "c"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "a"}, - &AnyNode{}, - &TextNode{Text: "c"}, - }, - }, - }, - }, - { - //pattern: "a**c", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Super, "**"}, - lexer.Token{lexer.Text, "c"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "a"}, - &SuperNode{}, - &TextNode{Text: "c"}, - }, - }, - }, - }, - { - //pattern: "a?c", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Single, "?"}, - lexer.Token{lexer.Text, "c"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "a"}, - &SingleNode{}, - &TextNode{Text: "c"}, - }, - }, - }, - }, - { - //pattern: "[!a-z]", - tokens: []lexer.Token{ - lexer.Token{lexer.RangeOpen, "["}, - lexer.Token{lexer.Not, "!"}, - lexer.Token{lexer.RangeLo, "a"}, - lexer.Token{lexer.RangeBetween, "-"}, - lexer.Token{lexer.RangeHi, "z"}, - lexer.Token{lexer.RangeClose, "]"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &RangeNode{Lo: 'a', Hi: 'z', Not: true}, - }, - }, - }, - }, - { - //pattern: "[az]", - tokens: []lexer.Token{ - lexer.Token{lexer.RangeOpen, "["}, - lexer.Token{lexer.Text, "az"}, - lexer.Token{lexer.RangeClose, "]"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &ListNode{Chars: "az"}, - }, - }, - }, - }, - { - //pattern: "{a,z}", - tokens: []lexer.Token{ - lexer.Token{lexer.TermsOpen, "{"}, - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.Text, "z"}, - lexer.Token{lexer.TermsClose, "}"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &AnyOfNode{node: node{children: []Node{ - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "a"}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "z"}, - }}, - }, - }}}, - }, - }, - }, - }, - { - //pattern: "/{z,ab}*", - tokens: []lexer.Token{ - lexer.Token{lexer.Text, "/"}, - lexer.Token{lexer.TermsOpen, "{"}, - lexer.Token{lexer.Text, "z"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.Text, "ab"}, - lexer.Token{lexer.TermsClose, "}"}, - lexer.Token{lexer.Any, "*"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &TextNode{Text: "/"}, - &AnyOfNode{node: node{children: []Node{ - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "z"}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "ab"}, - }}, - }, - }}}, - &AnyNode{}, - }, - }, - }, - }, - { - //pattern: "{a,{x,y},?,[a-z],[!qwe]}", - tokens: []lexer.Token{ - lexer.Token{lexer.TermsOpen, "{"}, - lexer.Token{lexer.Text, "a"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.TermsOpen, "{"}, - lexer.Token{lexer.Text, "x"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.Text, "y"}, - lexer.Token{lexer.TermsClose, "}"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.Single, "?"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.RangeOpen, "["}, - lexer.Token{lexer.RangeLo, "a"}, - lexer.Token{lexer.RangeBetween, "-"}, - lexer.Token{lexer.RangeHi, "z"}, - lexer.Token{lexer.RangeClose, "]"}, - lexer.Token{lexer.Separator, ","}, - lexer.Token{lexer.RangeOpen, "["}, - lexer.Token{lexer.Not, "!"}, - lexer.Token{lexer.Text, "qwe"}, - lexer.Token{lexer.RangeClose, "]"}, - lexer.Token{lexer.TermsClose, "}"}, - lexer.Token{lexer.EOF, ""}, - }, - tree: &PatternNode{ - node: node{ - children: []Node{ - &AnyOfNode{node: node{children: []Node{ - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "a"}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &AnyOfNode{node: node{children: []Node{ - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "x"}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &TextNode{Text: "y"}, - }}, - }, - }}}, - }}, - }, - &PatternNode{ - node: node{children: []Node{ - &SingleNode{}, - }}, - }, - &PatternNode{ - node: node{ - children: []Node{ - &RangeNode{Lo: 'a', Hi: 'z', Not: false}, - }, - }, - }, - &PatternNode{ - node: node{ - children: []Node{ - &ListNode{Chars: "qwe", Not: true}, - }, - }, - }, - }}}, - }, - }, - }, - }, - } { - lexer := &stubLexer{tokens: test.tokens} - result, err := Parse(lexer) - if err != nil { - t.Errorf("[%d] unexpected error: %s", id, err) - } - if !reflect.DeepEqual(test.tree, result) { - t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree) - } - } -} - -const abstractNodeImpl = "nodeImpl" - -func nodeEqual(a, b Node) error { - if (a == nil || b == nil) && a != b { - return fmt.Errorf("nodes are not equal: exp %s, act %s", a, b) - } - - aValue, bValue := reflect.Indirect(reflect.ValueOf(a)), reflect.Indirect(reflect.ValueOf(b)) - aType, bType := aValue.Type(), bValue.Type() - if aType != bType { - return fmt.Errorf("nodes are not equal: exp %s, act %s", aValue.Type(), bValue.Type()) - } - - for i := 0; i < aType.NumField(); i++ { - var eq bool - - f := aType.Field(i).Name - if f == abstractNodeImpl { - continue - } - - af, bf := aValue.FieldByName(f), bValue.FieldByName(f) - - switch af.Kind() { - case reflect.String: - eq = af.String() == bf.String() - case reflect.Bool: - eq = af.Bool() == bf.Bool() - default: - eq = fmt.Sprint(af) == fmt.Sprint(bf) - } - - if !eq { - return fmt.Errorf("nodes<%s> %q fields are not equal: exp %q, act %q", aType, f, af, bf) - } - } - - for i, aDesc := range a.Children() { - if len(b.Children())-1 < i { - return fmt.Errorf("node does not have enough children (got %d children, wanted %d-th token)", len(b.Children()), i) - } - - bDesc := b.Children()[i] - - if err := nodeEqual(aDesc, bDesc); err != nil { - return err - } - } - - return nil -} diff --git a/syntax/ast/ast.go b/syntax/ast/ast.go new file mode 100644 index 0000000..3e499bc --- /dev/null +++ b/syntax/ast/ast.go @@ -0,0 +1,85 @@ +package ast + +type Visitor interface { + Visit(*Node) Visitor +} + +func Walk(v Visitor, n *Node) { + if v = v.Visit(n); v == nil { + return + } + for _, c := range n.Children { + Walk(v, c) + } +} + +type Node struct { + Parent *Node + Children []*Node + Value interface{} + Kind Kind +} + +func NewNode(k Kind, v interface{}, ch ...*Node) *Node { + n := &Node{ + Kind: k, + Value: v, + } + for _, c := range ch { + Insert(n, c) + } + return n +} + +func (a *Node) Equal(b *Node) bool { + if a.Kind != b.Kind { + return false + } + if a.Value != b.Value { + return false + } + if len(a.Children) != len(b.Children) { + return false + } + for i, c := range a.Children { + if !c.Equal(b.Children[i]) { + return false + } + } + return true +} + +func Insert(parent *Node, children ...*Node) { + parent.Children = append(parent.Children, children...) + for _, ch := range children { + ch.Parent = parent + } +} + +type List struct { + Not bool + Chars string +} + +type Range struct { + Not bool + Lo, Hi rune +} + +type Text struct { + Text string +} + +type Kind int + +const ( + KindNothing Kind = iota + KindPattern + KindList + KindRange + KindText + KindAny + KindSuper + KindSingle + KindAnyOf +) diff --git a/parser/parser.go b/syntax/ast/parser.go similarity index 66% rename from parser/parser.go rename to syntax/ast/parser.go index 6adea39..93caa33 100644 --- a/parser/parser.go +++ b/syntax/ast/parser.go @@ -1,9 +1,9 @@ -package parser +package ast import ( "errors" "fmt" - "github.com/gobwas/glob/lexer" + "github.com/gobwas/glob/syntax/lexer" "unicode/utf8" ) @@ -11,15 +11,15 @@ type Lexer interface { Next() lexer.Token } -type parseFn func(Node, Lexer) (parseFn, Node, error) +type parseFn func(*Node, Lexer) (parseFn, *Node, error) -func Parse(lexer Lexer) (*PatternNode, error) { +func Parse(lexer Lexer) (*Node, error) { var parser parseFn - root := &PatternNode{} + root := NewNode(KindPattern, nil) var ( - tree Node + tree *Node err error ) for parser, tree = parserMain, root; parser != nil; { @@ -32,7 +32,7 @@ func Parse(lexer Lexer) (*PatternNode, error) { return root, nil } -func parserMain(tree Node, lex Lexer) (parseFn, Node, error) { +func parserMain(tree *Node, lex Lexer) (parseFn, *Node, error) { for { token := lex.Next() switch token.Type { @@ -43,28 +43,41 @@ func parserMain(tree Node, lex Lexer) (parseFn, Node, error) { return nil, tree, errors.New(token.Raw) case lexer.Text: - return parserMain, tree.append(&TextNode{Text: token.Raw}), nil + Insert(tree, NewNode(KindText, &Text{token.Raw})) + return parserMain, tree, nil case lexer.Any: - return parserMain, tree.append(&AnyNode{}), nil + Insert(tree, NewNode(KindAny, nil)) + return parserMain, tree, nil case lexer.Super: - return parserMain, tree.append(&SuperNode{}), nil + Insert(tree, NewNode(KindSuper, nil)) + return parserMain, tree, nil case lexer.Single: - return parserMain, tree.append(&SingleNode{}), nil + Insert(tree, NewNode(KindSingle, nil)) + return parserMain, tree, nil case lexer.RangeOpen: return parserRange, tree, nil case lexer.TermsOpen: - return parserMain, tree.append(&AnyOfNode{}).append(&PatternNode{}), nil + a := NewNode(KindAnyOf, nil) + Insert(tree, a) + + p := NewNode(KindPattern, nil) + Insert(a, p) + + return parserMain, p, nil case lexer.Separator: - return parserMain, tree.Parent().append(&PatternNode{}), nil + p := NewNode(KindPattern, nil) + Insert(tree.Parent, p) + + return parserMain, p, nil case lexer.TermsClose: - return parserMain, tree.Parent().Parent(), nil + return parserMain, tree.Parent.Parent, nil default: return nil, tree, fmt.Errorf("unexpected token: %s", token) @@ -73,7 +86,7 @@ func parserMain(tree Node, lex Lexer) (parseFn, Node, error) { return nil, tree, fmt.Errorf("unknown error") } -func parserRange(tree Node, lex Lexer) (parseFn, Node, error) { +func parserRange(tree *Node, lex Lexer) (parseFn, *Node, error) { var ( not bool lo rune @@ -126,16 +139,16 @@ func parserRange(tree Node, lex Lexer) (parseFn, Node, error) { } if isRange { - tree = tree.append(&RangeNode{ + Insert(tree, NewNode(KindRange, &Range{ Lo: lo, Hi: hi, Not: not, - }) + })) } else { - tree = tree.append(&ListNode{ + Insert(tree, NewNode(KindList, &List{ Chars: chars, Not: not, - }) + })) } return parserMain, tree, nil diff --git a/syntax/ast/parser_test.go b/syntax/ast/parser_test.go new file mode 100644 index 0000000..84e8d45 --- /dev/null +++ b/syntax/ast/parser_test.go @@ -0,0 +1,256 @@ +package ast + +import ( + "github.com/gobwas/glob/syntax" + "reflect" + "testing" +) + +type stubLexer struct { + tokens []syntax.Token + pos int +} + +func (s *stubLexer) Next() (ret syntax.Token) { + if s.pos == len(s.tokens) { + return syntax.Token{syntax.EOF, ""} + } + ret = s.tokens[s.pos] + s.pos++ + return +} + +func TestParseString(t *testing.T) { + for id, test := range []struct { + tokens []syntax.Token + tree Node + }{ + { + //pattern: "abc", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "abc"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "abc"}), + ), + }, + { + //pattern: "a*c", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Any, "*"}, + syntax.Token{syntax.Text, "c"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + NewNode(KindAny, nil), + NewNode(KindText, &Text{Text: "c"}), + ), + }, + { + //pattern: "a**c", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Super, "**"}, + syntax.Token{syntax.Text, "c"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + NewNode(KindSuper, nil), + NewNode(KindText, &Text{Text: "c"}), + ), + }, + { + //pattern: "a?c", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Single, "?"}, + syntax.Token{syntax.Text, "c"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + NewNode(KindSingle, nil), + NewNode(KindText, &Text{Text: "c"}), + ), + }, + { + //pattern: "[!a-z]", + tokens: []syntax.Token{ + syntax.Token{syntax.RangeOpen, "["}, + syntax.Token{syntax.Not, "!"}, + syntax.Token{syntax.RangeLo, "a"}, + syntax.Token{syntax.RangeBetween, "-"}, + syntax.Token{syntax.RangeHi, "z"}, + syntax.Token{syntax.RangeClose, "]"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindRange, &Range{Lo: 'a', Hi: 'z', Not: true}), + ), + }, + { + //pattern: "[az]", + tokens: []syntax.Token{ + syntax.Token{syntax.RangeOpen, "["}, + syntax.Token{syntax.Text, "az"}, + syntax.Token{syntax.RangeClose, "]"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindList, &List{Chars: "az"}), + ), + }, + { + //pattern: "{a,z}", + tokens: []syntax.Token{ + syntax.Token{syntax.TermsOpen, "{"}, + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.Text, "z"}, + syntax.Token{syntax.TermsClose, "}"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindAnyOf, nil, + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + ), + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "z"}), + ), + ), + ), + }, + { + //pattern: "/{z,ab}*", + tokens: []syntax.Token{ + syntax.Token{syntax.Text, "/"}, + syntax.Token{syntax.TermsOpen, "{"}, + syntax.Token{syntax.Text, "z"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.Text, "ab"}, + syntax.Token{syntax.TermsClose, "}"}, + syntax.Token{syntax.Any, "*"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "/"}), + NewNode(KindAnyOf, nil, + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "z"}), + ), + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "ab"}), + ), + ), + NewNode(KindAny, nil), + ), + }, + { + //pattern: "{a,{x,y},?,[a-z],[!qwe]}", + tokens: []syntax.Token{ + syntax.Token{syntax.TermsOpen, "{"}, + syntax.Token{syntax.Text, "a"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.TermsOpen, "{"}, + syntax.Token{syntax.Text, "x"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.Text, "y"}, + syntax.Token{syntax.TermsClose, "}"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.Single, "?"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.RangeOpen, "["}, + syntax.Token{syntax.RangeLo, "a"}, + syntax.Token{syntax.RangeBetween, "-"}, + syntax.Token{syntax.RangeHi, "z"}, + syntax.Token{syntax.RangeClose, "]"}, + syntax.Token{syntax.Separator, ","}, + syntax.Token{syntax.RangeOpen, "["}, + syntax.Token{syntax.Not, "!"}, + syntax.Token{syntax.Text, "qwe"}, + syntax.Token{syntax.RangeClose, "]"}, + syntax.Token{syntax.TermsClose, "}"}, + syntax.Token{syntax.EOF, ""}, + }, + tree: NewNode(KindPattern, nil, + NewNode(KindAnyOf, nil, + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "a"}), + ), + NewNode(KindPattern, nil, + NewNode(KindAnyOf, nil, + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "x"}), + ), + NewNode(KindPattern, nil, + NewNode(KindText, &Text{Text: "y"}), + ), + ), + ), + NewNode(KindPattern, nil, + NewNode(KindSingle, nil), + ), + NewNode(KindPattern, nil, + NewNode(KindRange, &Range{Lo: 'a', Hi: 'z', Not: false}), + ), + NewNode(KindPattern, nil, + NewNode(KindList, &List{Chars: "qwe", Not: true}), + ), + ), + ), + }, + } { + lexer := &stubLexer{tokens: test.tokens} + result, err := Parse(lexer) + if err != nil { + t.Errorf("[%d] unexpected error: %s", id, err) + } + if !reflect.DeepEqual(test.tree, result) { + t.Errorf("[%d] Parse():\nact:\t%s\nexp:\t%s\n", id, result, test.tree) + } + } +} + +type kv struct { + kind Kind + value interface{} +} + +type visitor struct { + visited []kv +} + +func (v *visitor) Visit(n Node) Visitor { + v.visited = append(v.visited, kv{n.Kind(), n.Value()}) + return v +} + +func TestWalkTree(t *testing.T) { + + for i, test := range []struct { + tree *Node + visited []kv + }{ + { + tree: NewNode(KindPattern, nil, + NewNode(KindSingle, nil), + ), + visited: []kv{ + kv{KindPattern, nil}, + kv{KindSingle, nil}, + }, + }, + } { + v := &visitor{} + Walk(v, test.tree) + + if !reflect.DeepEqual(test.visited, v.visited) { + t.Errorf("[%d] unexpected result of Walk():\nvisited:\t%v\nwant:\t\t%v", i, v.visited, test.visited) + } + } +} diff --git a/lexer/lexer.go b/syntax/lexer/lexer.go similarity index 97% rename from lexer/lexer.go rename to syntax/lexer/lexer.go index e074b09..a1c8d19 100644 --- a/lexer/lexer.go +++ b/syntax/lexer/lexer.go @@ -66,7 +66,7 @@ type lexer struct { hasRune bool } -func newLexer(source string) *lexer { +func NewLexer(source string) *lexer { l := &lexer{ data: source, tokens: tokens(make([]Token, 0, 4)), @@ -74,6 +74,18 @@ func newLexer(source string) *lexer { return l } +func (l *lexer) Next() Token { + if l.err != nil { + return Token{Error, l.err.Error()} + } + if !l.tokens.empty() { + return l.tokens.shift() + } + + l.fetchItem() + return l.Next() +} + func (l *lexer) peek() (r rune, w int) { if l.pos == len(l.data) { return eof, 0 @@ -134,18 +146,6 @@ func (l *lexer) termsLeave() { l.termsLevel-- } -func (l *lexer) nextItem() Token { - if l.err != nil { - return Token{Error, l.err.Error()} - } - if !l.tokens.empty() { - return l.tokens.shift() - } - - l.fetchItem() - return l.nextItem() -} - var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open} var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma) diff --git a/lexer/lexer_test.go b/syntax/lexer/lexer_test.go similarity index 98% rename from lexer/lexer_test.go rename to syntax/lexer/lexer_test.go index 8af3aa6..26c8983 100644 --- a/lexer/lexer_test.go +++ b/syntax/lexer/lexer_test.go @@ -178,9 +178,9 @@ func TestLexGood(t *testing.T) { }, }, } { - lexer := newLexer(test.pattern) + lexer := NewLexer(test.pattern) for i, exp := range test.items { - act := lexer.nextItem() + act := lexer.Next() if act.Type != exp.Type { t.Errorf("#%d %q: wrong %d-th item type: exp: %q; act: %q\n\t(%s vs %s)", id, test.pattern, i, exp.Type, act.Type, exp, act) } diff --git a/lexer/token.go b/syntax/lexer/token.go similarity index 100% rename from lexer/token.go rename to syntax/lexer/token.go diff --git a/syntax/syntax.go b/syntax/syntax.go new file mode 100644 index 0000000..1d168b1 --- /dev/null +++ b/syntax/syntax.go @@ -0,0 +1,14 @@ +package syntax + +import ( + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/syntax/lexer" +) + +func Parse(s string) (*ast.Node, error) { + return ast.Parse(lexer.NewLexer(s)) +} + +func Special(b byte) bool { + return lexer.Special(b) +}