WIP: Add support for git-style .gitignore patterns to zoekt-index.

cmd/zoekt-index/main.go

@@ -15,6 +15,8 @@
 package main
 
 import (
+	"bytes"
+	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -27,6 +29,7 @@ import (
 	"github.com/google/zoekt"
 	"github.com/google/zoekt/build"
 	"github.com/google/zoekt/cmd"
+	"github.com/google/zoekt/ignore"
 	"go.uber.org/automaxprocs/maxprocs"
 )
 
@@ -36,11 +39,15 @@ type fileInfo struct {
 }
 
 type fileAggregator struct {
-	ignoreDirs map[string]struct{}
-	sizeMax    int64
-	sink       chan fileInfo
+	ignoreDirs     map[string]struct{}
+	ignoreMatchers []*ignore.Matcher
+	sizeMax        int64
+	baseDir        string
+	sink           chan fileInfo
 }
 
+var SkipFile error = errors.New("Skipped this file")
+
 func (a *fileAggregator) add(path string, info os.FileInfo, err error) error {
 	if err != nil {
 		return err
@@ -51,6 +58,13 @@ func (a *fileAggregator) add(path string, info os.FileInfo, err error) error {
 		if _, ok := a.ignoreDirs[base]; ok {
 			return filepath.SkipDir
 		}
+		skip, err := shouldSkip(a.baseDir, path, a.ignoreMatchers)
+		if err != nil {
+			return err
+		}
+		if skip {
+			return filepath.SkipDir
+		}
 	}
 
 	if info.Mode().IsRegular() {
@@ -62,6 +76,8 @@ func (a *fileAggregator) add(path string, info os.FileInfo, err error) error {
 func main() {
 	cpuProfile := flag.String("cpu_profile", "", "write cpu profile to file")
 	ignoreDirs := flag.String("ignore_dirs", ".git,.hg,.svn", "comma separated list of directories to ignore.")
+	ignoreFiles := flag.String("ignore_files", ".gitignore,.ignore", "comma separated list of files containing "+
+		"git-style ignore patterns. Paths matching the patterns in these files will not be indexed")
 	flag.Parse()
 
 	// Tune GOMAXPROCS to match Linux container CPU quota.
@@ -89,15 +105,33 @@ func main() {
 			}
 		}
 	}
+	var ignoreMatchers []*ignore.Matcher
+	if *ignoreFiles != "" {
+		files := strings.Split(*ignoreFiles, ",")
+		for _, f := range files {
+			if _, err := os.Stat(f); errors.Is(err, os.ErrNotExist) {
+				continue
+			}
+			content, err := ioutil.ReadFile(f)
+			if err != nil {
+				continue
+			}
+			m, err := ignore.ParseIgnoreFile(bytes.NewReader(content))
+			if err != nil {
+				continue
+			}
+			ignoreMatchers = append(ignoreMatchers, m)
+		}
+	}
 	for _, arg := range flag.Args() {
 		opts.RepositoryDescription.Source = arg
-		if err := indexArg(arg, *opts, ignoreDirMap); err != nil {
+		if err := indexArg(arg, *opts, ignoreDirMap, ignoreMatchers); err != nil {
 			log.Fatal(err)
 		}
 	}
 }
 
-func indexArg(arg string, opts build.Options, ignore map[string]struct{}) error {
+func indexArg(arg string, opts build.Options, ignore map[string]struct{}, ignoreMatchers []*ignore.Matcher) error {
 	dir, err := filepath.Abs(filepath.Clean(arg))
 	if err != nil {
 		return err
@@ -114,9 +148,11 @@ func indexArg(arg string, opts build.Options, ignore map[string]struct{}) error
 
 	comm := make(chan fileInfo, 100)
 	agg := fileAggregator{
-		ignoreDirs: ignore,
-		sink:       comm,
-		sizeMax:    int64(opts.SizeMax),
+		ignoreDirs:     ignore,
+		ignoreMatchers: ignoreMatchers,
+		baseDir:        dir,
+		sink:           comm,
+		sizeMax:        int64(opts.SizeMax),
 	}
 
 	go func() {
@@ -127,6 +163,15 @@ func indexArg(arg string, opts build.Options, ignore map[string]struct{}) error
 	}()
 
 	for f := range comm {
+		skip, err := shouldSkip(dir, f.name, ignoreMatchers)
+		if err != nil {
+			return err
+		}
+		if skip {
+			log.Printf("Skipping %v", f.name)
+			continue
+		}
+
 		displayName := strings.TrimPrefix(f.name, dir+"/")
 		if f.size > int64(opts.SizeMax) && !opts.IgnoreSizeMax(displayName) {
 			if err := builder.Add(zoekt.Document{
@@ -149,3 +194,16 @@ func indexArg(arg string, opts build.Options, ignore map[string]struct{}) error
 
 	return builder.Finish()
 }
+
+func shouldSkip(baseDir, path string, ignoreMatchers []*ignore.Matcher) (bool, error) {
+	relPath, err := filepath.Rel(baseDir, path)
+	if err != nil {
+		return false, err
+	}
+	for _, m := range ignoreMatchers {
+		if m.Match(relPath) {
+			return true, nil
+		}
+	}
+	return false, nil
+}