Skip to content

Commit

Permalink
Remove status printing from the hot path, by using a ticker
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 698017422
  • Loading branch information
vpasdf authored and copybara-github committed Nov 19, 2024
1 parent c53a552 commit cbfa908
Showing 1 changed file with 22 additions and 6 deletions.
28 changes: 22 additions & 6 deletions extractor/filesystem/extractor.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,23 @@ func RunFS(ctx context.Context, config *Config, wc *walkContext) ([]*extractor.I
if len(wc.filesToExtract) > 0 {
err = walkIndividualFiles(wc.fs, wc.filesToExtract, wc.handleFile)
} else {
ticker := time.NewTicker(2 * time.Second)
quit := make(chan struct{})
go func() {
for {
select {
case <-ticker.C:
wc.printStatus()
case <-quit:
ticker.Stop()
return
}
}
}()

err = internal.WalkDirUnsorted(wc.fs, ".", wc.handleFile)

close(quit)
}

log.Infof("End status: %d dirs visited, %d inodes visited, %d Extract calls, %s elapsed",
Expand Down Expand Up @@ -235,6 +251,8 @@ type walkContext struct {
lastInodes int
extractCalls int
lastExtracts int

currentPath string
}

func walkIndividualFiles(fsys scalibrfs.FS, paths []string, fn fs.WalkDirFunc) error {
Expand All @@ -253,7 +271,7 @@ func walkIndividualFiles(fsys scalibrfs.FS, paths []string, fn fs.WalkDirFunc) e
}

func (wc *walkContext) handleFile(path string, d fs.DirEntry, fserr error) error {
wc.printStatus(path)
wc.currentPath = path

wc.inodesVisited++
if wc.maxInodes > 0 && wc.inodesVisited > wc.maxInodes {
Expand Down Expand Up @@ -456,14 +474,12 @@ func errToExtractorStatus(extractors []Extractor, foundInv map[string]bool, erro
return result
}

func (wc *walkContext) printStatus(path string) {
if time.Since(wc.lastStatus) < 2*time.Second {
return
}
func (wc *walkContext) printStatus() {
log.Infof("Status: new inodes: %d, %.1f inodes/s, new extract calls: %d, path: %q\n",
wc.inodesVisited-wc.lastInodes,
float64(wc.inodesVisited-wc.lastInodes)/time.Since(wc.lastStatus).Seconds(),
wc.extractCalls-wc.lastExtracts, path)
wc.extractCalls-wc.lastExtracts, wc.currentPath)

wc.lastStatus = time.Now()
wc.lastInodes = wc.inodesVisited
wc.lastExtracts = wc.extractCalls
Expand Down

0 comments on commit cbfa908

Please sign in to comment.