From b4d1c838c5ba2a8befbb929c55a70c738ed6e7c0 Mon Sep 17 00:00:00 2001
From: Ben Boyter <ben@boyter.org>
Date: Mon, 30 Sep 2024 16:08:03 +1000
Subject: [PATCH] More options to control goroutines

---
 SCC-OUTPUT-REPORT.html | 90 ++++++++++++++++--------------------------
 main.go                | 25 ++++++++++++
 processor/processor.go |  4 ++
 3 files changed, 64 insertions(+), 55 deletions(-)

diff --git a/SCC-OUTPUT-REPORT.html b/SCC-OUTPUT-REPORT.html
index 333e18d57..ab781d0fe 100644
--- a/SCC-OUTPUT-REPORT.html
+++ b/SCC-OUTPUT-REPORT.html
@@ -12,14 +12,14 @@
 	</tr></thead>
 	<tbody><tr>
 		<th>Go</th>
-		<th>30</th>
-		<th>9637</th>
-		<th>1482</th>
-		<th>469</th>
-		<th>7686</th>
-		<th>1536</th>
-		<th>256898</th>
-		<th>4098</th>
+		<th>28</th>
+		<th>9570</th>
+		<th>1462</th>
+		<th>452</th>
+		<th>7656</th>
+		<th>1528</th>
+		<th>255227</th>
+		<th>4068</th>
 	</tr><tr>
 		<td>processor/formatters.go</td>
 		<td></td>
@@ -63,23 +63,23 @@
 	</tr><tr>
 		<td>processor/processor.go</td>
 		<td></td>
-		<td>671</td>
+		<td>672</td>
 		<td>141</td>
 		<td>104</td>
-		<td>426</td>
+		<td>427</td>
 		<td>92</td>
-	    <td>19411</td>
-		<td>438</td>
+	    <td>19457</td>
+		<td>439</td>
 	</tr><tr>
 		<td>main.go</td>
 		<td></td>
-		<td>404</td>
+		<td>429</td>
 		<td>10</td>
 		<td>6</td>
-		<td>388</td>
+		<td>413</td>
 		<td>10</td>
-	    <td>8969</td>
-		<td>256</td>
+	    <td>9693</td>
+		<td>271</td>
 	</tr><tr>
 		<td>processor/detector_test.go</td>
 		<td></td>
@@ -220,16 +220,6 @@
 		<td>19</td>
 	    <td>2043</td>
 		<td>60</td>
-	</tr><tr>
-		<td>processor/processor_unix.go</td>
-		<td></td>
-		<td>69</td>
-		<td>14</td>
-		<td>14</td>
-		<td>41</td>
-		<td>8</td>
-	    <td>2030</td>
-		<td>52</td>
 	</tr><tr>
 		<td>processor/filereader.go</td>
 		<td></td>
@@ -250,16 +240,6 @@
 		<td>0</td>
 	    <td>2209</td>
 		<td>35</td>
-	</tr><tr>
-		<td>processor/bloom.go</td>
-		<td></td>
-		<td>37</td>
-		<td>7</td>
-		<td>12</td>
-		<td>18</td>
-		<td>2</td>
-	    <td>1062</td>
-		<td>29</td>
 	</tr><tr>
 		<td>processor/cocomo_test.go</td>
 		<td></td>
@@ -270,6 +250,16 @@
 		<td>6</td>
 	    <td>686</td>
 		<td>23</td>
+	</tr><tr>
+		<td>processor/bloom.go</td>
+		<td></td>
+		<td>37</td>
+		<td>7</td>
+		<td>12</td>
+		<td>18</td>
+		<td>2</td>
+	    <td>1062</td>
+		<td>29</td>
 	</tr><tr>
 		<td>processor/helpers_test.go</td>
 		<td></td>
@@ -280,16 +270,6 @@
 		<td>4</td>
 	    <td>434</td>
 		<td>18</td>
-	</tr><tr>
-		<td>processor/processor_unix_test.go</td>
-		<td></td>
-		<td>24</td>
-		<td>6</td>
-		<td>3</td>
-		<td>15</td>
-		<td>0</td>
-	    <td>411</td>
-		<td>16</td>
 	</tr><tr>
 		<td>examples/language/go.go</td>
 		<td></td>
@@ -323,16 +303,16 @@
 	</tr></tbody>
 	<tfoot><tr>
 		<th>Total</th>
-		<th>30</th>
-		<th>9637</th>
-		<th>1482</th>
-		<th>469</th>
-		<th>7686</th>
-		<th>1536</th>
-    	<th>256898</th>
-		<th>4098</th>
+		<th>28</th>
+		<th>9570</th>
+		<th>1462</th>
+		<th>452</th>
+		<th>7656</th>
+		<th>1528</th>
+    	<th>255227</th>
+		<th>4068</th>
 	</tr>
 	<tr>
-		<th colspan="9">Estimated Cost to Develop (organic) $229,922<br>Estimated Schedule Effort (organic) 7.87 months<br>Estimated People Required (organic) 2.60<br></th>
+		<th colspan="9">Estimated Cost to Develop (organic) $228,979<br>Estimated Schedule Effort (organic) 7.85 months<br>Estimated People Required (organic) 2.59<br></th>
 	</tr></tfoot>
 	</table></body></html>
\ No newline at end of file
diff --git a/main.go b/main.go
index b2898a9c4..2cf5ea34b 100644
--- a/main.go
+++ b/main.go
@@ -5,6 +5,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"runtime"
 	"strings"
 
 	"github.com/boyter/scc/v3/processor"
@@ -140,6 +141,30 @@ func main() {
 		10000,
 		"number of files to parse before turning the GC on",
 	)
+	flags.IntVar(
+		&processor.FileListQueueSize,
+		"file-list-queue-size",
+		runtime.NumCPU(),
+		"the size of the queue of files found and ready to be read into memory",
+	)
+	flags.IntVar(
+		&processor.FileProcessJobWorkers,
+		"file-process-job-workers",
+		runtime.NumCPU(),
+		"number of goroutine workers that process files collecting stats",
+	)
+	flags.IntVar(
+		&processor.FileSummaryJobQueueSize,
+		"file-summary-job-queue-size",
+		runtime.NumCPU(),
+		"the size of the queue used to hold processed file statistics before formatting",
+	)
+	flags.IntVar(
+		&processor.DirectoryWalkerJobWorkers,
+		"directory-walker-job-workers",
+		8,
+		"controls the maximum number of workers which will walk the directory tree",
+	)
 	flags.StringVarP(
 		&processor.Format,
 		"format",
diff --git a/processor/processor.go b/processor/processor.go
index b674212dd..47f48da3f 100644
--- a/processor/processor.go
+++ b/processor/processor.go
@@ -165,6 +165,9 @@ var FileProcessJobWorkers = runtime.NumCPU() * 4
 // FileSummaryJobQueueSize is the queue used to hold processed file statistics before formatting
 var FileSummaryJobQueueSize = runtime.NumCPU()
 
+// DirectoryWalkerJobWorkers is the number of workers which will walk the directory tree
+var DirectoryWalkerJobWorkers = 8
+
 // AllowListExtensions is a list of extensions which are allowed to be processed
 var AllowListExtensions = []string{}
 
@@ -609,6 +612,7 @@ func Process() {
 	fileWalker.IgnoreGitModules = GitModuleIgnore
 	fileWalker.IncludeHidden = true
 	fileWalker.ExcludeDirectory = PathDenyList
+	fileWalker.SetConcurrency(DirectoryWalkerJobWorkers)
 
 	for _, exclude := range Exclude {
 		regexpResult, err := regexp.Compile(exclude)