Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support --exclude-content #509

Closed
wants to merge 1 commit into from
Closed

feat: support --exclude-content #509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 72 additions & 62 deletions SCC-OUTPUT-REPORT.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
</tr></thead>
<tbody><tr>
<th>Go</th>
<th>30</th>
<th>9515</th>
<th>1460</th>
<th>456</th>
<th>7599</th>
<th>1516</th>
<th>254099</th>
<th>4050</th>
<th>31</th>
<th>9667</th>
<th>1488</th>
<th>470</th>
<th>7709</th>
<th>1540</th>
<th>257316</th>
<th>4107</th>
</tr><tr>
<td>processor/formatters.go</td>
<td></td>
Expand Down Expand Up @@ -53,33 +53,33 @@
</tr><tr>
<td>processor/workers.go</td>
<td></td>
<td>870</td>
<td>128</td>
<td>892</td>
<td>131</td>
<td>91</td>
<td>651</td>
<td>225</td>
<td>25495</td>
<td>494</td>
<td>670</td>
<td>231</td>
<td>25897</td>
<td>503</td>
</tr><tr>
<td>processor/processor.go</td>
<td></td>
<td>667</td>
<td>140</td>
<td>103</td>
<td>424</td>
<td>92</td>
<td>19295</td>
<td>435</td>
<td>680</td>
<td>142</td>
<td>104</td>
<td>434</td>
<td>96</td>
<td>19711</td>
<td>443</td>
</tr><tr>
<td>main.go</td>
<td></td>
<td>398</td>
<td>404</td>
<td>10</td>
<td>6</td>
<td>382</td>
<td>388</td>
<td>10</td>
<td>8856</td>
<td>253</td>
<td>9010</td>
<td>256</td>
</tr><tr>
<td>processor/detector_test.go</td>
<td></td>
Expand All @@ -98,7 +98,7 @@
<td>14</td>
<td>269</td>
<td>47</td>
<td>7991</td>
<td>7998</td>
<td>202</td>
</tr><tr>
<td>processor/workers_tokei_test.go</td>
Expand Down Expand Up @@ -160,6 +160,16 @@
<td>50</td>
<td>3766</td>
<td>99</td>
</tr><tr>
<td>cmd/badges/simplecache.go</td>
<td></td>
<td>161</td>
<td>28</td>
<td>13</td>
<td>120</td>
<td>20</td>
<td>3070</td>
<td>94</td>
</tr><tr>
<td>processor/processor_test.go</td>
<td></td>
Expand All @@ -181,15 +191,15 @@
<td>1911</td>
<td>60</td>
</tr><tr>
<td>cmd/badges/simplecache.go</td>
<td>cmd/badges/simplecache_test.go</td>
<td></td>
<td>109</td>
<td>102</td>
<td>21</td>
<td>3</td>
<td>78</td>
<td>17</td>
<td>4</td>
<td>88</td>
<td>14</td>
<td>1931</td>
<td>75</td>
<td>2024</td>
<td>52</td>
</tr><tr>
<td>processor/structs_test.go</td>
<td></td>
Expand Down Expand Up @@ -230,16 +240,6 @@
<td>6</td>
<td>1316</td>
<td>37</td>
</tr><tr>
<td>cmd/badges/simplecache_test.go</td>
<td></td>
<td>52</td>
<td>12</td>
<td>0</td>
<td>40</td>
<td>9</td>
<td>1041</td>
<td>30</td>
</tr><tr>
<td>processor/cocomo.go</td>
<td></td>
Expand All @@ -250,16 +250,6 @@
<td>0</td>
<td>2209</td>
<td>35</td>
</tr><tr>
<td>processor/bloom.go</td>
<td></td>
<td>37</td>
<td>7</td>
<td>12</td>
<td>18</td>
<td>2</td>
<td>1062</td>
<td>29</td>
</tr><tr>
<td>processor/cocomo_test.go</td>
<td></td>
Expand All @@ -270,6 +260,16 @@
<td>6</td>
<td>686</td>
<td>23</td>
</tr><tr>
<td>processor/bloom.go</td>
<td></td>
<td>37</td>
<td>7</td>
<td>12</td>
<td>18</td>
<td>2</td>
<td>1062</td>
<td>29</td>
</tr><tr>
<td>processor/helpers_test.go</td>
<td></td>
Expand Down Expand Up @@ -310,6 +310,16 @@
<td>0</td>
<td>378</td>
<td>14</td>
</tr><tr>
<td>examples/issue507/exclude.go</td>
<td></td>
<td>9</td>
<td>3</td>
<td>1</td>
<td>5</td>
<td>0</td>
<td>116</td>
<td>7</td>
</tr><tr>
<td>processor/constants.go</td>
<td></td>
Expand All @@ -323,16 +333,16 @@
</tr></tbody>
<tfoot><tr>
<th>Total</th>
<th>30</th>
<th>9515</th>
<th>1460</th>
<th>456</th>
<th>7599</th>
<th>1516</th>
<th>254099</th>
<th>4050</th>
<th>31</th>
<th>9667</th>
<th>1488</th>
<th>470</th>
<th>7709</th>
<th>1540</th>
<th>257316</th>
<th>4107</th>
</tr>
<tr>
<th colspan="9">Estimated Cost to Develop (organic) $227,190<br>Estimated Schedule Effort (organic) 7.83 months<br>Estimated People Required (organic) 2.58<br></th>
<th colspan="9">Estimated Cost to Develop (organic) $230,644<br>Estimated Schedule Effort (organic) 7.88 months<br>Estimated People Required (organic) 2.60<br></th>
</tr></tfoot>
</table></body></html>
9 changes: 9 additions & 0 deletions examples/issue507/exclude.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package main

import "fmt"

func main() {
fmt.Println("Hello, world!")
}

// exclude-content testing, DO NOT EDIT.
3 changes: 3 additions & 0 deletions examples/issue507/exclude.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# exclude-content testing, DO NOT EDIT.
def test(s: str) -> bool:
return s == 'exclude'
3 changes: 3 additions & 0 deletions examples/issue507/include.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# exclude-content testing, this file should be included
if __name__ == '__main__':
print('Hello, world!')
6 changes: 6 additions & 0 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,12 @@ func main() {
"$",
"set currency symbol",
)
flags.StringVar(
&processor.ExcludeContent,
"exclude-content",
"",
"exclude files containing text that matches the given regular expression",
)

if err := rootCmd.Execute(); err != nil {
os.Exit(1)
Expand Down
13 changes: 13 additions & 0 deletions processor/processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,10 @@ var ExcludeListExtensions = []string{}
// ExcludeFilename is a list of filenames which should be ignored
var ExcludeFilename = []string{}

// ExcludeContent is a regular expression which is used to exclude files containing text that matches it
var ExcludeContent string
var excludeContentPattern *regexp.Regexp

// AverageWage is the average wage in dollars used for the COCOMO cost estimate
var AverageWage int64 = 56286

Expand Down Expand Up @@ -486,6 +490,14 @@ func processFlags() {
UlocMode = true
}

if ExcludeContent != "" {
var err error
excludeContentPattern, err = regexp.Compile(ExcludeContent)
if err != nil {
printError("ExcludeContent: " + err.Error())
}
}

if Debug {
printDebug(fmt.Sprintf("Path Deny List: %v", PathDenyList))
printDebug(fmt.Sprintf("Sort By: %s", SortBy))
Expand All @@ -502,6 +514,7 @@ func processFlags() {
printDebug(fmt.Sprintf("IncludeSymLinks: %t", IncludeSymLinks))
printDebug(fmt.Sprintf("Uloc: %t", UlocMode))
printDebug(fmt.Sprintf("Dryness: %t", Dryness))
printDebug(fmt.Sprintf("ExcludeContent: %s", ExcludeContent))
}
}

Expand Down
22 changes: 22 additions & 0 deletions processor/workers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package processor

import (
"bufio"
"bytes"
"fmt"
"hash"
Expand Down Expand Up @@ -639,6 +640,20 @@ func checkBomSkip(fileJob *FileJob) int {
return 0
}

func excludeByContents(data []byte) bool {
if excludeContentPattern == nil {
return false
}

scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
if excludeContentPattern.Match(scanner.Bytes()) {
return true
}
}
return false
}

// Reads and processes files from input chan in parallel, and sends results to
// output chan
func fileProcessorWorker(input chan *FileJob, output chan *FileJob) {
Expand All @@ -662,6 +677,13 @@ func fileProcessorWorker(input chan *FileJob, output chan *FileJob) {

fileStartTime := makeTimestampNano()
content, err := reader.ReadFile(loc, int(job.Bytes))
if excludeByContents(content) {
if Verbose {
printWarnf("exclude %s by its content", loc)
}
continue
}

atomic.AddInt64(&fileCount, 1)

if atomic.LoadInt64(&gcEnabled) == 0 && atomic.LoadInt64(&fileCount) >= int64(GcFileCount) {
Expand Down
24 changes: 24 additions & 0 deletions test-all.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1061,6 +1061,30 @@ else
echo -e "${GREEN}PASSED Issue379 Regression Check"
fi

Issue507GoFilesWithoutExclude=$(./scc -f csv "examples/issue507/" | grep 'Go' | cut -d ',' -f 8)
Issue507PythonFilesWithoutExclude=$(./scc -f csv "examples/issue507/" | grep 'Python' | cut -d ',' -f 8)
excludeContent="^(#|//) *exclude-content test.+, DO NOT EDIT."
Issue507GoFilesExcluded=$(./scc -f csv --exclude-content="${excludeContent}" "examples/issue507/" | grep 'Go' | cut -d ',' -f 8)
Issue507PythonFilesExcluded=$(./scc -f csv --exclude-content="${excludeContent}" "examples/issue507/" | grep 'Python' | cut -d ',' -f 8)
if [ ${Issue507GoFilesWithoutExclude:-0} -ne 1 ] || [ ${Issue507PythonFilesWithoutExclude:-0} -ne 2 ] ; then
echo -e "${RED}======================================================="
echo -e "FAILED Issue507 should contains 1 Go file and 2 Python files"
echo -e "=======================================================${NC}"
exit
elif [ ${Issue507GoFilesExcluded:-0} -ne 0 ] ; then
echo -e "${RED}======================================================="
echo -e "FAILED Issue507 exclude Go files by contents"
echo -e "=======================================================${NC}"
exit
elif [ ${Issue507PythonFilesExcluded:-0} -ne 1 ] ; then
echo -e "${RED}======================================================="
echo -e "FAILED Issue507 exclude Python files by contents"
echo -e "=======================================================${NC}"
exit
else
echo -e "${GREEN}PASSED Issue507 Exclude Content Check"
fi

# Extra case for longer languages that are normally truncated
for i in 'CloudFormation (YAM' 'CloudFormation (JSO'
do
Expand Down
Loading