refactor: skip sources, test, javadocs jars

aquasecurity · Mar 6, 2024 · 4b4ea82 · 4b4ea82
1 parent c10d096
commit 4b4ea82
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 14 deletions.
diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go
@@ -257,7 +257,8 @@ func (c *Crawler) sha1Urls(ctx context.Context, url string) ([]string, error) {
 	var sha1URLs []string
 	d.Find("a").Each(func(i int, selection *goquery.Selection) {
 		link := selection.Text()
-		if strings.HasSuffix(link, ".jar.sha1") {
+		if strings.HasSuffix(link, ".jar.sha1") && !strings.HasSuffix(link, "sources.jar.sha1") &&
+			!strings.HasSuffix(link, "test.jar.sha1") && !strings.HasSuffix(link, "javadoc.jar.sha1") {
 			sha1URLs = append(sha1URLs, url+link)
 		}
 	})

diff --git a/pkg/crawler/crawler_test.go b/pkg/crawler/crawler_test.go
@@ -22,17 +22,17 @@ func TestCrawl(t *testing.T) {
 		{
 			name: "happy path",
 			fileNames: map[string]string{
-				"/maven2/":                                               "testdata/index.html",
-				"/maven2/abbot/":                                         "testdata/abbot.html",
-				"/maven2/abbot/abbot/":                                   "testdata/abbot_abbot.html",
-				"/maven2/abbot/abbot/maven-metadata.xml":                 "testdata/maven-metadata.xml",
-				"/maven2/abbot/abbot/0.12.3/":                            "testdata/abbot_abbot_0.12.3.html",
-				"/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1":       "testdata/abbot-0.12.3.jar.sha1",
-				"/maven2/abbot/abbot/0.13.0/":                            "testdata/abbot_abbot_0.13.0.html",
-				"/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1":       "testdata/abbot-0.13.0.jar.sha1",
-				"/maven2/abbot/abbot/1.4.0/":                             "testdata/abbot_abbot_1.4.0.html",
-				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1":         "testdata/abbot-1.4.0.jar.sha1",
-				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.sha1": "testdata/abbot-1.4.0-sources.jar.sha1",
+				"/maven2/":                                            "testdata/index.html",
+				"/maven2/abbot/":                                      "testdata/abbot.html",
+				"/maven2/abbot/abbot/":                                "testdata/abbot_abbot.html",
+				"/maven2/abbot/abbot/maven-metadata.xml":              "testdata/maven-metadata.xml",
+				"/maven2/abbot/abbot/0.12.3/":                         "testdata/abbot_abbot_0.12.3.html",
+				"/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1":    "testdata/abbot-0.12.3.jar.sha1",
+				"/maven2/abbot/abbot/0.13.0/":                         "testdata/abbot_abbot_0.13.0.html",
+				"/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1":    "testdata/abbot-0.13.0.jar.sha1",
+				"/maven2/abbot/abbot/1.4.0/":                          "testdata/abbot_abbot_1.4.0.html",
+				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1":      "testdata/abbot-1.4.0.jar.sha1",
+				"/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1": "testdata/abbot-1.4.0-lite.jar.sha1",
 			},
 			goldenPath: "testdata/golden/abbot.json",
 			filePath:   "indexes/abbot/abbot.json",

diff --git a/...ler/testdata/abbot-1.4.0-sources.jar.sha1 → ...rawler/testdata/abbot-1.4.0-lite.jar.sha1 b/...ler/testdata/abbot-1.4.0-sources.jar.sha1 → ...rawler/testdata/abbot-1.4.0-lite.jar.sha1
diff --git a/pkg/crawler/testdata/abbot_abbot_1.4.0.html b/pkg/crawler/testdata/abbot_abbot_1.4.0.html
@@ -17,6 +17,11 @@ <h1>abbot/abbot/1.4.0</h1>
 	<hr>
 	<main>
 		<pre id="contents"><a href="https://repo.maven.apache.org/maven2/abbot/abbot/">../</a>
+			<a href="jasypt-1.9.3-javadoc.jar" title="jasypt-1.9.3-javadoc.jar">jasypt-1.9.3-javadoc.jar</a>                          2019-05-25 16:34    748409
+<a href="https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar" title="abbot-1.4.0-lite.jar">abbot-1.4.0-lite.jar</a>                             2019-05-25 16:34     74953
+<a href="https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.asc" title="abbot-1.4.0-lite.jar.asc">abbot-1.4.0-lite.jar.asc</a>                         2019-05-25 16:34       516
+<a href="https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.md5" title="abbot-1.4.0-lite.jar.md5">abbot-1.4.0-lite.jar.md5</a>                         2019-05-25 16:34        32
+<a href="https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1" title="abbot-1.4.0-lite.jar.sha1">abbot-1.4.0-lite.jar.sha1</a>                    2019-05-25 16:34        40
 <a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar" title="abbot-1.4.0-sources.jar">abbot-1.4.0-sources.jar</a>                           2015-09-22 16:03    310023      
 <a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.asc" title="abbot-1.4.0-sources.jar.asc">abbot-1.4.0-sources.jar.asc</a>                       2015-09-22 16:03       490      
 <a href="https://repo.maven.apache.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.asc.md5" title="abbot-1.4.0-sources.jar.asc.md5">abbot-1.4.0-sources.jar.asc.md5</a>                   2015-09-22 16:03        32      

diff --git a/pkg/crawler/testdata/golden/abbot.json b/pkg/crawler/testdata/golden/abbot.json
@@ -11,7 +11,7 @@
       "SHA1": "WW2R5nYxsN6wX7aF2NG2c18+T2A="
     },
     {
-      "Version": "1.4.0-sources",
+      "Version": "1.4.0-lite",
       "SHA1": "BUerA3Bor6ICaSW9lL+5/Pzsl2E="
     },
     {

diff --git a/pkg/crawler/testdata/maven-metadata.xml b/pkg/crawler/testdata/maven-metadata.xml
@@ -7,7 +7,6 @@
     <release>1.4.0</release>
     <versions>
       <version>0.12.3</version>
-      <version>0.13.0</version>
       <version>1.4.0</version>
     </versions>
     <lastUpdated>20150924141841</lastUpdated>