diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go index 794717f..63bde2b 100644 --- a/pkg/crawler/crawler.go +++ b/pkg/crawler/crawler.go @@ -257,7 +257,8 @@ func (c *Crawler) sha1Urls(ctx context.Context, url string) ([]string, error) { var sha1URLs []string d.Find("a").Each(func(i int, selection *goquery.Selection) { link := selection.Text() - if strings.HasSuffix(link, ".jar.sha1") { + if strings.HasSuffix(link, ".jar.sha1") && !strings.HasSuffix(link, "sources.jar.sha1") && + !strings.HasSuffix(link, "test.jar.sha1") && !strings.HasSuffix(link, "javadoc.jar.sha1") { sha1URLs = append(sha1URLs, url+link) } }) diff --git a/pkg/crawler/crawler_test.go b/pkg/crawler/crawler_test.go index 8edc800..a04d707 100644 --- a/pkg/crawler/crawler_test.go +++ b/pkg/crawler/crawler_test.go @@ -22,17 +22,17 @@ func TestCrawl(t *testing.T) { { name: "happy path", fileNames: map[string]string{ - "/maven2/": "testdata/index.html", - "/maven2/abbot/": "testdata/abbot.html", - "/maven2/abbot/abbot/": "testdata/abbot_abbot.html", - "/maven2/abbot/abbot/maven-metadata.xml": "testdata/maven-metadata.xml", - "/maven2/abbot/abbot/0.12.3/": "testdata/abbot_abbot_0.12.3.html", - "/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1": "testdata/abbot-0.12.3.jar.sha1", - "/maven2/abbot/abbot/0.13.0/": "testdata/abbot_abbot_0.13.0.html", - "/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1": "testdata/abbot-0.13.0.jar.sha1", - "/maven2/abbot/abbot/1.4.0/": "testdata/abbot_abbot_1.4.0.html", - "/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1": "testdata/abbot-1.4.0.jar.sha1", - "/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.sha1": "testdata/abbot-1.4.0-sources.jar.sha1", + "/maven2/": "testdata/index.html", + "/maven2/abbot/": "testdata/abbot.html", + "/maven2/abbot/abbot/": "testdata/abbot_abbot.html", + "/maven2/abbot/abbot/maven-metadata.xml": "testdata/maven-metadata.xml", + "/maven2/abbot/abbot/0.12.3/": "testdata/abbot_abbot_0.12.3.html", + "/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1": "testdata/abbot-0.12.3.jar.sha1", + "/maven2/abbot/abbot/0.13.0/": "testdata/abbot_abbot_0.13.0.html", + "/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1": "testdata/abbot-0.13.0.jar.sha1", + "/maven2/abbot/abbot/1.4.0/": "testdata/abbot_abbot_1.4.0.html", + "/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1": "testdata/abbot-1.4.0.jar.sha1", + "/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1": "testdata/abbot-1.4.0-lite.jar.sha1", }, goldenPath: "testdata/golden/abbot.json", filePath: "indexes/abbot/abbot.json", diff --git a/pkg/crawler/testdata/abbot-1.4.0-sources.jar.sha1 b/pkg/crawler/testdata/abbot-1.4.0-lite.jar.sha1 similarity index 100% rename from pkg/crawler/testdata/abbot-1.4.0-sources.jar.sha1 rename to pkg/crawler/testdata/abbot-1.4.0-lite.jar.sha1 diff --git a/pkg/crawler/testdata/abbot_abbot_1.4.0.html b/pkg/crawler/testdata/abbot_abbot_1.4.0.html index 61ba51c..4dd9c58 100644 --- a/pkg/crawler/testdata/abbot_abbot_1.4.0.html +++ b/pkg/crawler/testdata/abbot_abbot_1.4.0.html @@ -17,6 +17,11 @@

abbot/abbot/1.4.0


../
+			jasypt-1.9.3-javadoc.jar                          2019-05-25 16:34    748409
+abbot-1.4.0-lite.jar                             2019-05-25 16:34     74953
+abbot-1.4.0-lite.jar.asc                         2019-05-25 16:34       516
+abbot-1.4.0-lite.jar.md5                         2019-05-25 16:34        32
+abbot-1.4.0-lite.jar.sha1                    2019-05-25 16:34        40
 abbot-1.4.0-sources.jar                           2015-09-22 16:03    310023      
 abbot-1.4.0-sources.jar.asc                       2015-09-22 16:03       490      
 abbot-1.4.0-sources.jar.asc.md5                   2015-09-22 16:03        32      
diff --git a/pkg/crawler/testdata/golden/abbot.json b/pkg/crawler/testdata/golden/abbot.json
index d81717b..89791ee 100644
--- a/pkg/crawler/testdata/golden/abbot.json
+++ b/pkg/crawler/testdata/golden/abbot.json
@@ -11,7 +11,7 @@
       "SHA1": "WW2R5nYxsN6wX7aF2NG2c18+T2A="
     },
     {
-      "Version": "1.4.0-sources",
+      "Version": "1.4.0-lite",
       "SHA1": "BUerA3Bor6ICaSW9lL+5/Pzsl2E="
     },
     {
diff --git a/pkg/crawler/testdata/maven-metadata.xml b/pkg/crawler/testdata/maven-metadata.xml
index 51ec1ab..8ef2187 100644
--- a/pkg/crawler/testdata/maven-metadata.xml
+++ b/pkg/crawler/testdata/maven-metadata.xml
@@ -7,7 +7,6 @@
     1.4.0
     
       0.12.3
-      0.13.0
       1.4.0
     
     20150924141841