diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go
index 794717f..63bde2b 100644
--- a/pkg/crawler/crawler.go
+++ b/pkg/crawler/crawler.go
@@ -257,7 +257,8 @@ func (c *Crawler) sha1Urls(ctx context.Context, url string) ([]string, error) {
var sha1URLs []string
d.Find("a").Each(func(i int, selection *goquery.Selection) {
link := selection.Text()
- if strings.HasSuffix(link, ".jar.sha1") {
+ if strings.HasSuffix(link, ".jar.sha1") && !strings.HasSuffix(link, "sources.jar.sha1") &&
+ !strings.HasSuffix(link, "test.jar.sha1") && !strings.HasSuffix(link, "javadoc.jar.sha1") {
sha1URLs = append(sha1URLs, url+link)
}
})
diff --git a/pkg/crawler/crawler_test.go b/pkg/crawler/crawler_test.go
index 8edc800..a04d707 100644
--- a/pkg/crawler/crawler_test.go
+++ b/pkg/crawler/crawler_test.go
@@ -22,17 +22,17 @@ func TestCrawl(t *testing.T) {
{
name: "happy path",
fileNames: map[string]string{
- "/maven2/": "testdata/index.html",
- "/maven2/abbot/": "testdata/abbot.html",
- "/maven2/abbot/abbot/": "testdata/abbot_abbot.html",
- "/maven2/abbot/abbot/maven-metadata.xml": "testdata/maven-metadata.xml",
- "/maven2/abbot/abbot/0.12.3/": "testdata/abbot_abbot_0.12.3.html",
- "/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1": "testdata/abbot-0.12.3.jar.sha1",
- "/maven2/abbot/abbot/0.13.0/": "testdata/abbot_abbot_0.13.0.html",
- "/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1": "testdata/abbot-0.13.0.jar.sha1",
- "/maven2/abbot/abbot/1.4.0/": "testdata/abbot_abbot_1.4.0.html",
- "/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1": "testdata/abbot-1.4.0.jar.sha1",
- "/maven2/abbot/abbot/1.4.0/abbot-1.4.0-sources.jar.sha1": "testdata/abbot-1.4.0-sources.jar.sha1",
+ "/maven2/": "testdata/index.html",
+ "/maven2/abbot/": "testdata/abbot.html",
+ "/maven2/abbot/abbot/": "testdata/abbot_abbot.html",
+ "/maven2/abbot/abbot/maven-metadata.xml": "testdata/maven-metadata.xml",
+ "/maven2/abbot/abbot/0.12.3/": "testdata/abbot_abbot_0.12.3.html",
+ "/maven2/abbot/abbot/0.12.3/abbot-0.12.3.jar.sha1": "testdata/abbot-0.12.3.jar.sha1",
+ "/maven2/abbot/abbot/0.13.0/": "testdata/abbot_abbot_0.13.0.html",
+ "/maven2/abbot/abbot/0.13.0/abbot-0.13.0.jar.sha1": "testdata/abbot-0.13.0.jar.sha1",
+ "/maven2/abbot/abbot/1.4.0/": "testdata/abbot_abbot_1.4.0.html",
+ "/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar.sha1": "testdata/abbot-1.4.0.jar.sha1",
+ "/maven2/abbot/abbot/1.4.0/abbot-1.4.0-lite.jar.sha1": "testdata/abbot-1.4.0-lite.jar.sha1",
},
goldenPath: "testdata/golden/abbot.json",
filePath: "indexes/abbot/abbot.json",
diff --git a/pkg/crawler/testdata/abbot-1.4.0-sources.jar.sha1 b/pkg/crawler/testdata/abbot-1.4.0-lite.jar.sha1
similarity index 100%
rename from pkg/crawler/testdata/abbot-1.4.0-sources.jar.sha1
rename to pkg/crawler/testdata/abbot-1.4.0-lite.jar.sha1
diff --git a/pkg/crawler/testdata/abbot_abbot_1.4.0.html b/pkg/crawler/testdata/abbot_abbot_1.4.0.html
index 61ba51c..4dd9c58 100644
--- a/pkg/crawler/testdata/abbot_abbot_1.4.0.html
+++ b/pkg/crawler/testdata/abbot_abbot_1.4.0.html
@@ -17,6 +17,11 @@
abbot/abbot/1.4.0
../
+ jasypt-1.9.3-javadoc.jar 2019-05-25 16:34 748409
+abbot-1.4.0-lite.jar 2019-05-25 16:34 74953
+abbot-1.4.0-lite.jar.asc 2019-05-25 16:34 516
+abbot-1.4.0-lite.jar.md5 2019-05-25 16:34 32
+abbot-1.4.0-lite.jar.sha1 2019-05-25 16:34 40
abbot-1.4.0-sources.jar 2015-09-22 16:03 310023
abbot-1.4.0-sources.jar.asc 2015-09-22 16:03 490
abbot-1.4.0-sources.jar.asc.md5 2015-09-22 16:03 32
diff --git a/pkg/crawler/testdata/golden/abbot.json b/pkg/crawler/testdata/golden/abbot.json
index d81717b..89791ee 100644
--- a/pkg/crawler/testdata/golden/abbot.json
+++ b/pkg/crawler/testdata/golden/abbot.json
@@ -11,7 +11,7 @@
"SHA1": "WW2R5nYxsN6wX7aF2NG2c18+T2A="
},
{
- "Version": "1.4.0-sources",
+ "Version": "1.4.0-lite",
"SHA1": "BUerA3Bor6ICaSW9lL+5/Pzsl2E="
},
{
diff --git a/pkg/crawler/testdata/maven-metadata.xml b/pkg/crawler/testdata/maven-metadata.xml
index 51ec1ab..8ef2187 100644
--- a/pkg/crawler/testdata/maven-metadata.xml
+++ b/pkg/crawler/testdata/maven-metadata.xml
@@ -7,7 +7,6 @@
1.4.0
0.12.3
- 0.13.0
1.4.0
20150924141841