diff --git a/crawlers.go b/crawlers.go index 52bbbac..4feb9ce 100644 --- a/crawlers.go +++ b/crawlers.go @@ -192,7 +192,6 @@ var crawlers = []string{ `Branch Metrics API`, `Branch-Passthrough`, `Brandprotect`, - `BrandVerity`, `Brandwatch`, `Brodie\/`, `Browsershots`, @@ -481,6 +480,7 @@ var crawlers = []string{ `Google-Youtube-Links`, `GoogleDocs`, `GoogleHC\/`, + `GoogleOther`, `GoogleProber`, `GoogleProducer`, `GoogleSites`, @@ -698,6 +698,7 @@ var crawlers = []string{ `longurl-r-package`, `looid\.com`, `looksystems\.net`, + `lscache_runner`, `ltx71`, `lua-resty-http`, `Lucee \(CFML Engine\)`, @@ -1244,6 +1245,7 @@ var crawlers = []string{ `VB Project`, `vBSEO`, `VCI`, + `Verity`, `via ggpht\.com GoogleImageProxy`, `Virusdie`, `visionutils`, @@ -1395,6 +1397,7 @@ var crawlers = []string{ `Zabbix`, `Zade`, `Zao`, + `Zapier`, `Zauba`, `Zemanta Aggregator`, `Zend\\Http\\Client`, diff --git a/testdata/crawlers.txt b/testdata/crawlers.txt index f551848..6b9eacf 100644 --- a/testdata/crawlers.txt +++ b/testdata/crawlers.txt @@ -3669,3 +3669,7 @@ Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/ Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GeedoProductSearch; +http://www.geedo.com/product-search.html) Chrome/79.0.3945.88 Safari/537.36 Mozilla/5.0 (compatible; +probely.com/sos) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36 ProbelySPDR/0.2.0 facebookcatalog/1.0 +Verity/1.1 (gumgum.com/verity; verity-support@gumgum.com) +Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/117.0.5938.132 Safari/537.36 +Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Mobile Safari/537.36 (compatible; GoogleOther) +Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; GoogleOther) \ No newline at end of file