diff --git a/crawlers.go b/crawlers.go index b48f539..f461e9e 100644 --- a/crawlers.go +++ b/crawlers.go @@ -1,6 +1,6 @@ package crawlerdetect -// auto-generated based on https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt +// auto-generated based on https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/v1.3.0/raw/Crawlers.txt var crawlers = []string{ ` YLT`, `^Aether`, diff --git a/exclusions.go b/exclusions.go index 73b6a7e..0498248 100644 --- a/exclusions.go +++ b/exclusions.go @@ -1,6 +1,6 @@ package crawlerdetect -// auto-generated based on https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.txt +// auto-generated based on https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/v1.3.0/raw/Exclusions.txt var exclusions = []string{ `Safari.[\d\.]*`, `Firefox.[\d\.]*`, @@ -52,5 +52,7 @@ var exclusions = []string{ `OCTOPUS-CORE`, `htc_botdugls`, `super\/\d+\/Android\/\d+`, + `"Yandex"`, + `YandexModule2`, `YandexSearch`, } diff --git a/testdata/crawlers.txt b/testdata/crawlers.txt index 92e3c06..72a77f8 100644 --- a/testdata/crawlers.txt +++ b/testdata/crawlers.txt @@ -1116,7 +1116,6 @@ ShowyouBot (http://showyou.com/crawler) Yoleo Consumer v0.2 Mozilla/5.0 (compatible; IstellaBot/1.10.2 +http://www.tiscali.it/) Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server; +http://ws.daum.net/aboutWebSearch.html) Daumoa/3.0 -Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 YaBrowser/15.12.0.6151 Safari/537.36 Yandex.Translate Mozilla/5.0 (compatible; TeeRaidBot; +https://teeraid.com/bot/) Mozilla/5.0 (compatible; Scopia Crawler 1.1; +http://www.scopia.co) CopperEgg/RevealUptime/DublinIE(aws) @@ -3674,4 +3673,6 @@ facebookcatalog/1.0 Verity/1.1 (gumgum.com/verity; verity-support@gumgum.com) Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/117.0.5938.132 Safari/537.36 Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Mobile Safari/537.36 (compatible; GoogleOther) -Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; GoogleOther) \ No newline at end of file +Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; GoogleOther) +Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) +Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots) \ No newline at end of file diff --git a/testdata/devices.txt b/testdata/devices.txt index 5d07f58..89ec1be 100644 --- a/testdata/devices.txt +++ b/testdata/devices.txt @@ -165632,4 +165632,6 @@ Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (K Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148/mvmkbtaqqdhshwcbsygjnuqcqmpndbrt Mozilla/5.0 (Linux; Android 5.0; SM-N9100 Build/LRX21V) > AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 > Chrome/37.0.0.0 Mobile Safari/537.36 > MicroMessenger/6.0.2.56_r958800.520 NetType/WIFI Mozilla/5.0 (Linux; U; Android 5.0; en-US; SM-N900S;iPhone: Build/LRX21V) -Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50 \ No newline at end of file +Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50 +Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 YaBrowser/24.1.2.86 (lite) TV Safari/537.36 +Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 Mobile Safari/537.36 \ No newline at end of file