diff --git a/crawlers.go b/crawlers.go index f35dfc8..1ab72d1 100644 --- a/crawlers.go +++ b/crawlers.go @@ -1,6 +1,6 @@ package crawlerdetect -// auto-generated based on https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt +// auto-generated based on https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/v1.2.116/raw/Crawlers.txt var crawlers = []string{ ` YLT`, `^Aether`, @@ -407,6 +407,7 @@ var crawlers = []string{ `forensiq`, `ForusP`, `FoundSeoTool`, + `fragFINN\.de`, `free thumbnails`, `Freeuploader`, `FreshRSS`, @@ -465,6 +466,7 @@ var crawlers = []string{ `Google-Calendar-Importer`, `Google-HotelAdsVerifier`, `Google-HTTP-Java-Client`, + `Google-InspectionTool`, `Google-Podcast`, `Google-Publisher-Plugin`, `Google-Read-Aloud`, @@ -601,6 +603,7 @@ var crawlers = []string{ `intraVnews`, `IODC`, `IOI`, + `Inboxb0t`, `iplabel`, `ips-agent`, `IPS\/[0-9]`, @@ -1108,9 +1111,11 @@ var crawlers = []string{ `SpamExperts`, `Spammen`, `Spanner`, + `Spawning-AI`, `spaziodati`, `SPDYCheck`, `Specificfeeds`, + `SpeedKit`, `speedy`, `SPEng`, `Spinn3r`, diff --git a/exclusions.go b/exclusions.go index 89d81c1..54a3339 100644 --- a/exclusions.go +++ b/exclusions.go @@ -1,6 +1,6 @@ package crawlerdetect -// auto-generated based on https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.txt +// auto-generated based on https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/v1.2.116/raw/Exclusions.txt var exclusions = []string{ `Safari.[\d\.]*`, `Firefox.[\d\.]*`, @@ -50,5 +50,7 @@ var exclusions = []string{ `; ID bot`, `; POWER BOT`, `OCTOPUS-CORE`, + `htc_botdugls`, + `super\/\d+\/Android\/\d+`, `YandexSearch`, } diff --git a/testdata/crawlers.txt b/testdata/crawlers.txt index 36e5b3e..7fdd08b 100644 --- a/testdata/crawlers.txt +++ b/testdata/crawlers.txt @@ -3658,3 +3658,6 @@ Mozilla/5.0 (compatible; seocompany/0.6; +; ) Visual Rights Group (https://visualrightsgroup.com/) Scanner Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io) Mollie HTTP client/1.0 +Inboxb0t-Image/2.0 +fragFINN.de HTTP-Collector +SpeedKit \ No newline at end of file