diff --git a/composer.lock b/composer.lock index 34cf661..627dcb1 100644 --- a/composer.lock +++ b/composer.lock @@ -9,16 +9,16 @@ "packages-dev": [ { "name": "myclabs/deep-copy", - "version": "1.11.1", + "version": "1.12.0", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c" + "reference": "3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", - "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c", + "reference": "3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c", "shasum": "" }, "require": { @@ -26,11 +26,12 @@ }, "conflict": { "doctrine/collections": "<1.6.8", - "doctrine/common": "<2.13.3 || >=3,<3.2.2" + "doctrine/common": "<2.13.3 || >=3 <3.2.2" }, "require-dev": { "doctrine/collections": "^1.6.8", "doctrine/common": "^2.13.3 || ^3.2.2", + "phpspec/prophecy": "^1.10", "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" }, "type": "library", @@ -56,7 +57,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.11.1" + "source": "https://github.com/myclabs/DeepCopy/tree/1.12.0" }, "funding": [ { @@ -64,20 +65,20 @@ "type": "tidelift" } ], - "time": "2023-03-08T13:26:56+00:00" + "time": "2024-06-12T14:39:25+00:00" }, { "name": "nikic/php-parser", - "version": "v5.0.2", + "version": "v5.1.0", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13" + "reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/139676794dc1e9231bf7bcd123cfc0c99182cb13", - "reference": "139676794dc1e9231bf7bcd123cfc0c99182cb13", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/683130c2ff8c2739f4822ff7ac5c873ec529abd1", + "reference": "683130c2ff8c2739f4822ff7ac5c873ec529abd1", "shasum": "" }, "require": { @@ -88,7 +89,7 @@ }, "require-dev": { "ircmaxell/php-yacc": "^0.0.7", - "phpunit/phpunit": "^7.0 || ^8.0 || ^9.0" + "phpunit/phpunit": "^9.0" }, "bin": [ "bin/php-parse" @@ -120,9 +121,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.0.2" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.1.0" }, - "time": "2024-03-05T20:51:40+00:00" + "time": "2024-07-01T20:03:41+00:00" }, { "name": "phar-io/manifest", @@ -244,16 +245,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.67", + "version": "1.11.9", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "16ddbe776f10da6a95ebd25de7c1dbed397dc493" + "reference": "e370bcddadaede0c1716338b262346f40d296f82" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/16ddbe776f10da6a95ebd25de7c1dbed397dc493", - "reference": "16ddbe776f10da6a95ebd25de7c1dbed397dc493", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/e370bcddadaede0c1716338b262346f40d296f82", + "reference": "e370bcddadaede0c1716338b262346f40d296f82", "shasum": "" }, "require": { @@ -298,20 +299,20 @@ "type": "github" } ], - "time": "2024-04-16T07:22:02+00:00" + "time": "2024-08-01T16:25:18+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "10.1.14", + "version": "10.1.15", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "e3f51450ebffe8e0efdf7346ae966a656f7d5e5b" + "reference": "5da8b1728acd1e6ffdf2ff32ffbdfd04307f26ae" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/e3f51450ebffe8e0efdf7346ae966a656f7d5e5b", - "reference": "e3f51450ebffe8e0efdf7346ae966a656f7d5e5b", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/5da8b1728acd1e6ffdf2ff32ffbdfd04307f26ae", + "reference": "5da8b1728acd1e6ffdf2ff32ffbdfd04307f26ae", "shasum": "" }, "require": { @@ -368,7 +369,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.14" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.15" }, "funding": [ { @@ -376,7 +377,7 @@ "type": "github" } ], - "time": "2024-03-12T15:33:41+00:00" + "time": "2024-06-29T08:25:15+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1698,5 +1699,5 @@ "php": ">=8.1" }, "platform-dev": [], - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } diff --git a/src/mappings/crawlers.php b/src/mappings/crawlers.php index b1da164..db8d90d 100644 --- a/src/mappings/crawlers.php +++ b/src/mappings/crawlers.php @@ -65,7 +65,8 @@ public static function getApp(string $value, array $data = []) : array { 'yisouspider' => 'search', '360spider' => 'search', 'sogou web spider' => 'search', - 'bytespider' => 'crawler' + 'bytespider' => 'crawler', + 'claudebot' => 'ai' ]; $apps = [ 'yacybot' => 'YacyBot', @@ -124,7 +125,9 @@ public static function getApp(string $value, array $data = []) : array { 'googledocs' => 'Google Docs', 'user-agent: seolyt' => 'SEOlyt', 'bytespider' => 'ByteDance Spider', - 'spider-feedback@bytedance.com' => 'ByteDance Spider' + 'spider-feedback@bytedance.com' => 'ByteDance Spider', + 'oai-searchbot' => 'OpenAI SearchBot', + 'chatgpt-user' => 'ChatGPT User' ]; $lower = \mb_strtolower($parts[0]); @@ -391,6 +394,7 @@ public static function get() : array { 'http-client/' => new props('any', $fn['scraper']), 'HttpClient/' => new props('any', $fn['scraper']), 'PowerShell/' => new props('start', $fn['scraper']), + 'OAI-SearchBot/' => new props('start', $fn['search']), 'GPTBot/' => new props('start', $fn['ai']), 'Diffbot/' => new props('start', $fn['ai']), 'Amazonbot/' => new props('start', $fn['ai']), @@ -398,7 +402,7 @@ public static function get() : array { 'PerplexityBot/' => new props('start', $fn['ai']), 'YouBot/' => new props('start', $fn['ai']), 'Google-Extended' => new props('start', $fn['ai']), - 'ChatGPT-User/' => new props('start', $fn['feed']), + 'ChatGPT-User/' => new props('start', $fn['ai']), 'facebookexternalhit/' => new props('start', $fn['feed']), 'facebookcatalog/' => new props('start', $fn['feed']), 'Validator' => new props('any', $fn['validator']), diff --git a/tests/crawlersTest.php b/tests/crawlersTest.php index 09070ca..6e182df 100644 --- a/tests/crawlersTest.php +++ b/tests/crawlersTest.php @@ -411,6 +411,15 @@ public function testSearch() : void { 'appname' => 'YandexRenderResourcesBot', 'appversion' => '1.0', 'url' => 'http://yandex.com/bots' + ], + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot' => [ + 'string' => 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot', + 'type' => 'robot', + 'category' => 'search', + 'app' => 'OpenAI SearchBot', + 'appname' => 'OAI-SearchBot', + 'appversion' => '1.0', + 'url' => 'https://openai.com/searchbot' ] ]; foreach ($strings AS $ua => $item) { @@ -589,6 +598,15 @@ public function testFeed() : void { 'appname' => 'filemanager.downloads.http.client - contentfilemanagerdaemons-downloadFromUrlTq2Worke', 'appversion' => '2.0', 'url' => 'http://dev.hubspot.com/' + ], + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot' => [ + 'string' => 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot', + 'type' => 'robot', + 'category' => 'ai', + 'app' => 'ChatGPT User', + 'appname' => 'ChatGPT-User', + 'appversion' => '1.0', + 'url' => 'https://openai.com/bot' ] ]; foreach ($strings AS $ua => $item) { @@ -1278,6 +1296,15 @@ public function testAi() : void { 'appname' => 'YouBot', 'appversion' => '1.0', 'url' => 'https://about.you.com/youbot/' + ], + 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot' => [ + 'string' => 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot', + 'type' => 'robot', + 'category' => 'ai', + 'app' => 'GPTBot', + 'appname' => 'GPTBot', + 'appversion' => '1.1', + 'url' => 'https://openai.com/gptbot' ] ]; foreach ($strings AS $ua => $item) {