update: robots.txt for other search engines

ShivaBhattacharjee · Mar 15, 2024 · ada84b0 · ada84b0
1 parent fe6da4d
commit ada84b0
Showing 1 changed file with 2 additions and 6 deletions.
diff --git a/src/app/robots.ts b/src/app/robots.ts
@@ -4,13 +4,9 @@ export default function robots(): MetadataRoute.Robots {
     return {
         rules: [
             {
-                userAgent: "Googlebot",
+                userAgent: ["Applebot", "Bingbot", "Googlebot", "Slurp", "DuckDuckBot", "Baiduspider", "Yandex"],
                 allow: ["/"],
-                disallow: "/private/",
-            },
-            {
-                userAgent: ["Applebot", "Bingbot"],
-                disallow: ["/"],
+                disallow: "/api/",
             },
         ],
         sitemap: "https://www.animetrix.xyz/sitemap.xml",