stricten robots.txt for better SEO

georgebaskervil · Apr 14, 2024 · 707a0c9 · 707a0c9
1 parent 069ce97
commit 707a0c9
Showing 1 changed file with 5 additions and 25 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,26 +1,6 @@
-User-agent: anthropic-ai
-Disallow: /
-
-User-agent: CCBot
-Disallow: /
-
-User-agent: ChatGPT-User
-Disallow: /
-
-User-agent: FacebookBot
-Disallow: /
-
-User-agent: GoogleOther
-Disallow: /
-
-User-agent: Google-Extended
-Disallow: /
-
-User-agent: GPTBot
-Disallow: /
-
-User-agent: OmigiliBot
-Disallow: /
-
 User-agent: *
-Disallow:
+Disallow: / 
+Allow: /$
+Allow: /posts/
+Allow: /posts$
+Allow: /privacy