From 707a0c97fab37769eb46fc9da4813dea762305dd Mon Sep 17 00:00:00 2001
From: George Baskerville <148500997+the-square-pixel@users.noreply.github.com>
Date: Sun, 14 Apr 2024 13:28:49 +0200
Subject: [PATCH] stricten robots.txt for better SEO

---
 public/robots.txt | 30 +++++-------------------------
 1 file changed, 5 insertions(+), 25 deletions(-)

diff --git a/public/robots.txt b/public/robots.txt
index 305445dc..7ae769a3 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,26 +1,6 @@
-User-agent: anthropic-ai
-Disallow: /
-
-User-agent: CCBot
-Disallow: /
-
-User-agent: ChatGPT-User
-Disallow: /
-
-User-agent: FacebookBot
-Disallow: /
-
-User-agent: GoogleOther
-Disallow: /
-
-User-agent: Google-Extended
-Disallow: /
-
-User-agent: GPTBot
-Disallow: /
-
-User-agent: OmigiliBot
-Disallow: /
-
 User-agent: *
-Disallow:
\ No newline at end of file
+Disallow: / 
+Allow: /$
+Allow: /posts/
+Allow: /posts$
+Allow: /privacy