Merge pull request #67 from Nightfirecat/semrushbot

Block SemrushBot
ai-robots-txt · Jan 6, 2025 · 565dca3 · 565dca3
2 parents 8e98cc6 + 143f8f2
commit 565dca3
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/robots.json b/robots.json
@@ -258,6 +258,13 @@
         "operator": "[Zyte](https://www.zyte.com)",
         "respect": "Unclear at this time."
     },
+    "SemrushBot": {
+        "operator": "[Semrush](https://www.semrush.com/)",
+        "respect": "[Yes](https://www.semrush.com/bot/)",
+        "function": "Scrapes data for use in LLM article-writing tool.",
+        "frequency": "Roughly once every 10 seconds.",
+        "description": "SemrushBot is a bot which, among other functions, scrapes data for use in ContentShake AI tool reports."
+    },
     "Sidetrade indexer bot": {
         "description": "AI product training.",
         "frequency": "No information.",