Merge pull request #334 from openml/PGijsbers-patch-2

Disallow crawlers on the `/data` subdirectory
openml · Nov 10, 2024 · e735f01 · e735f01
2 parents 6b5a41f + 5030d15
commit e735f01
Showing 1 changed file with 1 addition and 33 deletions.
diff --git a/server/src/client/app/public/robots.txt b/server/src/client/app/public/robots.txt
@@ -1,35 +1,3 @@
-User-agent: Googlebot
-Disallow: 
-User-agent: googlebot-image
-Disallow: 
-User-agent: googlebot-mobile
-Disallow: 
-User-agent: MSNBot
-Disallow: 
-User-agent: Slurp
-Disallow: 
-User-agent: Teoma
-Disallow: 
-User-agent: Gigabot
-Disallow: 
-User-agent: Robozilla
-Disallow: 
-User-agent: Nutch
-Disallow: 
-User-agent: ia_archiver
-Disallow: 
-User-agent: baiduspider
-Disallow: 
-User-agent: naverbot
-Disallow: 
-User-agent: yeti
-Disallow: 
-User-agent: yahoo-mmcrawler
-Disallow: 
-User-agent: psbot
-Disallow: 
-User-agent: yahoo-blogs/v3.9
-Disallow: 
 User-agent: *
-Disallow: 
+Disallow: /data/
 Disallow: /cgi-bin/