Merge pull request #616 from opendatacube/update_robots_txt

Update robots.txt default content
opendatacube · Jul 11, 2024 · 09e2e61 · 09e2e61
2 parents 5100d50 + 8d47d48
commit 09e2e61
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/cubedash/_pages.py b/cubedash/_pages.py
@@ -40,9 +40,12 @@
 
 _DEFAULT_ARRIVALS_DAYS = 14
 
-_ROBOTS_TXT_DEFAULT = (
-    "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**"
-)
+_ROBOTS_TXT_DEFAULT = """User-Agent: *
+Allow: /
+Disallow: /products/*/*
+Disallow: /stac/**
+Disallow: /dataset/*
+"""
 
 
 @bp.route("/<product_name>")