update robots.txt to also disallow /dataset/* pages

opendatacube · Jul 8, 2024 · c688069 · c688069
1 parent 0ed0a01
commit c688069
Showing 1 changed file with 1 addition and 3 deletions.
diff --git a/cubedash/_pages.py b/cubedash/_pages.py
@@ -40,9 +40,7 @@
 
 _DEFAULT_ARRIVALS_DAYS = 14
 
-_ROBOTS_TXT_DEFAULT = (
-    "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**"
-)
+_ROBOTS_TXT_DEFAULT = "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**\nDisallow: /dataset/*"
 
 
 @bp.route("/<product_name>")