diff --git a/cubedash/_pages.py b/cubedash/_pages.py index cc4d5e8a1..d978e783d 100644 --- a/cubedash/_pages.py +++ b/cubedash/_pages.py @@ -40,9 +40,12 @@ _DEFAULT_ARRIVALS_DAYS = 14 -_ROBOTS_TXT_DEFAULT = ( - "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**" -) +_ROBOTS_TXT_DEFAULT = """User-Agent: * +Allow: / +Disallow: /products/*/* +Disallow: /stac/** +Disallow: /dataset/* +""" @bp.route("/")