From 8d47d482de75d9c5684f31cee0a18460db7bd10d Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Wed, 10 Jul 2024 07:53:33 +0000 Subject: [PATCH] make robots default multiline string --- cubedash/_pages.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cubedash/_pages.py b/cubedash/_pages.py index 31e8520e6..d978e783d 100644 --- a/cubedash/_pages.py +++ b/cubedash/_pages.py @@ -40,7 +40,12 @@ _DEFAULT_ARRIVALS_DAYS = 14 -_ROBOTS_TXT_DEFAULT = "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**\nDisallow: /dataset/*" +_ROBOTS_TXT_DEFAULT = """User-Agent: * +Allow: / +Disallow: /products/*/* +Disallow: /stac/** +Disallow: /dataset/* +""" @bp.route("/")