From c6880699e8ea3472ce7509654bbc59b1ba4de816 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Mon, 8 Jul 2024 06:26:35 +0000 Subject: [PATCH 1/2] update robots.txt to also disallow /dataset/* pages --- cubedash/_pages.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cubedash/_pages.py b/cubedash/_pages.py index cc4d5e8a1..31e8520e6 100644 --- a/cubedash/_pages.py +++ b/cubedash/_pages.py @@ -40,9 +40,7 @@ _DEFAULT_ARRIVALS_DAYS = 14 -_ROBOTS_TXT_DEFAULT = ( - "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**" -) +_ROBOTS_TXT_DEFAULT = "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**\nDisallow: /dataset/*" @bp.route("/") From 8d47d482de75d9c5684f31cee0a18460db7bd10d Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Wed, 10 Jul 2024 07:53:33 +0000 Subject: [PATCH 2/2] make robots default multiline string --- cubedash/_pages.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cubedash/_pages.py b/cubedash/_pages.py index 31e8520e6..d978e783d 100644 --- a/cubedash/_pages.py +++ b/cubedash/_pages.py @@ -40,7 +40,12 @@ _DEFAULT_ARRIVALS_DAYS = 14 -_ROBOTS_TXT_DEFAULT = "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**\nDisallow: /dataset/*" +_ROBOTS_TXT_DEFAULT = """User-Agent: * +Allow: / +Disallow: /products/*/* +Disallow: /stac/** +Disallow: /dataset/* +""" @bp.route("/")