From 40a3cb21004c0244d4ee94957622994ba4e4df34 Mon Sep 17 00:00:00 2001
From: Michael Berkowski <mjb@umn.edu>
Date: Tue, 26 Nov 2024 08:19:44 -0600
Subject: [PATCH] Update robots.text.erb

Block imagesiftbot (https://imagesift.com/about) and introduce 5s crawl delay for other well behaved bots
---
 app/views/robots/robots.text.erb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/views/robots/robots.text.erb b/app/views/robots/robots.text.erb
index 18bafc477..ee22a3d71 100644
--- a/app/views/robots/robots.text.erb
+++ b/app/views/robots/robots.text.erb
@@ -9,6 +9,7 @@ User-agent: *
 Disallow: /
 <%- else -%>
 User-agent: *
+Crawl-delay: 5
 Disallow: /?q=
 Disallow: /?f
 Disallow: /?_
@@ -37,4 +38,6 @@ User-agent: DotBot
 Disallow: /
 User-agent: DataForSeoBot
 Disallow: /
+User-agent: ImagesiftBot
+Disallow: /
 <%- end -%>