Removes links to crawlers (#678)

* crawlers * testing * added test * Rubocop nitpicking --------- Co-authored-by: Hector Correa <[email protected]>
pulibrary · Sep 11, 2024 · 6d49001 · 6d49001
1 parent 7901b6f
commit 6d49001
Show file tree

Hide file tree

Showing 6 changed files with 4,969 additions and 2 deletions.
diff --git a/Gemfile b/Gemfile
@@ -39,6 +39,7 @@ gem 'turbolinks', '~> 5'
 gem 'twitter-typeahead-rails', '0.11.1.pre.corejavascript'
 gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby]
 gem 'vite_rails'
+gem 'voight_kampff', require: 'voight_kampff/rails'
 gem 'whenever'
 
 group :development, :test do

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -552,6 +552,8 @@ GEM
       dry-cli (>= 0.7, < 2)
       rack-proxy (~> 0.6, >= 0.6.1)
       zeitwerk (~> 2.2)
+    voight_kampff (2.0.0)
+      rack (>= 1.4)
     warden (1.2.9)
       rack (>= 2.0.9)
     web-console (4.2.1)
@@ -647,6 +649,7 @@ DEPENDENCIES
   twitter-typeahead-rails (= 0.11.1.pre.corejavascript)
   tzinfo-data
   vite_rails
+  voight_kampff
   web-console (>= 4.1.0)
   webmock
   whenever

diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb
@@ -252,10 +252,17 @@ def retry_on_exception
       :doi, :ark, :id,
       :a # this is in the search parameters becuase the search bar is shown on the error page
     ]
+
+    # Sets up Blacklight crawler detection
+    config.crawler_detector = lambda { |request|
+      return true if request.env['HTTP_USER_AGENT'].blank?
+      request.bot?
+    }
   end
 
   def show
-    super()
+    @render_links = !agent_is_crawler?
+    super
     if params["format"] == "json"
       render json: DocumentExport.new(@document)
     end

diff --git a/app/views/catalog/_show_documents.html.erb b/app/views/catalog/_show_documents.html.erb
@@ -30,7 +30,11 @@
                   <td>
                     <span>
                       <i class="bi bi-file-arrow-down"></i>
-                      <a href="<%= file.download_url %>" class="documents-file-link" target="_blank" title="<%= file.full_path %>"><%= truncate(file.full_path, length: 60) %></a>
+                      <% if @render_links%>
+                        <a href="<%= file.download_url %>" class="documents-file-link" target="_blank" title="<%= file.full_path %>"><%= truncate(file.full_path, length: 60) %></a>
+                      <% else %>
+                        <%= truncate(file.full_path, length: 60) %>
+                      <% end %>
                     </span>
                   </td>
                   <% if @document.data_source == "dataspace" %>