From 95ee97015343cd9ea93eb16ac8aebf70a293d808 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 25 Oct 2023 11:44:08 -0500 Subject: [PATCH 01/15] README: bump for v5.0.3 release --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ebe6d8d53..c3c849505 100644 --- a/README.md +++ b/README.md @@ -30,4 +30,4 @@ Visit the [project wiki](https://github.com/BTAA-Geospatial-Data-Project/geoport #### Release Version -B1G Geoportal Version v5.0.2 / GeoBlacklight v4.1.0 / GeoBlacklight Admin v0.3.0 +B1G Geoportal Version v5.0.3 / GeoBlacklight v4.1.0 / GeoBlacklight Admin v0.3.0 From cba2dd400e6f021f2c2e8822718e78eef1a99509 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 09:30:24 -0600 Subject: [PATCH 02/15] Image Harvest: skip :failed and :placeheld states We don't need to attempt image harvesting for placeheld or failed items. Addresses #465 --- config/schedule.rb | 2 +- lib/tasks/geoportal/geoportal_images.rake | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/config/schedule.rb b/config/schedule.rb index 059ad6789..491ff5492 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -11,7 +11,7 @@ end # Harvest thumbnail images for search results every :day, at: '12:05am', roles: [:app] do - rake 'gblsci:images:harvest_retry' + rake 'geoportal:queue_incomplete_states' end every :day, at: '12:30am', roles: [:app] do rake 'sitemap:refresh' diff --git a/lib/tasks/geoportal/geoportal_images.rake b/lib/tasks/geoportal/geoportal_images.rake index fc9843aa9..ea823b40f 100644 --- a/lib/tasks/geoportal/geoportal_images.rake +++ b/lib/tasks/geoportal/geoportal_images.rake @@ -27,7 +27,25 @@ namespace :geoportal do desc 'Harvest images - Queue incomplete states for reprocessing' task queue_incomplete_states: :environment do - puts "Deprecated / Instead try: bundle exec rake gblsci:images:harvest_retry" + # Skipping :failed, :placeheld + states = [ + :initialized, + :queued, + :processing + ] + + states.each do |state| + sidecars = SolrDocumentSidecar.in_state(state) + + puts "#{state} - #{sidecars.size}" + + sidecars.each do |sc| + document = Geoblacklight::SolrDocument.find(sc.document_id) + GeoblacklightSidecarImages::StoreImageJob.perform_later(document.id) + rescue + puts "orphaned / #{sc.document_id}" + end + end end desc 'Failed State - Inspect metadata' From 07d404d4d33f9fc864a9bf71d2455fb5ce2e59d7 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 09:35:53 -0600 Subject: [PATCH 03/15] Dev mode: stick to blacklight-core --- lib/tasks/geoportal.rake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/tasks/geoportal.rake b/lib/tasks/geoportal.rake index 29679aedb..9929e7c67 100644 --- a/lib/tasks/geoportal.rake +++ b/lib/tasks/geoportal.rake @@ -24,9 +24,9 @@ namespace :geoportal do shared_solr_opts = { managed: true, verbose: true, persist: false, download_dir: 'tmp' } shared_solr_opts[:version] = ENV['SOLR_VERSION'] if ENV['SOLR_VERSION'] - SolrWrapper.wrap(shared_solr_opts.merge(port: 8983, instance_dir: 'tmp/geoportal-core-development')) do |solr| - solr.with_collection(name: "geoportal-core-development", dir: Rails.root.join("solr", "conf").to_s) do - puts "Solr running at http://localhost:8983/solr/geoportal-core-development/, ^C to exit" + SolrWrapper.wrap(shared_solr_opts.merge(port: 8983, instance_dir: 'tmp/blacklight-core')) do |solr| + solr.with_collection(name: "blacklight-core", dir: Rails.root.join("solr", "conf").to_s) do + puts "Solr running at http://localhost:8983/solr/blacklight-core/, ^C to exit" puts ' ' begin Rake::Task['geoportal:index:seed'].invoke From 289c3cdf6fc82a80c071f90efb44beed92ca517f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 09:44:43 -0600 Subject: [PATCH 04/15] ItemViewer: move IIIF manifests above IIIF in viewer preference selection order Fixes #534 --- config/initializers/item_viewer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/item_viewer.rb b/config/initializers/item_viewer.rb index a8a8f943e..09db7de9b 100644 --- a/config/initializers/item_viewer.rb +++ b/config/initializers/item_viewer.rb @@ -86,8 +86,8 @@ def viewer_preference oembed, index_map, wms, - iiif, iiif_manifest, + iiif, b1g_image, download, tiled_map_layer, From 72281f6dcd0f773edef241d3502ef280f6826590 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 13:22:59 -0600 Subject: [PATCH 05/15] CI: try new ci task setup --- .github/workflows/ci.yml | 4 +++- lib/tasks/geoportal.rake | 17 +++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b58374f63..accfaa081 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,12 +69,14 @@ jobs: bin/rails db:migrate RAILS_ENV=test - name: Rails - Run tests - run: bundle exec rake ci env: + RAILS_ENV: test rails_version: 6.1.5 ENGINE_CART_RAILS_OPTIONS: '--skip-git --skip-listen --skip-spring --skip-keeps --skip-action-cable --skip-coffee --skip-test' SOLR_URL: http://solr:SolrRocks@localhost:8983/solr/blacklight-core FARADAY_VERSION: ${{ matrix.faraday_version }} + run: bundle exec rake ci + - name: Artifacts - Upload coverage uses: actions/upload-artifact@v2 diff --git a/lib/tasks/geoportal.rake b/lib/tasks/geoportal.rake index 9929e7c67..03a9484d9 100644 --- a/lib/tasks/geoportal.rake +++ b/lib/tasks/geoportal.rake @@ -1,19 +1,12 @@ # frozen_string_literal: true -desc 'Run test suite' task :ci do - shared_solr_opts = { managed: true, verbose: true, persist: false, download_dir: 'tmp' } - shared_solr_opts[:version] = ENV['SOLR_VERSION'] if ENV['SOLR_VERSION'] - - success = true - SolrWrapper.wrap(shared_solr_opts.merge(port: 8985, instance_dir: 'tmp/geoportal-core-test')) do |solr| - solr.with_collection(name: "geoportal-core-test", dir: Rails.root.join("solr", "conf").to_s) do - system 'RAILS_ENV=test bundle exec rake geoportal:index:seed' - system('RAILS_ENV=test bundle exec rails test:system test') || success = false - end + if Rails.env.test? + success = true + Rake::Task['geoportal:index:seed'].invoke + system('RAILS_ENV=test bundle exec rails test:system test') || success = false + exit!(1) unless success end - - exit!(1) unless success end namespace :geoportal do From 87111145648c89e1e09da85c25288c6f93288aee Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 13:29:40 -0600 Subject: [PATCH 06/15] Update ci.yml --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index accfaa081..7704a5d1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,9 +35,6 @@ jobs: with: ruby-version: ${{ matrix.ruby }} - - name: Solr - Create container - run: docker run -d -p 8983:8983 geoblacklight/solr:8.9-v1.0.0 server/scripts/ci-start.sh - - name: Bundler - Install run: gem install bundler -v 2.3.5 @@ -77,7 +74,6 @@ jobs: FARADAY_VERSION: ${{ matrix.faraday_version }} run: bundle exec rake ci - - name: Artifacts - Upload coverage uses: actions/upload-artifact@v2 if: always() From 6854d300c3aafa5f1e98691b352a0bd5736f1bcc Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 13:34:56 -0600 Subject: [PATCH 07/15] Update ci.yml --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7704a5d1c..419dddae9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,9 @@ jobs: with: ruby-version: ${{ matrix.ruby }} + - name: Solr - Create container + run: docker run -d -p 8983:8983 geoblacklight/solr:8.9-v1.0.0 server/scripts/ci-start.sh + - name: Bundler - Install run: gem install bundler -v 2.3.5 From f645eebda29451860b35f4e66292ddee5d41f1b7 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 13:38:50 -0600 Subject: [PATCH 08/15] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 419dddae9..a5aa28e66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: ENGINE_CART_RAILS_OPTIONS: '--skip-git --skip-listen --skip-spring --skip-keeps --skip-action-cable --skip-coffee --skip-test' SOLR_URL: http://solr:SolrRocks@localhost:8983/solr/blacklight-core FARADAY_VERSION: ${{ matrix.faraday_version }} - run: bundle exec rake ci + run: bundle exec rake ci --trace - name: Artifacts - Upload coverage uses: actions/upload-artifact@v2 From 48c9735110d685885f3858f5e32775e4ee60eebf Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 13:46:01 -0600 Subject: [PATCH 09/15] CI: rake geoportal:ci GH Actions was calling rake ci, which was returning GBL Admin's rake ci task; This change nests our local CI task under geoportal:ci to make sure "our" ci task is the one running. --- .github/workflows/ci.yml | 2 +- lib/tasks/geoportal.rake | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5aa28e66..e856a1d9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: ENGINE_CART_RAILS_OPTIONS: '--skip-git --skip-listen --skip-spring --skip-keeps --skip-action-cable --skip-coffee --skip-test' SOLR_URL: http://solr:SolrRocks@localhost:8983/solr/blacklight-core FARADAY_VERSION: ${{ matrix.faraday_version }} - run: bundle exec rake ci --trace + run: bundle exec rake geoportal:ci --trace - name: Artifacts - Upload coverage uses: actions/upload-artifact@v2 diff --git a/lib/tasks/geoportal.rake b/lib/tasks/geoportal.rake index 03a9484d9..399cf6e3d 100644 --- a/lib/tasks/geoportal.rake +++ b/lib/tasks/geoportal.rake @@ -1,15 +1,14 @@ # frozen_string_literal: true - -task :ci do - if Rails.env.test? - success = true - Rake::Task['geoportal:index:seed'].invoke - system('RAILS_ENV=test bundle exec rails test:system test') || success = false - exit!(1) unless success +namespace :geoportal do + task :ci do + if Rails.env.test? + success = true + Rake::Task['geoportal:index:seed'].invoke + system('RAILS_ENV=test bundle exec rails test:system test') || success = false + exit!(1) unless success + end end -end -namespace :geoportal do desc 'Run Solr and GeoBlacklight for interactive development' task :server, [:rails_server_args] do require 'solr_wrapper' From f1b2514ec695fab43b56385a83d02aa4762fe9d5 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 14:28:13 -0600 Subject: [PATCH 10/15] Update home.js --- app/assets/javascripts/geoportal/modules/home.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/geoportal/modules/home.js b/app/assets/javascripts/geoportal/modules/home.js index 9e62487ad..eed15b47f 100644 --- a/app/assets/javascripts/geoportal/modules/home.js +++ b/app/assets/javascripts/geoportal/modules/home.js @@ -49,7 +49,7 @@ Blacklight.onLoad(function() { // Oboe - SAX steam JSON results from Solr /export // oboe('http://localhost:8983/solr/geoportal/export?fl=uuid_sdv,dc_title_sdv,centroid_sdv&indent=on&q=*:*&wt=json&sort=dc_title_sdv%20asc&rows=10000') - oboe('/centroids.json') + oboe('./centroids.json') .node('*', function( doc ){ if(typeof doc.c != 'undefined'){ var latlng = doc.c.split(",") From c7593c4c6f5ce851aa2de0f2c178a0678c844095 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 6 Nov 2023 15:08:53 -0600 Subject: [PATCH 11/15] Update routes.rb --- config/routes.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/routes.rb b/config/routes.rb index eb5c99bb1..2aea4eb29 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -2,6 +2,7 @@ get 'about', :to => redirect('https://sites.google.com/umn.edu/btaa-gdp/about') get 'help', :to => redirect('https://sites.google.com/umn.edu/btaa-gdp/help') get 'robots.:format' => 'robots#robots' + get '/centroids.json', :to => redirect('/centroids.json') # Feedback mount PointlessFeedback::Engine, :at => '/feedback' From 58c725371c6cfbea4f0416587022050aafbe7b50 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 10 Nov 2023 13:45:35 -0600 Subject: [PATCH 12/15] CatalogController: remove Carto show tools link Fixes #528 --- app/controllers/catalog_controller.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 825f1f70e..0ae0dddfd 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -311,7 +311,6 @@ class CatalogController < ApplicationController # Custom tools for GeoBlacklight config.add_show_tools_partial :more_details, partial: 'more_details', if: proc { |_context, _config, options| options[:document] && (!options[:document].references.nil? & !options[:document].references.url.nil?)} config.add_show_tools_partial :metadata, if: proc { |_context, _config, options| options[:document] && (Settings.METADATA_SHOWN & options[:document].references.refs.map(&:type).map(&:to_s)).any? } - config.add_show_tools_partial :carto, partial: 'carto', if: proc { |_context, _config, options| options[:document] && options[:document].carto_reference.present? } config.add_show_tools_partial :arcgis, partial: 'arcgis', if: proc { |_context, _config, options| options[:document] && options[:document].arcgis_urls.present? } config.add_show_tools_partial :data_dictionary, partial: 'data_dictionary', if: proc { |_context, _config, options| options[:document] && options[:document].data_dictionary_download.present? } config.add_show_tools_partial(:citation, if: proc { |_context, _config, options| options[:document] && (options[:document].resource_class != 'Collections')}) From 8c01f86c8bc9aa27e39fdbc297129338d96b5306 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 10 Nov 2023 13:48:13 -0600 Subject: [PATCH 13/15] Schedule: remove geoportal:export_data job Fixes #546 --- config/schedule.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/config/schedule.rb b/config/schedule.rb index 491ff5492..62492adce 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -24,10 +24,13 @@ every :day, at: '2:30am', roles: [:app] do rake 'blacklight:delete_old_searches[7]' end + +# No longer required - EWL 11/10/23 # Exports SOLR data to public/data.json -every '30 3 1 * *', roles: [:app] do - rake 'geoportal:export_data' -end +# every '30 3 1 * *', roles: [:app] do +# rake 'geoportal:export_data' +# end + # Clean Carrierwave tmp file directory every :day, at: '4:30 am', roles: [:app] do runner "CarrierWave.clean_cached_files!" From 3f7cc9d4c0bde7eafc90f240e9e34bd94a4fd3d3 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 10 Nov 2023 13:59:09 -0600 Subject: [PATCH 14/15] JS Behaviors: add linkify to .gbl-display-note elements Fixes #529 --- app/views/shared/_javascript_behaviors.html.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/views/shared/_javascript_behaviors.html.erb b/app/views/shared/_javascript_behaviors.html.erb index b6003cd55..37c84bb28 100644 --- a/app/views/shared/_javascript_behaviors.html.erb +++ b/app/views/shared/_javascript_behaviors.html.erb @@ -1,5 +1,6 @@