From aeac3ffcdbe4d19ef439539083e02d21dc17968d Mon Sep 17 00:00:00 2001 From: dleadbetter <> Date: Mon, 10 Jun 2024 08:51:31 -0400 Subject: [PATCH 01/12] Archnet #1117 - Adding instructions and files for Cantaloupe set up --- lib/iiif/README.md | 75 +++++++++++++++++++++++++++++++++++++ lib/iiif/cantaloupe.service | 17 +++++++++ lib/iiif/start.sh | 1 + 3 files changed, 93 insertions(+) create mode 100644 lib/iiif/README.md create mode 100644 lib/iiif/cantaloupe.service create mode 100644 lib/iiif/start.sh diff --git a/lib/iiif/README.md b/lib/iiif/README.md new file mode 100644 index 0000000..bb87e53 --- /dev/null +++ b/lib/iiif/README.md @@ -0,0 +1,75 @@ +# Cantaloupe IIIF +Instructions for setting up a Cantaloupe IIIF instance + +## Requirements +- Cantaloupe 5.0 +- Java 11+ + +## Install Dependencies + +```bash +sudo apt install default-jre unzip wget -y +``` + +## Download Cantaloupe + +```bash +sudo wget https://github.com/cantaloupe-project/cantaloupe/releases/download/v5.0.6/cantaloupe-5.0.6.zip +``` + +## Custom Delegate +Add the contents of `custom_delegate.rb` to: + +```bash +/root/cantaloupe-5.0.6/delegates.rb +``` + +## Modify cantaloupe.properties (Optional) +Set custom properties in `/root/cantaloupe-5.0.6/cantalouple.properties` + +### Enable Admin Console + +```bash +endpoint.admin.enabled = true +endpoint.admin.username = admin +endpoint.admin.secret = +``` + +### Enable Delegate Script + +```bash +delegate_script.enabled = true +``` + +## Create a service +Cantaloupe run as a Java application. To set up Cantaloupe to run as a service on Ubuntu, add the `cantaloupe.service` file to: + +``` +/etc/systemd/system/cantaloupe.service +``` + +## Create start.sh +Add the `start.sh` file to `/root`. Change the Cantaloupe version if necessary. + +## Enable/start the service + +```bash +sudo systemctl daemon-reload +sudo systemctl enable cantaloupe.service +sudo systemctl start cantaloupe +sudo systemctl status cantaloupe +``` + +## Logging + +### Setup + +```bash +sudo journalctl --unit=cantaloupe +``` + +### View + +```bash +sudo journalctl -f -u cantaloupe +``` \ No newline at end of file diff --git a/lib/iiif/cantaloupe.service b/lib/iiif/cantaloupe.service new file mode 100644 index 0000000..4dd9371 --- /dev/null +++ b/lib/iiif/cantaloupe.service @@ -0,0 +1,17 @@ +[Unit] +Description=Cantaloupe IIIF Image Server Service + +[Service] +User=root + +# The configuration file application.properties should be here: + +#change this to your workspace +WorkingDirectory=/root + +#path to executable. +#executable is a bash script which calls jar file +ExecStart=/bin/bash -c "/root/start.sh" + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/lib/iiif/start.sh b/lib/iiif/start.sh new file mode 100644 index 0000000..4719c5e --- /dev/null +++ b/lib/iiif/start.sh @@ -0,0 +1 @@ +java -Dcantaloupe.config=/root/cantaloupe-5.0.5/cantaloupe.properties -Xmx6g -jar /root/cantaloupe-5.0.5/cantaloupe-5.0.5.jar \ No newline at end of file From 50e46ad3673f9810431360f739fdccda3b6602d8 Mon Sep 17 00:00:00 2001 From: dleadbetter <> Date: Mon, 10 Jun 2024 08:52:36 -0400 Subject: [PATCH 02/12] Archnet #1117 - Adding executable privileges to start.sh --- lib/iiif/start.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 lib/iiif/start.sh diff --git a/lib/iiif/start.sh b/lib/iiif/start.sh old mode 100644 new mode 100755 From 664a73fb7466d18fc97dc21b582afa70be2c3838 Mon Sep 17 00:00:00 2001 From: dleadbetter <> Date: Wed, 12 Jun 2024 11:50:08 -0400 Subject: [PATCH 03/12] Archnet #1117 - Updating resources.uuid from "string" to "uuid" and setting default value --- db/migrate/20240612152139_enable_pgcrypto.rb | 5 +++++ db/migrate/20240612152246_set_resource_uuid_default_value.rb | 5 +++++ db/schema.rb | 5 +++-- 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20240612152139_enable_pgcrypto.rb create mode 100644 db/migrate/20240612152246_set_resource_uuid_default_value.rb diff --git a/db/migrate/20240612152139_enable_pgcrypto.rb b/db/migrate/20240612152139_enable_pgcrypto.rb new file mode 100644 index 0000000..9573ea3 --- /dev/null +++ b/db/migrate/20240612152139_enable_pgcrypto.rb @@ -0,0 +1,5 @@ +class EnablePgcrypto < ActiveRecord::Migration[7.0] + def change + enable_extension 'pgcrypto' + end +end diff --git a/db/migrate/20240612152246_set_resource_uuid_default_value.rb b/db/migrate/20240612152246_set_resource_uuid_default_value.rb new file mode 100644 index 0000000..c55be83 --- /dev/null +++ b/db/migrate/20240612152246_set_resource_uuid_default_value.rb @@ -0,0 +1,5 @@ +class SetResourceUuidDefaultValue < ActiveRecord::Migration[7.0] + def change + change_column :resources, :uuid, :uuid, default: 'gen_random_uuid()', using: 'uuid::uuid' + end +end diff --git a/db/schema.rb b/db/schema.rb index 5c05a06..92f6dc8 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,8 +10,9 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_10_25_202507) do +ActiveRecord::Schema[7.0].define(version: 2024_06_12_152246) do # These are extensions that must be enabled in order to support this database + enable_extension "pgcrypto" enable_extension "plpgsql" create_table "active_storage_attachments", force: :cascade do |t| @@ -94,7 +95,7 @@ t.text "exif" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "uuid" + t.uuid "uuid", default: -> { "gen_random_uuid()" } t.text "manifest" t.jsonb "user_defined", default: {} t.index ["project_id"], name: "index_resources_on_project_id" From 093b6053ac47ab23147027c30b91ae19947d414a Mon Sep 17 00:00:00 2001 From: dleadbetter <> Date: Mon, 17 Jun 2024 09:40:32 -0400 Subject: [PATCH 04/12] Archnet #1117 - Fixing a bug where /resources/:id/iiif was requiring authentication --- app/controllers/public/resources_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/public/resources_controller.rb b/app/controllers/public/resources_controller.rb index 461abb0..a315921 100644 --- a/app/controllers/public/resources_controller.rb +++ b/app/controllers/public/resources_controller.rb @@ -6,7 +6,7 @@ class Public::ResourcesController < Api::ResourcesController prepend_before_action :set_project_id, only: :index prepend_before_action :set_resource_id, only: [:show, :destroy, :update] prepend_before_action :set_resource_project_id, only: [:create, :update] - skip_before_action :authenticate_request, only: [:content, :download, :inline, :manifest, :preview, :thumbnail] + skip_before_action :authenticate_request, only: [:content, :download, :iiif, :inline, :manifest, :preview, :thumbnail] def content redirect_resource :content_url From 2db1db50cd733efffe4a5e5324f1917b99402e53 Mon Sep 17 00:00:00 2001 From: dleadbetter <> Date: Mon, 17 Jun 2024 09:41:13 -0400 Subject: [PATCH 05/12] Archnet #1117 - Adding rake tasks for converting all images to pyramidal tiffs and extracting EXIF data from images --- lib/tasks/iiif.rake | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/tasks/iiif.rake b/lib/tasks/iiif.rake index 720b2f7..a089bfb 100644 --- a/lib/tasks/iiif.rake +++ b/lib/tasks/iiif.rake @@ -1,5 +1,14 @@ namespace :iiif do + desc 'Converts the source images to pyramidal TIFFs for all resources' + task convert_images: :environment do + Resource.all.in_batches do |resources| + resources.pluck(:id).each do |resource_id| + ConvertImageJob.perform_now(resource_id) + end + end + end + desc 'Creates a new IIIF manifest for all resources' task create_manifests: :environment do Resource.all.in_batches do |resources| @@ -9,6 +18,15 @@ namespace :iiif do end end + desc 'Extracts the EXIF data from all resources' + task extract_exif: :environment do + Resource.all.in_batches do |resources| + resources.pluck(:id).each do |resource_id| + ExtractExifJob.perform_now(resource_id) + end + end + end + desc 'Transfers resources from one storage service to another' task transfer_resources: :environment do # Parse the arguments From f9673262abf27251d253df464f0e164ae7f9cddd Mon Sep 17 00:00:00 2001 From: dleadbetter <> Date: Mon, 17 Jun 2024 09:42:02 -0400 Subject: [PATCH 06/12] Archnet #1117 - Updating resources_controller#redirect_resource to return 404 if resource or URL attribute is nil --- app/controllers/public/resources_controller.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/controllers/public/resources_controller.rb b/app/controllers/public/resources_controller.rb index a315921..70761e3 100644 --- a/app/controllers/public/resources_controller.rb +++ b/app/controllers/public/resources_controller.rb @@ -41,7 +41,12 @@ def thumbnail def redirect_resource(attribute) resource = Resource.find_by_uuid(params[:id]) - redirect_to resource.send(attribute), allow_other_host: true + render status: :not_found and return if resource.nil? + + redirect = resource.send(attribute) + render status: :not_found and return if redirect.nil? + + redirect_to redirect, allow_other_host: true end def set_project_id From ddcad81678edf34735864b1dca2c4dc0e1442953 Mon Sep 17 00:00:00 2001 From: dleadbetter <> Date: Mon, 24 Jun 2024 07:16:17 -0400 Subject: [PATCH 07/12] Archnet #1117 - Updating preview_url and thumbnail_url properties to use "^" modifier to allow upscaling --- app/models/concerns/attachable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/concerns/attachable.rb b/app/models/concerns/attachable.rb index 4bf9e61..41cdb04 100644 --- a/app/models/concerns/attachable.rb +++ b/app/models/concerns/attachable.rb @@ -103,7 +103,7 @@ def generate_url_method(name) return nil if attachment.audio? - "#{self.send("#{name}_base_url")}/full/^500,/0/default.jpg" + "#{self.send("#{name}_base_url")}/full/^!500,500/0/default.jpg" end define_method("#{name}_thumbnail_url") do @@ -112,7 +112,7 @@ def generate_url_method(name) return nil if attachment.audio? - "#{self.send("#{name}_base_url")}/square/!250,250/0/default.jpg" + "#{self.send("#{name}_base_url")}/square/^!250,250/0/default.jpg" end end From 662d0abd474c07b460442e8bcf2b29373c7ba117 Mon Sep 17 00:00:00 2001 From: dleadbetter Date: Tue, 25 Jun 2024 07:20:21 -0400 Subject: [PATCH 08/12] Archnet #1117 - Adding error logging to ConvertImageJob --- app/jobs/convert_image_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/jobs/convert_image_job.rb b/app/jobs/convert_image_job.rb index 311df9a..24e2cfa 100644 --- a/app/jobs/convert_image_job.rb +++ b/app/jobs/convert_image_job.rb @@ -21,8 +21,10 @@ def perform(resource_id) content_type: 'image/tiff' ) - rescue MiniMagick::Error + rescue MiniMagick::Error => e # Content cannot be converted + Rails.logger.error e.message + Rails.logger.error e.backtrace.join("\n") end end end From 4712518b99b044e32cab2dd1c715e39c2b8a0519 Mon Sep 17 00:00:00 2001 From: dleadbetter Date: Tue, 25 Jun 2024 09:14:46 -0400 Subject: [PATCH 09/12] Archnet #1117 - Updating rake tasks to include version which runs for only empty resources --- lib/tasks/iiif.rake | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/lib/tasks/iiif.rake b/lib/tasks/iiif.rake index a089bfb..7c52fd4 100644 --- a/lib/tasks/iiif.rake +++ b/lib/tasks/iiif.rake @@ -9,6 +9,24 @@ namespace :iiif do end end + desc 'Converts the source images to pyramidal TIFFs for resources with no converted content' + task convert_images_empty: :environment do + query = Resource + .where.not( + ActiveStorage::Attachment + .where(ActiveStorage::Attachment.arel_table[:record_id].eq(Resource.arel_table[:id])) + .where(record_type: Resource.to_s, name: 'content_converted') + .arel + .exists + ) + + query.in_batches do |resources| + resources.pluck(:id).each do |resource_id| + ConvertImageJob.perform_now(resource_id) + end + end + end + desc 'Creates a new IIIF manifest for all resources' task create_manifests: :environment do Resource.all.in_batches do |resources| @@ -18,6 +36,15 @@ namespace :iiif do end end + desc 'Creates a new IIIF manifest for resources with no generated manifest' + task create_manifests_empty: :environment do + Resource.where(manifest: nil).in_batches do |resources| + resources.pluck(:id).each do |resource_id| + CreateManifestJob.perform_now(resource_id) + end + end + end + desc 'Extracts the EXIF data from all resources' task extract_exif: :environment do Resource.all.in_batches do |resources| @@ -27,6 +54,15 @@ namespace :iiif do end end + desc 'Extracts the EXIF data from resources with no EXIF data' + task extract_exif_empty: :environment do + Resource.where(exif: nil).in_batches do |resources| + resources.pluck(:id).each do |resource_id| + ExtractExifJob.perform_now(resource_id) + end + end + end + desc 'Transfers resources from one storage service to another' task transfer_resources: :environment do # Parse the arguments From b259cea665a688239439c744575473d92dcfa52b Mon Sep 17 00:00:00 2001 From: dleadbetter Date: Tue, 25 Jun 2024 12:58:22 -0400 Subject: [PATCH 10/12] Archnet #1117 - Adding CORS rule for "/public/resources/:id/content" --- config/initializers/cors.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/initializers/cors.rb b/config/initializers/cors.rb index 3e130d6..daabfa7 100644 --- a/config/initializers/cors.rb +++ b/config/initializers/cors.rb @@ -9,6 +9,7 @@ allow do origins '*' resource '/public/resources/:id/manifest', headers: :any, methods: :get + resource '/public/resources/:id/content', headers: :any, methods: :get resource '*/rails/active_storage/blobs/redirect/*', headers: :any, methods: :get end end From ed737bf6532ec5993268a9d655ddeb4ae13e127d Mon Sep 17 00:00:00 2001 From: dleadbetter Date: Thu, 27 Jun 2024 14:24:59 -0400 Subject: [PATCH 11/12] Archnet #1117 - Adding ffmpeg dependency to IIIF README.md --- lib/iiif/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/iiif/README.md b/lib/iiif/README.md index bb87e53..e0b19fd 100644 --- a/lib/iiif/README.md +++ b/lib/iiif/README.md @@ -4,11 +4,12 @@ Instructions for setting up a Cantaloupe IIIF instance ## Requirements - Cantaloupe 5.0 - Java 11+ +- FFMPEG ## Install Dependencies ```bash -sudo apt install default-jre unzip wget -y +sudo apt install default-jre unzip wget ffmpeg -y ``` ## Download Cantaloupe From 9e48e03dfb0db109339ae764fb1253afb3c2538a Mon Sep 17 00:00:00 2001 From: dleadbetter Date: Mon, 8 Jul 2024 09:22:28 -0400 Subject: [PATCH 12/12] Archnet #1117 - Adding restart to the cantaloupe.service --- lib/iiif/cantaloupe.service | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/iiif/cantaloupe.service b/lib/iiif/cantaloupe.service index 4dd9371..2287b27 100644 --- a/lib/iiif/cantaloupe.service +++ b/lib/iiif/cantaloupe.service @@ -6,12 +6,15 @@ User=root # The configuration file application.properties should be here: -#change this to your workspace +# Change this to your workspace WorkingDirectory=/root -#path to executable. -#executable is a bash script which calls jar file +# Path to executable. Executable is a bash script which calls jar file ExecStart=/bin/bash -c "/root/start.sh" +# Restart the service on failure +Restart=always +RestartSec=3 + [Install] WantedBy=multi-user.target \ No newline at end of file