From 53d1862b839dcdc51f4ee97cc4bbe2c44ccbcb13 Mon Sep 17 00:00:00 2001 From: Pavel Balashou Date: Fri, 30 Jun 2023 14:10:04 +0200 Subject: [PATCH] [#48717] Replace DelayedJob with GoodJob. https://community.openproject.org/work_packages/48717 --- Gemfile | 3 +- Gemfile.lock | 20 ++--- .../attachments/cleanup_uncontainered_job.rb | 7 +- app/workers/cron/clear_old_sessions_job.rb | 5 +- app/workers/cron/clear_tmp_cache_job.rb | 5 +- app/workers/cron/clear_uploaded_files_job.rb | 5 +- app/workers/cron/cron_job.rb | 76 ---------------- app/workers/ldap/synchronization_job.rb | 5 +- .../schedule_date_alerts_notifications_job.rb | 8 +- .../schedule_reminder_mails_job.rb | 5 +- app/workers/oauth/cleanup_job.rb | 5 +- app/workers/paper_trail_audits/cleanup_job.rb | 5 +- bin/delayed_job | 5 -- config/application.rb | 14 ++- config/initializers/cronjobs.rb | 47 +++++++--- config/initializers/database_pool_size.rb | 28 ++++++ config/initializers/delayed_job_config.rb | 55 ------------ config/initializers/time_with_zone_as_json.rb | 10 +-- config/routes.rb | 4 + db/migrate/20230627173959_create_good_jobs.rb | 89 +++++++++++++++++++ docker-compose.yml | 2 +- .../patches/delayed_job_adapter.rb | 48 ---------- lib/open_project/patches/delivery_job.rb | 40 --------- lib/open_project/plugins/acts_as_op_engine.rb | 6 +- lib/tasks/cron.rake | 41 --------- lib/tasks/delayed_job.rake | 43 --------- .../cron/clear_old_pull_requests_job.rb | 5 +- .../open_project/github_integration/engine.rb | 12 ++- .../cron/clear_old_job_status_job.rb | 4 +- .../lib/open_project/job_status/engine.rb | 12 ++- .../ldap_groups/synchronization_job.rb | 5 +- .../lib/open_project/ldap_groups/engine.rb | 9 +- .../cleanup_uncontainered_file_links_job.rb | 4 +- .../manage_nextcloud_integration_job.rb | 6 +- .../lib/open_project/storages/engine.rb | 14 ++- .../app/workers/cleanup_webhook_logs_job.rb | 5 +- .../lib/open_project/webhooks/engine.rb | 9 +- 37 files changed, 248 insertions(+), 418 deletions(-) delete mode 100644 app/workers/cron/cron_job.rb delete mode 100755 bin/delayed_job delete mode 100644 config/initializers/delayed_job_config.rb create mode 100644 db/migrate/20230627173959_create_good_jobs.rb delete mode 100644 lib/open_project/patches/delayed_job_adapter.rb delete mode 100644 lib/open_project/patches/delivery_job.rb delete mode 100644 lib/tasks/cron.rake delete mode 100644 lib/tasks/delayed_job.rake diff --git a/Gemfile b/Gemfile index 952f07bac165..e03223790a69 100644 --- a/Gemfile +++ b/Gemfile @@ -123,8 +123,7 @@ gem 'multi_json', '~> 1.15.0' gem 'oj', '~> 3.15.0' gem 'daemons' -gem 'delayed_cron_job', '~> 0.9.0' -gem 'delayed_job_active_record', '~> 4.1.5' +gem 'good_job' gem 'rack-protection', '~> 3.0.0' diff --git a/Gemfile.lock b/Gemfile.lock index 95b2b0e7ad0c..ee252cdedf72 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -388,13 +388,6 @@ GEM deckar01-task_list (2.3.2) html-pipeline declarative (0.0.20) - delayed_cron_job (0.9.0) - fugit (>= 1.5) - delayed_job (4.1.11) - activesupport (>= 3.0, < 8.0) - delayed_job_active_record (4.1.7) - activerecord (>= 3.0, < 8.0) - delayed_job (>= 3.0, < 5) diff-lcs (1.5.0) disposable (0.6.3) declarative (>= 0.0.9, < 1.0.0) @@ -488,6 +481,14 @@ GEM i18n (>= 0.7) multi_json request_store (>= 1.0) + good_job (3.15.13) + activejob (>= 6.0.0) + activerecord (>= 6.0.0) + concurrent-ruby (>= 1.0.2) + fugit (>= 1.1) + railties (>= 6.0.0) + thor (>= 0.14.1) + webrick (>= 1.3) google-apis-core (0.11.0) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) @@ -1017,8 +1018,6 @@ DEPENDENCIES date_validator (~> 0.12.0) debug deckar01-task_list (~> 2.3.1) - delayed_cron_job (~> 0.9.0) - delayed_job_active_record (~> 4.1.5) disposable (~> 0.6.2) doorkeeper (~> 5.6.6) dotenv-rails @@ -1033,6 +1032,7 @@ DEPENDENCIES friendly_id (~> 5.5.0) fuubar (~> 2.5.0) gon (~> 6.4.0) + good_job google-apis-gmail_v1 googleauth grape (~> 1.7.0) @@ -1165,4 +1165,4 @@ RUBY VERSION ruby 3.2.1p31 BUNDLED WITH - 2.4.7 + 2.4.6 diff --git a/app/workers/attachments/cleanup_uncontainered_job.rb b/app/workers/attachments/cleanup_uncontainered_job.rb index 5f94daaa428f..ce7fc94eada4 100644 --- a/app/workers/attachments/cleanup_uncontainered_job.rb +++ b/app/workers/attachments/cleanup_uncontainered_job.rb @@ -26,12 +26,9 @@ # See COPYRIGHT and LICENSE files for more details. #++ -class Attachments::CleanupUncontaineredJob < Cron::CronJob +class Attachments::CleanupUncontaineredJob < ApplicationJob queue_with_priority :low - # runs at 10:03 pm - self.cron_expression = '03 22 * * *' - def perform Attachment .where(container: nil) @@ -50,7 +47,7 @@ def too_old attachment_table = Attachment.arel_table attachment_table[:created_at] - .lteq(Time.now - OpenProject::Configuration.attachments_grace_period.minutes) + .lteq(Time.zone.now - OpenProject::Configuration.attachments_grace_period.minutes) .to_sql end end diff --git a/app/workers/cron/clear_old_sessions_job.rb b/app/workers/cron/clear_old_sessions_job.rb index 95b9536afc38..53d9ea745719 100644 --- a/app/workers/cron/clear_old_sessions_job.rb +++ b/app/workers/cron/clear_old_sessions_job.rb @@ -27,12 +27,9 @@ #++ module Cron - class ClearOldSessionsJob < CronJob + class ClearOldSessionsJob < ApplicationJob include ::RakeJob - # runs at 1:15 nightly - self.cron_expression = '15 1 * * *' - def perform super 'db:sessions:expire', 7 end diff --git a/app/workers/cron/clear_tmp_cache_job.rb b/app/workers/cron/clear_tmp_cache_job.rb index c8a83e16b4cd..467505598f61 100644 --- a/app/workers/cron/clear_tmp_cache_job.rb +++ b/app/workers/cron/clear_tmp_cache_job.rb @@ -27,12 +27,9 @@ #++ module Cron - class ClearTmpCacheJob < CronJob + class ClearTmpCacheJob < ApplicationJob include ::RakeJob - # runs at 02:45 sundays - self.cron_expression = '45 2 * * 7' - def perform super 'tmp:cache:clear' end diff --git a/app/workers/cron/clear_uploaded_files_job.rb b/app/workers/cron/clear_uploaded_files_job.rb index fc2bf251f001..83a58c296459 100644 --- a/app/workers/cron/clear_uploaded_files_job.rb +++ b/app/workers/cron/clear_uploaded_files_job.rb @@ -27,12 +27,9 @@ #++ module Cron - class ClearUploadedFilesJob < CronJob + class ClearUploadedFilesJob < ApplicationJob include ::RakeJob - # Runs 23pm fridays - self.cron_expression = '0 23 * * 5' - def perform super 'attachments:clear' end diff --git a/app/workers/cron/cron_job.rb b/app/workers/cron/cron_job.rb deleted file mode 100644 index 31369eaef547..000000000000 --- a/app/workers/cron/cron_job.rb +++ /dev/null @@ -1,76 +0,0 @@ -#-- copyright -# OpenProject is an open source project management software. -# Copyright (C) 2012-2023 the OpenProject GmbH -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License version 3. -# -# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: -# Copyright (C) 2006-2013 Jean-Philippe Lang -# Copyright (C) 2010-2013 the ChiliProject Team -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# See COPYRIGHT and LICENSE files for more details. -#++ - -module Cron - class CronJob < ApplicationJob - class_attribute :cron_expression - - # List of registered jobs, requires eager load in dev(!) - class_attribute :registered_jobs, default: [] - - include ScheduledJob - - class << self - ## - # Register new job class(es) - def register!(*job_classes) - Array(job_classes).each do |clz| - raise ArgumentError, "Needs to be subclass of ::Cron::CronJob" unless clz.ancestors.include?(self) - - registered_jobs << clz - end - end - - def schedule_registered_jobs! - registered_jobs.each do |job_class| - job_class.ensure_scheduled! - end - end - - ## - # Ensure the job is scheduled unless it is already - def ensure_scheduled! - # Ensure scheduled only once - return if scheduled? - - Rails.logger.info { "Scheduling #{name} recurrent background job." } - set(cron: cron_expression).perform_later - end - - ## - # Remove the scheduled job, if any - def remove - delayed_job&.destroy - end - - def delayed_job - delayed_job_query.first - end - end - end -end diff --git a/app/workers/ldap/synchronization_job.rb b/app/workers/ldap/synchronization_job.rb index 81dfbe4710b4..e0335c1d15a0 100644 --- a/app/workers/ldap/synchronization_job.rb +++ b/app/workers/ldap/synchronization_job.rb @@ -27,10 +27,7 @@ #++ module Ldap - class SynchronizationJob < ::Cron::CronJob - # Run once per night at 11:30pm - self.cron_expression = '30 23 * * *' - + class SynchronizationJob < ApplicationJob def perform run_user_sync end diff --git a/app/workers/notifications/schedule_date_alerts_notifications_job.rb b/app/workers/notifications/schedule_date_alerts_notifications_job.rb index dc5406c788ba..dd1355030535 100644 --- a/app/workers/notifications/schedule_date_alerts_notifications_job.rb +++ b/app/workers/notifications/schedule_date_alerts_notifications_job.rb @@ -28,15 +28,11 @@ module Notifications # Creates date alert jobs for users whose local time is 1:00 am. - class ScheduleDateAlertsNotificationsJob < Cron::CronJob - # runs every quarter of an hour, so 00:00, 00:15,..., 15:30, 15:45, 16:00, ... - self.cron_expression = '*/15 * * * *' - + class ScheduleDateAlertsNotificationsJob < ApplicationJob def perform return unless EnterpriseToken.allows_to?(:date_alerts) - service = Service.new(times_from_scheduled_to_execution) - service.call + Service.new(times_from_scheduled_to_execution).call end # Returns times from scheduled execution time to current time in 15 minutes diff --git a/app/workers/notifications/schedule_reminder_mails_job.rb b/app/workers/notifications/schedule_reminder_mails_job.rb index a2d51b1ddabe..2dc47f8a909c 100644 --- a/app/workers/notifications/schedule_reminder_mails_job.rb +++ b/app/workers/notifications/schedule_reminder_mails_job.rb @@ -27,10 +27,7 @@ #++ module Notifications - class ScheduleReminderMailsJob < Cron::CronJob - # runs every quarter of an hour, so 00:00, 00:15... - self.cron_expression = '*/15 * * * *' - + class ScheduleReminderMailsJob < ApplicationJob def perform User.having_reminder_mail_to_send(run_at).pluck(:id).each do |user_id| Mails::ReminderJob.perform_later(user_id) diff --git a/app/workers/oauth/cleanup_job.rb b/app/workers/oauth/cleanup_job.rb index 9a39253e96b0..afa7f7385ce2 100644 --- a/app/workers/oauth/cleanup_job.rb +++ b/app/workers/oauth/cleanup_job.rb @@ -27,12 +27,9 @@ #++ module OAuth - class CleanupJob < ::Cron::CronJob + class CleanupJob < ApplicationJob include ::RakeJob - # runs at 1:52 nightly - self.cron_expression = '52 1 * * *' - queue_with_priority :low def perform diff --git a/app/workers/paper_trail_audits/cleanup_job.rb b/app/workers/paper_trail_audits/cleanup_job.rb index 67cfc7bfa83d..3806e21d2392 100644 --- a/app/workers/paper_trail_audits/cleanup_job.rb +++ b/app/workers/paper_trail_audits/cleanup_job.rb @@ -27,10 +27,7 @@ #++ module PaperTrailAudits - class CleanupJob < ::Cron::CronJob - # runs at 4:03 on Saturday - self.cron_expression = '3 4 * * 6' - + class CleanupJob < ApplicationJob # Clean any paper trails older than 60 days def perform ::PaperTrailAudit diff --git a/bin/delayed_job b/bin/delayed_job deleted file mode 100755 index edf195985f69..000000000000 --- a/bin/delayed_job +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env ruby - -require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment')) -require 'delayed/command' -Delayed::Command.new(ARGV).daemonize diff --git a/config/application.rb b/config/application.rb index 1d9f183ed37b..10ab55ce3fb4 100644 --- a/config/application.rb +++ b/config/application.rb @@ -134,7 +134,7 @@ class Application < Rails::Application # config.time_zone = 'Central Time (US & Canada)' # Add locales from crowdin translations to i18n - config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'crowdin', '*.{rb,yml}').to_s] + config.i18n.load_path += Dir[Rails.root.join("config/locales/crowdin/*.{rb,yml}").to_s] config.i18n.default_locale = :en # Fall back to default locale @@ -207,7 +207,17 @@ class Application < Rails::Application # This allows for setting the root either via config file or via environment variable. config.action_controller.relative_url_root = OpenProject::Configuration['rails_relative_url_root'] - config.active_job.queue_adapter = :delayed_job + config.active_job.queue_adapter = :good_job + + config.good_job.preserve_job_records = true + config.good_job.retry_on_unhandled_error = false + # config.good_job.on_thread_error = -> (exception) { Rails.error.report(exception) } + config.good_job.execution_mode = :external + config.good_job.queues = '*' + config.good_job.max_threads = 20 + config.good_job.poll_interval = 30 + config.good_job.shutdown_timeout = 25 + config.good_job.enable_cron = true config.action_controller.asset_host = OpenProject::Configuration::AssetHost.value diff --git a/config/initializers/cronjobs.rb b/config/initializers/cronjobs.rb index 086811b26f71..36e16edfcbe1 100644 --- a/config/initializers/cronjobs.rb +++ b/config/initializers/cronjobs.rb @@ -1,15 +1,40 @@ # Register "Cron-like jobs" OpenProject::Application.configure do |application| - application.config.to_prepare do - Cron::CronJob.register! Cron::ClearOldSessionsJob, - Cron::ClearTmpCacheJob, - Cron::ClearUploadedFilesJob, - OAuth::CleanupJob, - PaperTrailAudits::CleanupJob, - Attachments::CleanupUncontaineredJob, - Notifications::ScheduleDateAlertsNotificationsJob, - Notifications::ScheduleReminderMailsJob, - Ldap::SynchronizationJob - end + application.config.good_job.cron.merge!( + { + 'Cron::ClearOldSessionsJob': { + cron: '15 1 * * *', # runs at 1:15 nightly + class: 'Cron::ClearOldSessionsJob' + }, + 'Cron::ClearTmpCacheJob': { + cron: '45 2 * * 7', # runs at 02:45 sundays + class: 'Cron::ClearTmpCacheJob' + }, + 'Cron::ClearUploadedFilesJob': { + cron: '0 23 * * 5', # runs 23:00 fridays + class: 'Cron::ClearUploadedFilesJob' + }, + 'OAuth::CleanupJob': { + cron: '52 1 * * *', + class: 'OAuth::CleanupJob' + }, + 'PaperTrailAudits::CleanupJob': { + cron: '3 4 * * 6', + class: 'PaperTrailAudits::CleanupJob' + }, + 'Attachments::CleanupUncontaineredJob': { + cron: '03 22 * * *', # runs at 10:03 pm + class: 'Attachments::CleanupUncontaineredJob' + }, + 'Notifications::ScheduleDateAlertsNotificationsJob': { + cron: '*/15 * * * *', # runs every 15th minute + class: 'Notifications::ScheduleDateAlertsNotificationsJob' + }, + 'Notifications::ScheduleReminderMailsJob': { + cron: '*/15 * * * *', # runs every 15th minute + class: 'Notifications::ScheduleReminderMailsJob' + } + } + ) end diff --git a/config/initializers/database_pool_size.rb b/config/initializers/database_pool_size.rb index 8809f2439de9..ac707aeef26f 100644 --- a/config/initializers/database_pool_size.rb +++ b/config/initializers/database_pool_size.rb @@ -1,3 +1,31 @@ +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) 2012-2023 the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + config = Rails.env.production? && Rails.application.config.database_configuration[Rails.env] pool_size = config && [OpenProject::Configuration.web_max_threads + 1, config['pool'].to_i].max diff --git a/config/initializers/delayed_job_config.rb b/config/initializers/delayed_job_config.rb deleted file mode 100644 index c5dbfd6d6acd..000000000000 --- a/config/initializers/delayed_job_config.rb +++ /dev/null @@ -1,55 +0,0 @@ -#-- copyright -# OpenProject is an open source project management software. -# Copyright (C) 2012-2023 the OpenProject GmbH -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License version 3. -# -# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: -# Copyright (C) 2006-2013 Jean-Philippe Lang -# Copyright (C) 2010-2013 the ChiliProject Team -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# See COPYRIGHT and LICENSE files for more details. -#++ - -# Disable delayed_job's own logging as we have activejob -Delayed::Worker.logger = nil - -# By default bypass worker queue and execute asynchronous tasks at once -Delayed::Worker.delay_jobs = true - -# Prevent loading ApplicationJob during initialization -Rails.application.reloader.to_prepare do - # Set default priority (lower = higher priority) - # Example ordering, see ApplicationJob.priority_number - Delayed::Worker.default_priority = ::ApplicationJob.priority_number(:default) -end - -# Do not retry jobs from delayed_job -# instead use 'retry_on' activejob functionality -Delayed::Worker.max_attempts = 1 - -# Remember DJ id in the payload object -class Delayed::ProviderJobIdPlugin < Delayed::Plugin - callbacks do |lifecycle| - lifecycle.before(:invoke_job) do |job| - job.payload_object.job_data['provider_job_id'] = job.id if job.payload_object.respond_to?(:job_data) - end - end -end - -Delayed::Worker.plugins << Delayed::ProviderJobIdPlugin diff --git a/config/initializers/time_with_zone_as_json.rb b/config/initializers/time_with_zone_as_json.rb index 7d1b17d621ce..9f12e6ae62fa 100644 --- a/config/initializers/time_with_zone_as_json.rb +++ b/config/initializers/time_with_zone_as_json.rb @@ -26,8 +26,8 @@ # See COPYRIGHT and LICENSE files for more details. #++ -class ActiveSupport::TimeWithZone - def as_json(_options = {}) - time.strftime('%m/%d/%Y/ %H:%M %p').to_s - end -end +# class ActiveSupport::TimeWithZone +# def as_json(_options = {}) +# time.strftime('%m/%d/%Y/ %H:%M %p').to_s +# end +# end diff --git a/config/routes.rb b/config/routes.rb index f1b2d784d457..39f098d5b0ca 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -587,4 +587,8 @@ # Routes for design related documentation and examples pages get '/design/styleguide' => redirect('/assets/styleguide.html') + + constraints(->(req) { User.exists?(id: req.session[:user_id], admin: true) }) do + mount GoodJob::Engine => 'good_job' + end end diff --git a/db/migrate/20230627173959_create_good_jobs.rb b/db/migrate/20230627173959_create_good_jobs.rb new file mode 100644 index 000000000000..82302d2a22f6 --- /dev/null +++ b/db/migrate/20230627173959_create_good_jobs.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +class CreateGoodJobs < ActiveRecord::Migration[7.0] + def change + enable_extension 'pgcrypto' + + create_table :good_jobs, id: :uuid do |t| + t.text :queue_name + t.integer :priority + t.jsonb :serialized_params + t.datetime :scheduled_at + t.datetime :performed_at + t.datetime :finished_at + t.text :error + + t.timestamps + + t.uuid :active_job_id + t.text :concurrency_key + t.text :cron_key + t.uuid :retried_good_job_id + t.datetime :cron_at + + t.uuid :batch_id + t.uuid :batch_callback_id + + t.boolean :is_discrete + t.integer :executions_count + t.text :job_class + end + + create_table :good_job_batches, id: :uuid do |t| + t.timestamps + t.text :description + t.jsonb :serialized_properties + t.text :on_finish + t.text :on_success + t.text :on_discard + t.text :callback_queue_name + t.integer :callback_priority + t.datetime :enqueued_at + t.datetime :discarded_at + t.datetime :finished_at + end + + create_table :good_job_executions, id: :uuid do |t| + t.timestamps + + t.uuid :active_job_id, null: false + t.text :job_class + t.text :queue_name + t.jsonb :serialized_params + t.datetime :scheduled_at + t.datetime :finished_at + t.text :error + end + + create_table :good_job_processes, id: :uuid do |t| + t.timestamps + t.jsonb :state + end + + create_table :good_job_settings, id: :uuid do |t| + t.timestamps + t.text :key + t.jsonb :value + t.index :key, unique: true + end + + add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: "index_good_jobs_on_scheduled_at" + add_index :good_jobs, %i[queue_name scheduled_at], where: "(finished_at IS NULL)", + name: :index_good_jobs_on_queue_name_and_scheduled_at + add_index :good_jobs, %i[active_job_id created_at], name: :index_good_jobs_on_active_job_id_and_created_at + add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", + name: :index_good_jobs_on_concurrency_key_when_unfinished + add_index :good_jobs, %i[cron_key created_at], name: :index_good_jobs_on_cron_key_and_created_at + add_index :good_jobs, %i[cron_key cron_at], name: :index_good_jobs_on_cron_key_and_cron_at, unique: true + add_index :good_jobs, [:active_job_id], name: :index_good_jobs_on_active_job_id + add_index :good_jobs, [:finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", + name: :index_good_jobs_jobs_on_finished_at + add_index :good_jobs, %i[priority created_at], order: { priority: "DESC NULLS LAST", created_at: :asc }, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished + add_index :good_jobs, [:batch_id], where: "batch_id IS NOT NULL" + add_index :good_jobs, [:batch_callback_id], where: "batch_callback_id IS NOT NULL" + + add_index :good_job_executions, %i[active_job_id created_at], + name: :index_good_job_executions_on_active_job_id_and_created_at + end +end diff --git a/docker-compose.yml b/docker-compose.yml index 6cab7e572494..bced69c13056 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,7 +68,7 @@ services: worker: <<: *backend - command: bundle exec rake jobs:work + command: bundle exec good_job start depends_on: - db - cache diff --git a/lib/open_project/patches/delayed_job_adapter.rb b/lib/open_project/patches/delayed_job_adapter.rb deleted file mode 100644 index 07185ad9e1ed..000000000000 --- a/lib/open_project/patches/delayed_job_adapter.rb +++ /dev/null @@ -1,48 +0,0 @@ -#-- copyright -# OpenProject is an open source project management software. -# Copyright (C) 2012-2023 the OpenProject GmbH -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License version 3. -# -# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: -# Copyright (C) 2006-2013 Jean-Philippe Lang -# Copyright (C) 2010-2013 the ChiliProject Team -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# See COPYRIGHT and LICENSE files for more details. -#++ - -# This patch adds our job status extension to background jobs carried out when mailing with -# perform_later. - -module OpenProject - module Patches - module DelayedJobAdapter - module AllowNonExistingJobClass - def log_arguments? - super - rescue NameError - false - end - end - end - end -end - -ActiveJob::QueueAdapters::DelayedJobAdapter::JobWrapper.prepend( - OpenProject::Patches::DelayedJobAdapter::AllowNonExistingJobClass -) diff --git a/lib/open_project/patches/delivery_job.rb b/lib/open_project/patches/delivery_job.rb deleted file mode 100644 index 21ae39c7331f..000000000000 --- a/lib/open_project/patches/delivery_job.rb +++ /dev/null @@ -1,40 +0,0 @@ -#-- copyright -# OpenProject is an open source project management software. -# Copyright (C) 2012-2023 the OpenProject GmbH -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License version 3. -# -# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: -# Copyright (C) 2006-2013 Jean-Philippe Lang -# Copyright (C) 2010-2013 the ChiliProject Team -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# See COPYRIGHT and LICENSE files for more details. -#++ - -# This patch adds our job status extension to background jobs carried out when mailing with -# perform_later. - -module OpenProject - module Patches - module DeliveryJob - # include ::JobStatus::ApplicationJobWithStatus - end - end -end - -ActionMailer::MailDeliveryJob.include JobStatus::ApplicationJobWithStatus diff --git a/lib/open_project/plugins/acts_as_op_engine.rb b/lib/open_project/plugins/acts_as_op_engine.rb index 7fbe28328e74..ada1116282c4 100644 --- a/lib/open_project/plugins/acts_as_op_engine.rb +++ b/lib/open_project/plugins/acts_as_op_engine.rb @@ -294,13 +294,9 @@ def activity_provider(event_type, options = {}) OpenProject::Activity.register(event_type, options) end - ## - # Register a "cron"-like background job def add_cron_jobs(&block) config.to_prepare do - Array(block.call).each do |clz| - ::Cron::CronJob.register!(clz.is_a?(Class) ? clz : clz.to_s.constantize) - end + Rails.application.config.good_job.cron.merge!(block.call) end end diff --git a/lib/tasks/cron.rake b/lib/tasks/cron.rake deleted file mode 100644 index 5a1b90227cc8..000000000000 --- a/lib/tasks/cron.rake +++ /dev/null @@ -1,41 +0,0 @@ -#-- copyright -# OpenProject is an open source project management software. -# Copyright (C) 2012-2023 the OpenProject GmbH -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License version 3. -# -# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: -# Copyright (C) 2006-2013 Jean-Philippe Lang -# Copyright (C) 2010-2013 the ChiliProject Team -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# See COPYRIGHT and LICENSE files for more details. -#++ - -namespace 'openproject:cron' do - desc 'An hourly cron job hook for plugin functionality' - task :hourly do - # Does nothing by default - end - - # This task will be automatically called when running jobs:work or jobs:workoff - # making sure cron jobs are scheduled. See lib/tasks/delayed_job.rake. - desc 'Ensure the cron-like background jobs are actively scheduled' - task schedule: [:environment] do - Cron::CronJob.schedule_registered_jobs! - end -end diff --git a/lib/tasks/delayed_job.rake b/lib/tasks/delayed_job.rake deleted file mode 100644 index fe64d07c4891..000000000000 --- a/lib/tasks/delayed_job.rake +++ /dev/null @@ -1,43 +0,0 @@ -#-- copyright -# OpenProject is an open source project management software. -# Copyright (C) 2012-2023 the OpenProject GmbH -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License version 3. -# -# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: -# Copyright (C) 2006-2013 Jean-Philippe Lang -# Copyright (C) 2010-2013 the ChiliProject Team -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# See COPYRIGHT and LICENSE files for more details. -#++ - -## -# Enhance the delayed_job prerequisites rake task to load the environment -unless Rake::Task.task_defined?('jobs:environment_options') && - Rake::Task['jobs:work'].prerequisites == %w(environment_options) - raise "Trying to load the full environment for delayed_job, but jobs:work seems to have changed." -end - -Rake::Task['jobs:environment_options'] - .clear_prerequisites - .enhance(['environment:full']) - -# Enhance delayed job workers to use cron -load 'lib/tasks/cron.rake' -Rake::Task["jobs:work"].enhance [:'openproject:cron:schedule'] -Rake::Task["jobs:workoff"].enhance [:'openproject:cron:schedule'] diff --git a/modules/github_integration/app/workers/cron/clear_old_pull_requests_job.rb b/modules/github_integration/app/workers/cron/clear_old_pull_requests_job.rb index d9297ea5b1c4..c064fd6e3397 100644 --- a/modules/github_integration/app/workers/cron/clear_old_pull_requests_job.rb +++ b/modules/github_integration/app/workers/cron/clear_old_pull_requests_job.rb @@ -27,12 +27,9 @@ #++ module Cron - class ClearOldPullRequestsJob < CronJob + class ClearOldPullRequestsJob < ApplicationJob priority_number :low - # runs at 1:25 nightly - self.cron_expression = '25 1 * * *' - def perform GithubPullRequest.without_work_package .find_each(&:destroy!) diff --git a/modules/github_integration/lib/open_project/github_integration/engine.rb b/modules/github_integration/lib/open_project/github_integration/engine.rb index 7b8fd686cdd3..8d59264f338e 100644 --- a/modules/github_integration/lib/open_project/github_integration/engine.rb +++ b/modules/github_integration/lib/open_project/github_integration/engine.rb @@ -27,7 +27,7 @@ #++ require 'open_project/plugins' -require_relative './patches/api/work_package_representer' +require_relative 'patches/api/work_package_representer' module OpenProject::GithubIntegration class Engine < ::Rails::Engine @@ -83,9 +83,13 @@ class Engine < ::Rails::Engine mount ::API::V3::GithubPullRequests::GithubPullRequestsAPI end - config.to_prepare do - # Register the cron job to clean up old github pull requests - ::Cron::CronJob.register! ::Cron::ClearOldPullRequestsJob + add_cron_jobs do + { + ClearOldPullRequestsJob: { + cron: '25 1 * * *', # runs at 1:25 nightly + class: 'ClearOldPullRequestsJob' + } + } end end end diff --git a/modules/job_status/app/workers/job_status/cron/clear_old_job_status_job.rb b/modules/job_status/app/workers/job_status/cron/clear_old_job_status_job.rb index 602ea527aa6f..876f48aca7ca 100644 --- a/modules/job_status/app/workers/job_status/cron/clear_old_job_status_job.rb +++ b/modules/job_status/app/workers/job_status/cron/clear_old_job_status_job.rb @@ -28,7 +28,7 @@ module JobStatus module Cron - class ClearOldJobStatusJob < ::Cron::CronJob + class ClearOldJobStatusJob < ApplicationJob # runs at 4:15 nightly self.cron_expression = '15 4 * * *' @@ -36,7 +36,7 @@ class ClearOldJobStatusJob < ::Cron::CronJob def perform ::JobStatus::Status - .where(::JobStatus::Status.arel_table[:updated_at].lteq(Time.now - RETENTION_PERIOD)) + .where(::JobStatus::Status.arel_table[:updated_at].lteq(Time.zone.now - RETENTION_PERIOD)) .destroy_all end end diff --git a/modules/job_status/lib/open_project/job_status/engine.rb b/modules/job_status/lib/open_project/job_status/engine.rb index c82cb228aa2f..fd0996a0a053 100644 --- a/modules/job_status/lib/open_project/job_status/engine.rb +++ b/modules/job_status/lib/open_project/job_status/engine.rb @@ -46,10 +46,16 @@ class Engine < ::Rails::Engine "#{root}/job_statuses/#{uuid}" end - config.to_prepare do - # Register the cron job to clear statuses periodically - ::Cron::CronJob.register! ::JobStatus::Cron::ClearOldJobStatusJob + add_cron_jobs do + { + '::JobStatus::Cron::ClearOldJobStatusJob': { + cron: '15 4 * * *', # runs at 4:15 nightly + class: '::JobStatus::Cron::ClearOldJobStatusJob' + } + } + end + config.to_prepare do # Extends the ActiveJob adapter in use (DelayedJob) by a Status which lives # indenpendently from the job itself (which is deleted once successful or after max attempts). # That way, the result of a background job is available even after the original job is gone. diff --git a/modules/ldap_groups/app/workers/ldap_groups/synchronization_job.rb b/modules/ldap_groups/app/workers/ldap_groups/synchronization_job.rb index 9b2a92a878ac..d3e5f9ede08c 100644 --- a/modules/ldap_groups/app/workers/ldap_groups/synchronization_job.rb +++ b/modules/ldap_groups/app/workers/ldap_groups/synchronization_job.rb @@ -27,10 +27,7 @@ #++ module LdapGroups - class SynchronizationJob < ::Cron::CronJob - # Run every 30 minutes - self.cron_expression = '*/30 * * * *' - + class SynchronizationJob < ApplicationJob def perform return unless EnterpriseToken.allows_to?(:ldap_groups) return if skipped? diff --git a/modules/ldap_groups/lib/open_project/ldap_groups/engine.rb b/modules/ldap_groups/lib/open_project/ldap_groups/engine.rb index 1a2795ad3fd3..9f76031a954e 100644 --- a/modules/ldap_groups/lib/open_project/ldap_groups/engine.rb +++ b/modules/ldap_groups/lib/open_project/ldap_groups/engine.rb @@ -19,7 +19,14 @@ class Engine < ::Rails::Engine enterprise_feature: 'ldap_groups' end - add_cron_jobs { LdapGroups::SynchronizationJob } + add_cron_jobs do + { + 'Ldap::SynchronizationJob': { + cron: '30 23 * * *', # Run once per night at 11:30pm + class: 'Ldap::SynchronizationJob' + } + } + end patches %i[AuthSource Group User] end diff --git a/modules/storages/app/workers/cleanup_uncontainered_file_links_job.rb b/modules/storages/app/workers/cleanup_uncontainered_file_links_job.rb index 96f27af0f2ce..e0fb8e32b9b7 100644 --- a/modules/storages/app/workers/cleanup_uncontainered_file_links_job.rb +++ b/modules/storages/app/workers/cleanup_uncontainered_file_links_job.rb @@ -26,11 +26,9 @@ # See COPYRIGHT and LICENSE files for more details. #++ -class CleanupUncontaineredFileLinksJob < Cron::CronJob +class CleanupUncontaineredFileLinksJob < ApplicationJob queue_with_priority :low - self.cron_expression = '06 22 * * *' - def perform Storages::FileLink .where(container: nil) diff --git a/modules/storages/app/workers/manage_nextcloud_integration_job.rb b/modules/storages/app/workers/manage_nextcloud_integration_job.rb index dd36c3617f5a..943b89507e17 100644 --- a/modules/storages/app/workers/manage_nextcloud_integration_job.rb +++ b/modules/storages/app/workers/manage_nextcloud_integration_job.rb @@ -26,13 +26,9 @@ # See COPYRIGHT and LICENSE files for more details. #++ -class ManageNextcloudIntegrationJob < Cron::CronJob +class ManageNextcloudIntegrationJob < ApplicationJob using ::Storages::Peripherals::ServiceResultRefinements - queue_with_priority :low - - self.cron_expression = '*/5 * * * *' - def perform Storages::NextcloudStorage .where("provider_fields->>'has_managed_project_folders' = 'true'") diff --git a/modules/storages/lib/open_project/storages/engine.rb b/modules/storages/lib/open_project/storages/engine.rb index 28c2e54bfdc6..2241ac700fa7 100644 --- a/modules/storages/lib/open_project/storages/engine.rb +++ b/modules/storages/lib/open_project/storages/engine.rb @@ -226,11 +226,17 @@ class Engine < ::Rails::Engine end add_cron_jobs do - [ - CleanupUncontaineredFileLinksJob, - ].tap do |cron_jobs| + { + 'CleanupUncontaineredFileLinksJob' => { + cron: '06 22 * * *', + class: 'CleanupUncontaineredFileLinksJob' + } + }.tap do |jobs| if OpenProject::FeatureDecisions.managed_project_folders_active? - cron_jobs << ManageNextcloudIntegrationJob + jobs['ManageNextcloudIntegrationJob'] = { + cron: '*/5 * * * *', + class: 'ManageNextcloudIntegrationJob' + } end end end diff --git a/modules/webhooks/app/workers/cleanup_webhook_logs_job.rb b/modules/webhooks/app/workers/cleanup_webhook_logs_job.rb index ea9bca03f292..3a27d5f76a37 100644 --- a/modules/webhooks/app/workers/cleanup_webhook_logs_job.rb +++ b/modules/webhooks/app/workers/cleanup_webhook_logs_job.rb @@ -26,10 +26,7 @@ # See COPYRIGHT and LICENSE files for more details. #++ -class CleanupWebhookLogsJob < Cron::CronJob - # runs at 5:28 on Sunday - self.cron_expression = '28 5 * * 7' - +class CleanupWebhookLogsJob < ApplicationJob # Clean any logs older than 7 days def perform ::Webhooks::Log diff --git a/modules/webhooks/lib/open_project/webhooks/engine.rb b/modules/webhooks/lib/open_project/webhooks/engine.rb index 9ae30e3497c0..f36fd362c1f2 100644 --- a/modules/webhooks/lib/open_project/webhooks/engine.rb +++ b/modules/webhooks/lib/open_project/webhooks/engine.rb @@ -51,6 +51,13 @@ class Engine < ::Rails::Engine end end - add_cron_jobs { CleanupWebhookLogsJob } + add_cron_jobs do + { + 'CleanupWebhookLogsJob' => { + cron: '28 5 * * 7', # runs at 5:28 on Sunday + class: 'CleanupWebhookLogsJob' + } + } + end end end