Added metrics to Scheduler and track in Process state

app/models/good_job/process.rb

@@ -50,8 +50,10 @@ def self.current_state
         proctitle: $PROGRAM_NAME,
         preserve_job_records: GoodJob.preserve_job_records,
         retry_on_unhandled_error: GoodJob.retry_on_unhandled_error,
-        schedulers: GoodJob::Scheduler.instances.map(&:name),
+        schedulers: GoodJob::Scheduler.instances.map(&:stats),
         cron_enabled: GoodJob.configuration.enable_cron?,
+        total_succeeded_executions_count: GoodJob::Scheduler.instances.sum(&:succeeded_executions_count),
+        total_failed_executions_count: GoodJob::Scheduler.instances.sum(&:failed_executions_count),
       }
     end
 

lib/good_job/cleanup_tracker.rb

@@ -17,7 +17,7 @@ def initialize(cleanup_interval_seconds: false, cleanup_interval_jobs: false)
     end
 
     # Increments job count.
-    # @return [void]
+    # @return [Integer]
     def increment
       self.job_count += 1
     end

lib/good_job/metrics.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+module GoodJob # :nodoc:
+  # Metrics for the scheduler.
+  class Metrics
+    def initialize
+      @failed_count = Concurrent::AtomicFixnum.new
+      @succeeded_count = Concurrent::AtomicFixnum.new
+    end
+
+    # Increments number of failed jobs.
+    # @return [Integer]
+    def increment_failed
+      @failed_count.increment
+    end
+
+    # Increments number of succeeded jobs.
+    # @return [Integer]
+    def increment_succeeded
+      @succeeded_count.increment
+    end
+
+    # Failed job count.
+    # @return [Integer]
+    def failed_count
+      @failed_count.value
+    end
+
+    # Number of succeeded jobs.
+    # @return [Integer]
+    def succeeded_count
+      @succeeded_count.value
+    end
+
+    # Total number of jobs (failed + succeeded).
+    # @return [Integer]
+    def total_count
+      failed_count + succeeded_count
+    end
+
+    # Reset counters.
+    # @return [void]
+    def reset
+      @failed_count.value = 0
+      @succeeded_count.value = 0
+    end
+  end
+end

lib/good_job/scheduler.rb

@@ -3,6 +3,7 @@
 require "concurrent/executor/timer_set"
 require "concurrent/scheduled_task"
 require "concurrent/utility/processor_counter"
+require 'good_job/metrics'
 
 module GoodJob # :nodoc:
   #
@@ -88,6 +89,8 @@ def initialize(performer, max_threads: nil, max_cache: nil, warm_cache_on_initia
       @executor_options[:name] = name
 
       @cleanup_tracker = CleanupTracker.new(cleanup_interval_seconds: cleanup_interval_seconds, cleanup_interval_jobs: cleanup_interval_jobs)
+      @metrics = ::GoodJob::Metrics.new
+      @executor_options[:name] = name
       create_executor
       warm_cache if warm_cache_on_initialize
     end
@@ -139,6 +142,7 @@ def restart(timeout: -1)
 
       instrument("scheduler_restart_pools") do
         shutdown(timeout: timeout)
+        @metrics.reset
         create_executor
         warm_cache
       end
@@ -203,6 +207,12 @@ def task_observer(time, output, thread_error)
       instrument("finished_job_task", { result: output, error: thread_error, time: time })
       return unless output
 
+      if error
+        @metrics.increment_failed
+      else
+        @metrics.increment_succeeded
+      end
+
       @cleanup_tracker.increment
       if @cleanup_tracker.cleanup?
         cleanup
@@ -222,6 +232,9 @@ def stats
         max_cache: @max_cache,
         active_cache: cache_count,
         available_cache: remaining_cache_count,
+        succeeded_executions_count: @metrics.succeeded_count,
+        failed_executions_count: @metrics.failed_count,
+        total_executions_count: @metrics.total_count,
       }
     end
 

spec/lib/good_job/scheduler_spec.rb

@@ -8,7 +8,7 @@
     described_class.instances.each(&:shutdown)
   end
 
-  describe 'name' do
+  describe '#name' do
     it 'is human readable and contains configuration values' do
       scheduler = described_class.new(performer)
       expect(scheduler.name).to eq('GoodJob::Scheduler(queues= max_threads=5)')
@@ -76,6 +76,34 @@
     end
   end
 
+  describe '#task_observer' do
+    it 'increases metric counters' do
+      allow(GoodJob).to receive(:on_thread_error)
+
+      failed_job_count = 0
+      succeeded_job_count = 0
+
+      allow(performer).to receive(:next) do
+        if failed_job_count < 7
+          failed_job_count += 1
+          GoodJob::ExecutionResult.new(value: nil, unhandled_error: StandardError.new("oopsy"))
+        elsif succeeded_job_count < 9
+          succeeded_job_count += 1
+          GoodJob::ExecutionResult.new(value: 'success')
+        end
+      end
+
+      scheduler = described_class.new(performer)
+      scheduler.create_thread
+
+      sleep_until { succeeded_job_count == 9 }
+
+      expect(scheduler.stats.fetch(:failed_executions_count)).to eq 7
+      expect(scheduler.stats.fetch(:succeeded_executions_count)).to eq 9
+      expect(scheduler.stats.fetch(:total_executions_count)).to eq 16
+    end
+  end
+
   describe '#shutdown' do
     it 'shuts down the theadpools' do
       scheduler = described_class.new(performer)
@@ -115,6 +143,25 @@
         .to change(scheduler, :running?).from(false).to(true)
     end
 
+    it 'resets metrics' do
+      allow(performer).to receive(:next).and_return GoodJob::ExecutionResult.new(value: 'hello'), nil
+
+      scheduler = described_class.new(performer)
+      scheduler.create_thread
+
+      sleep_until do
+        scheduler.stats.fetch(:succeeded_executions_count) == 1
+      end
+
+      scheduler.shutdown
+      expect(scheduler.stats.fetch(:succeeded_executions_count)).to eq 1
+
+      expect { scheduler.restart }
+        .to change(scheduler, :running?).from(false).to(true)
+
+      expect(scheduler.stats.fetch(:succeeded_executions_count)).to eq 0
+    end
+
     it 'can be called multiple times' do
       scheduler = described_class.new(performer)
       scheduler.shutdown
@@ -177,6 +224,9 @@
                                       max_cache: max_cache,
                                       active_cache: 0,
                                       available_cache: max_cache,
+                                      failed_executions_count: 0,
+                                      succeeded_executions_count: 0,
+                                      total_executions_count: 0,
                                     })
     end
   end