From 6ca884ccc573eea5c56c524409428f05ae6fff79 Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Wed, 30 Oct 2019 10:01:00 -0400 Subject: [PATCH 1/7] remove extra begin end for rescue --- app/models/metric/capture.rb | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/models/metric/capture.rb b/app/models/metric/capture.rb index 6bd0d99f557..aa90ab798d4 100644 --- a/app/models/metric/capture.rb +++ b/app/models/metric/capture.rb @@ -207,14 +207,12 @@ def self.queue_captures(targets, target_options) options = target_options[target] - begin - target.perf_capture_queue(interval_name, options) - if !target.kind_of?(Storage) && use_historical && target.last_perf_capture_on.nil? - target.perf_capture_queue('historical') - end - rescue => err - _log.warn("Failed to queue perf_capture for target [#{target.class.name}], [#{target.id}], [#{target.name}]: #{err}") + target.perf_capture_queue(interval_name, options) + if !target.kind_of?(Storage) && use_historical && target.last_perf_capture_on.nil? + target.perf_capture_queue('historical') end + rescue => err + _log.warn("Failed to queue perf_capture for target [#{target.class.name}], [#{target.id}], [#{target.name}]: #{err}") end end private_class_method :queue_captures From 8bbdc3cc0fa9070cd4a8e9c22244da7a209af0ee Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Wed, 30 Oct 2019 10:03:01 -0400 Subject: [PATCH 2/7] capture queue - only process task_id for realtime task_id is used to run a rollup task after the fact to create a rollup of hosts by ems_cluster. Currently when running realtime metrics for the first time, we call queue a second time with historical (for non storages). But this second time runs without a rollup (by not passing a task_id) Now, we are only setting the task_id in the queue for realtime entries. --- app/models/metric/ci_mixin/capture.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/metric/ci_mixin/capture.rb b/app/models/metric/ci_mixin/capture.rb index 4d99262121d..ac28583f69f 100644 --- a/app/models/metric/ci_mixin/capture.rb +++ b/app/models/metric/ci_mixin/capture.rb @@ -86,7 +86,7 @@ def perf_capture_queue(interval_name, options = {}) next if item_interval != 'realtime' && messages[start_and_end_time].try(:priority) == priority MiqQueue.put_or_update(queue_item_options) do |msg, qi| # reason for setting MiqQueue#miq_task_id is to initializes MiqTask.started_on column when message delivered. - qi[:miq_task_id] = task_id if task_id + qi[:miq_task_id] = task_id if task_id && item_interval == "realtime" if msg.nil? qi[:priority] = priority qi.delete(:state) @@ -98,9 +98,9 @@ def perf_capture_queue(interval_name, options = {}) qi[:priority] = priority # rerun the job (either with new task or higher priority) qi.delete(:state) - if task_id + if task_id && item_interval == "realtime" existing_tasks = (((msg.miq_callback || {})[:args] || []).first) || [] - qi[:miq_callback] = cb.merge(:args => [existing_tasks + [task_id]]) if item_interval == "realtime" + qi[:miq_callback] = cb.merge(:args => [existing_tasks + [task_id]]) end qi else From 6d35c026fdbfa6fbf4e354594dbdf50926712f0a Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Wed, 30 Oct 2019 10:04:18 -0400 Subject: [PATCH 3/7] metrics - call perf_capture_queue once per object --- app/models/metric/capture.rb | 5 ----- app/models/metric/ci_mixin/capture.rb | 8 +++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/models/metric/capture.rb b/app/models/metric/capture.rb index aa90ab798d4..35666ee72ea 100644 --- a/app/models/metric/capture.rb +++ b/app/models/metric/capture.rb @@ -204,13 +204,8 @@ def self.queue_captures(targets, target_options) targets.each do |target| interval_name = perf_target_to_interval_name(target) - options = target_options[target] - target.perf_capture_queue(interval_name, options) - if !target.kind_of?(Storage) && use_historical && target.last_perf_capture_on.nil? - target.perf_capture_queue('historical') - end rescue => err _log.warn("Failed to queue perf_capture for target [#{target.class.name}], [#{target.id}], [#{target.name}]: #{err}") end diff --git a/app/models/metric/ci_mixin/capture.rb b/app/models/metric/ci_mixin/capture.rb index ac28583f69f..c889602bf47 100644 --- a/app/models/metric/ci_mixin/capture.rb +++ b/app/models/metric/ci_mixin/capture.rb @@ -57,7 +57,13 @@ def perf_capture_queue(interval_name, options = {}) realtime_cut_off = 4.hours.ago.utc.beginning_of_day items = if last_perf_capture_on.nil? - [[interval_name, realtime_cut_off]] + # for initial refresh of non-Storage objects, also go back historically + if !kind_of?(Storage) && Metric::Capture.historical_days != 0 + [[interval_name, realtime_cut_off]] + + split_capture_intervals("historical", Metric::Capture.historical_start_time, 1.day.from_now.utc.beginning_of_day) + else + [[interval_name, realtime_cut_off]] + end elsif last_perf_capture_on < realtime_cut_off [[interval_name, realtime_cut_off]] + split_capture_intervals("historical", last_perf_capture_on, realtime_cut_off) From 1426723eeae8ba55201fe9f6723a12778eecd59f Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Wed, 30 Oct 2019 10:07:38 -0400 Subject: [PATCH 4/7] metrics - separate callback from interval items will extract the logic to derive the items needed to properly run a collection. This is a different concern than the callback that will cluster rollup hosts. --- app/models/metric/ci_mixin/capture.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/models/metric/ci_mixin/capture.rb b/app/models/metric/ci_mixin/capture.rb index c889602bf47..4ba2752f693 100644 --- a/app/models/metric/ci_mixin/capture.rb +++ b/app/models/metric/ci_mixin/capture.rb @@ -45,7 +45,7 @@ def perf_capture_queue(interval_name, options = {}) # Determine the items to queue up # cb is the task used to group cluster realtime metrics - cb = nil + cb = {:class_name => self.class.name, :instance_id => id, :method_name => :perf_capture_callback, :args => [[task_id]]} if task_id && interval_name == 'realtime' if interval_name == 'historical' start_time = Metric::Capture.historical_start_time if start_time.nil? end_time ||= 1.day.from_now.utc.beginning_of_day # Ensure no more than one historical collection is queue up in the same day @@ -70,8 +70,6 @@ def perf_capture_queue(interval_name, options = {}) else [interval_name] end - - cb = {:class_name => self.class.name, :instance_id => id, :method_name => :perf_capture_callback, :args => [[task_id]]} if task_id end # Queue up the actual items From 79cbdfb6198115da927f3432f4047033ef03f5e8 Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Wed, 30 Oct 2019 10:12:27 -0400 Subject: [PATCH 5/7] metrics queue capture: simplify date logic --- app/models/metric/ci_mixin/capture.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/models/metric/ci_mixin/capture.rb b/app/models/metric/ci_mixin/capture.rb index 4ba2752f693..b34b051e8ae 100644 --- a/app/models/metric/ci_mixin/capture.rb +++ b/app/models/metric/ci_mixin/capture.rb @@ -28,8 +28,8 @@ def split_capture_intervals(interval_name, start_time, end_time, threshold = 1.d private :split_capture_intervals def perf_capture_queue(interval_name, options = {}) - start_time = options[:start_time] - end_time = options[:end_time] + start_time = options[:start_time]&.utc + end_time = options[:end_time]&.utc priority = options[:priority] || Metric::Capture.const_get("#{interval_name.upcase}_PRIORITY") task_id = options[:task_id] zone = options[:zone] || my_zone @@ -40,9 +40,6 @@ def perf_capture_queue(interval_name, options = {}) raise ArgumentError, "end_time cannot be specified if start_time is nil" if start_time.nil? && !end_time.nil? raise ArgumentError, "target does not have an ExtManagementSystem" if ems.nil? - start_time = start_time.utc unless start_time.nil? - end_time = end_time.utc unless end_time.nil? - # Determine the items to queue up # cb is the task used to group cluster realtime metrics cb = {:class_name => self.class.name, :instance_id => id, :method_name => :perf_capture_callback, :args => [[task_id]]} if task_id && interval_name == 'realtime' From b7af5d176da07e93eb2856a3eddb9c1a72baf99e Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Wed, 30 Oct 2019 11:35:07 -0400 Subject: [PATCH 6/7] move config archived_for to where it's used --- app/models/metric/capture.rb | 5 ----- app/models/metric/targets.rb | 7 ++++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/models/metric/capture.rb b/app/models/metric/capture.rb index 35666ee72ea..ec7998ebcc3 100644 --- a/app/models/metric/capture.rb +++ b/app/models/metric/capture.rb @@ -20,11 +20,6 @@ def self.historical_start_time historical_days.days.ago.utc.beginning_of_day end - def self.targets_archived_from - archived_for_setting = Settings.performance.targets.archived_for - archived_for_setting.to_i_with_method.seconds.ago.utc - end - def self.concurrent_requests(interval_name) requests = ::Settings.performance.concurrent_requests[interval_name] requests = 20 if requests < 20 && interval_name == 'realtime' diff --git a/app/models/metric/targets.rb b/app/models/metric/targets.rb index 10269f310f6..c859fde9cf4 100644 --- a/app/models/metric/targets.rb +++ b/app/models/metric/targets.rb @@ -8,6 +8,11 @@ def self.perf_capture_always=(options) MiqRegion.my_region.perf_capture_always = options end + def self.targets_archived_from + archived_for_setting = Settings.performance.targets.archived_for + archived_for_setting.to_i_with_method.seconds.ago.utc + end + def self.capture_ems_targets(ems, options = {}) case ems when EmsCloud then capture_cloud_targets([ems], options) @@ -49,7 +54,7 @@ def self.capture_cloud_targets(emses, options = {}) def self.with_archived(scope) # We will look also for freshly archived entities, if the entity was short-lived or even sub-hour - archived_from = Metric::Capture.targets_archived_from + archived_from = targets_archived_from scope.where(:deleted_on => nil).or(scope.where(:deleted_on => (archived_from..Time.now.utc))) end From 8a6a0ea970abd8f8f7cb164ce3ec40c7a1126a5b Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Wed, 30 Oct 2019 11:51:18 -0400 Subject: [PATCH 7/7] metrics: extract queue_items_for_interval --- app/models/metric/ci_mixin/capture.rb | 55 ++++++++++++++------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/app/models/metric/ci_mixin/capture.rb b/app/models/metric/ci_mixin/capture.rb index b34b051e8ae..be7716a09b1 100644 --- a/app/models/metric/ci_mixin/capture.rb +++ b/app/models/metric/ci_mixin/capture.rb @@ -40,34 +40,9 @@ def perf_capture_queue(interval_name, options = {}) raise ArgumentError, "end_time cannot be specified if start_time is nil" if start_time.nil? && !end_time.nil? raise ArgumentError, "target does not have an ExtManagementSystem" if ems.nil? - # Determine the items to queue up # cb is the task used to group cluster realtime metrics cb = {:class_name => self.class.name, :instance_id => id, :method_name => :perf_capture_callback, :args => [[task_id]]} if task_id && interval_name == 'realtime' - if interval_name == 'historical' - start_time = Metric::Capture.historical_start_time if start_time.nil? - end_time ||= 1.day.from_now.utc.beginning_of_day # Ensure no more than one historical collection is queue up in the same day - items = split_capture_intervals(interval_name, start_time, end_time) - else - # if last_perf_capture_on is earlier than 4.hour.ago.beginning_of_day, - # then create *one* realtime capture for start_time = 4.hours.ago.beginning_of_day (no end_time) - # and create historical captures for each day from last_perf_capture_on until 4.hours.ago.beginning_of_day - realtime_cut_off = 4.hours.ago.utc.beginning_of_day - items = - if last_perf_capture_on.nil? - # for initial refresh of non-Storage objects, also go back historically - if !kind_of?(Storage) && Metric::Capture.historical_days != 0 - [[interval_name, realtime_cut_off]] + - split_capture_intervals("historical", Metric::Capture.historical_start_time, 1.day.from_now.utc.beginning_of_day) - else - [[interval_name, realtime_cut_off]] - end - elsif last_perf_capture_on < realtime_cut_off - [[interval_name, realtime_cut_off]] + - split_capture_intervals("historical", last_perf_capture_on, realtime_cut_off) - else - [interval_name] - end - end + items = queue_items_for_interval(interval_name, start_time, end_time) # Queue up the actual items queue_item = { @@ -114,6 +89,34 @@ def perf_capture_queue(interval_name, options = {}) end end + def queue_items_for_interval(interval_name, start_time, end_time) + if interval_name == 'historical' + start_time = Metric::Capture.historical_start_time if start_time.nil? + end_time ||= 1.day.from_now.utc.beginning_of_day # Ensure no more than one historical collection is queue up in the same day + split_capture_intervals(interval_name, start_time, end_time) + else + # if last_perf_capture_on is earlier than 4.hour.ago.beginning_of_day, + # then create *one* realtime capture for start_time = 4.hours.ago.beginning_of_day (no end_time) + # and create historical captures for each day from last_perf_capture_on until 4.hours.ago.beginning_of_day + realtime_cut_off = 4.hours.ago.utc.beginning_of_day + if last_perf_capture_on.nil? + # for initial refresh of non-Storage objects, also go back historically + if !kind_of?(Storage) && Metric::Capture.historical_days != 0 + [[interval_name, realtime_cut_off]] + + split_capture_intervals("historical", Metric::Capture.historical_start_time, 1.day.from_now.utc.beginning_of_day) + else + [[interval_name, realtime_cut_off]] + end + elsif last_perf_capture_on < realtime_cut_off + [[interval_name, realtime_cut_off]] + + split_capture_intervals("historical", last_perf_capture_on, realtime_cut_off) + else + [interval_name] + end + end + end + + def perf_capture_realtime(*args) perf_capture('realtime', *args) end