Merge pull request #102 from DataDog/olivielpeau/catch-event-emit-errors

Improve error handling and logging when submitting data
DataDog · May 13, 2019 · f3afbf1 · f3afbf1
2 parents 87bf803 + af8cdc9
commit f3afbf1
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 15 deletions.
diff --git a/chef-handler-datadog.gemspec b/chef-handler-datadog.gemspec
@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
   gem.extra_rdoc_files = ['README.md', 'LICENSE.txt']
 
-  gem.add_dependency 'dogapi', '>= 1.23'
+  gem.add_dependency 'dogapi', '>= 1.31'
 
   gem.add_development_dependency 'appraisal', '~> 2.0.1'
   gem.add_development_dependency 'bundler'

diff --git a/lib/chef/handler/datadog.rb b/lib/chef/handler/datadog.rb
@@ -31,6 +31,7 @@ def report
 
         @dogs.each do |dog|
           # post the report information to the datadog service
+          Chef::Log.debug("Sending Chef report to #{dog.datadog_host}")
           send_report_to_datadog dog
         end
       ensure

diff --git a/lib/chef/handler/datadog_chef_events.rb b/lib/chef/handler/datadog_chef_events.rb
@@ -66,17 +66,17 @@ def with_tags(tags)
   def emit_to_datadog(dog)
     @event_body = ''
     build_event_data
-    evt = dog.emit_event(Dogapi::Event.new(@event_body,
-                                           msg_title: @event_title,
-                                           event_type: 'config_management.run',
-                                           event_object: @hostname,
-                                           alert_type: @alert_type,
-                                           priority: @event_priority,
-                                           source_type_name: 'chef',
-                                           tags: @tags
-                                          ), host: @hostname)
 
     begin
+      evt = dog.emit_event(Dogapi::Event.new(@event_body,
+                                             msg_title: @event_title,
+                                             event_type: 'config_management.run',
+                                             event_object: @hostname,
+                                             alert_type: @alert_type,
+                                             priority: @event_priority,
+                                             source_type_name: 'chef',
+                                             tags: @tags
+                                            ), host: @hostname)
       # FIXME: nice-to-have: abstract format of return value away a bit
       # in dogapi directly. See https://github.com/DataDog/dogapi-rb/issues/18
       if evt.length < 2
@@ -90,8 +90,8 @@ def emit_to_datadog(dog)
           Chef::Log.debug("Successfully submitted Chef event to Datadog for #{@hostname} at #{evt[1]['event']['url']}")
         end
       end
-    rescue
-      Chef::Log.warn("Could not determine whether chef run was successfully submitted to Datadog: #{evt}")
+    rescue StandardError => e
+      Chef::Log.warn("Could not determine whether Chef event was successfully submitted to Datadog: #{evt}. Error:\n#{e}")
     end
   end
 

diff --git a/lib/chef/handler/datadog_chef_metrics.rb b/lib/chef/handler/datadog_chef_metrics.rb
@@ -48,6 +48,8 @@ def emit_to_datadog(dog)
     dog.emit_point('chef.resources.elapsed_time', @run_status.elapsed_time, host: @hostname)
     Chef::Log.debug('Submitted Chef metrics back to Datadog')
   rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
-    Chef::Log.error("Could not send metrics to Datadog. Connection error:\n" + e)
+    Chef::Log.warn("Could not send metrics to Datadog. Connection error:\n" + e)
+  rescue StandardError => e
+    Chef::Log.warn("Could not determine whether chef run metrics were successfully submitted to Datadog: #{evt}. Error:\n#{e}")
   end
 end # end class DatadogChefMetrics
diff --git a/lib/chef/handler/datadog_chef_tags.rb b/lib/chef/handler/datadog_chef_tags.rb
@@ -113,8 +113,8 @@ def send_update_to_datadog(dog)
         end
         break unless should_retry
       end
-    rescue
-      Chef::Log.warn("Could not determine whether #{@hostname}'s tags were successfully submitted to Datadog: #{rc}")
+    rescue StandardError => e
+      Chef::Log.warn("Could not determine whether #{@hostname}'s tags were successfully submitted to Datadog: #{rc}. Error:\n#{e}")
     end
   end