Refine error handling policy for outputs

1. Ignore TaskCanceledException (expected to occur when pipeline is shutting down) 2. Report data transmission errors as warnings instead of errors Justification: a. These errors might be transient b. Core service functionality is not necessarily affected, so reporting warninngs is more appropriate
Azure · Jan 25, 2018 · b03c28d · b03c28d
1 parent d4e0d48
commit b03c28d
Show file tree

Hide file tree

Showing 15 changed files with 82 additions and 23 deletions.
diff --git a/src/Microsoft.Diagnostics.EventFlow.Core/Implementations/DiagnosticPipeline.cs b/src/Microsoft.Diagnostics.EventFlow.Core/Implementations/DiagnosticPipeline.cs
@@ -13,7 +13,7 @@
 using Validation;
 
 using Microsoft.Diagnostics.EventFlow.Configuration;
-using System.Diagnostics;
+using Microsoft.Diagnostics.EventFlow.Utilities;
 
 namespace Microsoft.Diagnostics.EventFlow
 {
@@ -358,9 +358,13 @@ public async Task SendDataAsync(EventData[] events)
                     }
                     catch (Exception e)
                     {
-                        this.healthReporter.ReportWarning(
-                            nameof(DiagnosticPipeline) + ": an output has thrown an exception while sending data" + Environment.NewLine + e.ToString(),
-                            EventFlowContextIdentifiers.Output);
+                        ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
+                        {
+                            this.healthReporter.ReportWarning(
+                                nameof(DiagnosticPipeline) + ": an output has thrown an exception while sending data" + Environment.NewLine + e.ToString(),
+                                EventFlowContextIdentifiers.Output);
+                        });
+
                     }
                 }
             }

diff --git a/src/Microsoft.Diagnostics.EventFlow.Core/Implementations/ErrorHandlingPolicies.cs b/src/Microsoft.Diagnostics.EventFlow.Core/Implementations/ErrorHandlingPolicies.cs
@@ -0,0 +1,27 @@
+// ------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation.  All rights reserved.
+//  Licensed under the MIT License (MIT). See License.txt in the repo root for license information.
+// ------------------------------------------------------------
+
+using System;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.Diagnostics.EventFlow.Utilities
+{
+    public class ErrorHandlingPolicies
+    {
+        public static void HandleOutputTaskError(Exception e, Action handler)
+        {
+            if (e is TaskCanceledException)
+            {
+                // This exception is expected when the pipeline is shutting down--disregard.
+            }
+            else
+            {
+                handler();
+            }
+        }
+    }
+}
diff --git a/src/Microsoft.Diagnostics.EventFlow.Core/Microsoft.Diagnostics.EventFlow.Core.csproj b/src/Microsoft.Diagnostics.EventFlow.Core/Microsoft.Diagnostics.EventFlow.Core.csproj
@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Description>Defines core interfaces and types that comprise Microsoft.Diagnostics.EventFlow library.</Description>
     <Copyright>© Microsoft Corporation. All rights reserved.</Copyright>
-    <VersionPrefix>1.1.9</VersionPrefix>
+    <VersionPrefix>1.1.10</VersionPrefix>
     <Authors>Microsoft</Authors>
     <TargetFrameworks>netstandard1.6;net451</TargetFrameworks>
     <DelaySign>true</DelaySign>

diff --git a/src/Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsights/ApplicationInsightsOutput.cs b/src/Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsights/ApplicationInsightsOutput.cs
@@ -19,6 +19,7 @@
 
 using Microsoft.Diagnostics.EventFlow.Configuration;
 using Microsoft.Diagnostics.EventFlow.Metadata;
+using Microsoft.Diagnostics.EventFlow.Utilities;
 using Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsights;
 
 namespace Microsoft.Diagnostics.EventFlow.Outputs
@@ -129,7 +130,11 @@ public Task SendEventsAsync(IReadOnlyCollection<EventData> events, long transmis
             }
             catch (Exception e)
             {
-                this.healthReporter.ReportProblem("Diagnostics data upload has failed." + Environment.NewLine + e.ToString());
+                ErrorHandlingPolicies.HandleOutputTaskError(e, () => 
+                {
+                    string errorMessage = nameof(ApplicationInsightsOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
+                    this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
+                });                
             }
 
             return CompletedTask;

diff --git a/...ts.ApplicationInsights/Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsights.csproj b/...ts.ApplicationInsights/Microsoft.Diagnostics.EventFlow.Outputs.ApplicationInsights.csproj
@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Description>Provides an output implementation that sends diagnostics data to Microsoft Application Insights service.</Description>
     <Copyright>© Microsoft Corporation. All rights reserved.</Copyright>
-    <VersionPrefix>1.2.1</VersionPrefix>
+    <VersionPrefix>1.2.2</VersionPrefix>
     <Authors>Microsoft</Authors>
     <TargetFrameworks>net451;netstandard1.6</TargetFrameworks>
     <DelaySign>true</DelaySign>

diff --git a/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/ElasticSearchOutput.cs b/src/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch/ElasticSearchOutput.cs
@@ -11,6 +11,7 @@
 using System.Threading.Tasks;
 using Microsoft.Diagnostics.EventFlow.Configuration;
 using Microsoft.Diagnostics.EventFlow.Metadata;
+using Microsoft.Diagnostics.EventFlow.Utilities;
 using Microsoft.Extensions.Configuration;
 using Nest;
 using Validation;
@@ -107,8 +108,11 @@ public async Task SendEventsAsync(IReadOnlyCollection<EventData> events, long tr
             }
             catch (Exception e)
             {
-                string errorMessage = nameof(ElasticSearchOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
-                this.healthReporter.ReportProblem(errorMessage);
+                ErrorHandlingPolicies.HandleOutputTaskError(e, () => 
+                {
+                    string errorMessage = nameof(ElasticSearchOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
+                    this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
+                });
             }
         }
 
@@ -439,7 +443,7 @@ private void ReportEsRequestError(IResponse response, string request)
                 errorMessage += "No further error information is available";
             }
 
-            this.healthReporter.ReportProblem(errorMessage);
+            this.healthReporter.ReportWarning(errorMessage);
         }
 
         private class ElasticSearchConnectionData

diff --git a/...ntFlow.Outputs.ElasticSearch/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch.csproj b/...ntFlow.Outputs.ElasticSearch/Microsoft.Diagnostics.EventFlow.Outputs.ElasticSearch.csproj
@@ -3,7 +3,7 @@
   <PropertyGroup>
     <Description>Provides an output implementation that sends diagnostics data to Elasticsearch.</Description>
     <Copyright>© Microsoft Corporation. All rights reserved.</Copyright>
-    <VersionPrefix>2.2.0</VersionPrefix>
+    <VersionPrefix>2.2.1</VersionPrefix>
     <Authors>Microsoft</Authors>
     <TargetFrameworks>netstandard1.6;net451</TargetFrameworks>
     <DelaySign>true</DelaySign>

diff --git a/src/Microsoft.Diagnostics.EventFlow.Outputs.EventHub/EventHubOutput.cs b/src/Microsoft.Diagnostics.EventFlow.Outputs.EventHub/EventHubOutput.cs
@@ -10,6 +10,7 @@
 using System.Threading.Tasks;
 using Microsoft.Azure.EventHubs;
 using Microsoft.Diagnostics.EventFlow.Configuration;
+using Microsoft.Diagnostics.EventFlow.Utilities;
 using Microsoft.Extensions.Configuration;
 using Validation;
 using MessagingEventData = Microsoft.Azure.EventHubs.EventData;
@@ -119,8 +120,11 @@ public async Task SendEventsAsync(IReadOnlyCollection<EventData> events, long tr
             }
             catch (Exception e)
             {
-                string errorMessage = nameof(EventHubOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
-                this.healthReporter.ReportProblem(errorMessage);
+                ErrorHandlingPolicies.HandleOutputTaskError(e, () => 
+                {
+                    string errorMessage = nameof(EventHubOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
+                    this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
+                });
             }
         }
 

diff --git a/...ostics.EventFlow.Outputs.EventHub/Microsoft.Diagnostics.EventFlow.Outputs.EventHub.csproj b/...ostics.EventFlow.Outputs.EventHub/Microsoft.Diagnostics.EventFlow.Outputs.EventHub.csproj
@@ -9,7 +9,7 @@
     <AssemblyName>Microsoft.Diagnostics.EventFlow.Outputs.EventHub</AssemblyName>
     <AssemblyOriginatorKeyFile>../../PublicKey.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
-    <VersionPrefix>1.2.0</VersionPrefix>
+    <VersionPrefix>1.2.1</VersionPrefix>
     <PublicSign Condition=" '$(OS)' != 'Windows_NT' ">true</PublicSign>
     <PackageId>Microsoft.Diagnostics.EventFlow.Outputs.EventHub</PackageId>
     <PackageTags>Microsoft;Diagnostics;EventFlow;Outputs;Event Hub;Event Hubs</PackageTags>

diff --git a/src/Microsoft.Diagnostics.EventFlow.Outputs.HttpOutput/HttpOutput.cs b/src/Microsoft.Diagnostics.EventFlow.Outputs.HttpOutput/HttpOutput.cs
@@ -13,6 +13,7 @@
 using System.Text;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Diagnostics.EventFlow.Configuration;
+using Microsoft.Diagnostics.EventFlow.Utilities;
 using Newtonsoft.Json;
 using Validation;
 
@@ -134,9 +135,13 @@ public async Task SendEventsAsync(IReadOnlyCollection<EventData> events, long tr
                 HttpResponseMessage response = await httpClient.PostAsync(new Uri(configuration.ServiceUri), contentPost);
                 response.EnsureSuccessStatusCode();
             }
-            catch (Exception ex)
+            catch (Exception e)
             {
-                this.healthReporter.ReportProblem($"{nameof(configuration)}: Fail to send events in batch. Error details: {ex.ToString()}");
+                ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
+                {
+                    string errorMessage = nameof(HttpOutput) + ": diagnostic data upload failed: " + Environment.NewLine + e.ToString();
+                    this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
+                });
             }
         }
 

diff --git a/...cs.EventFlow.Outputs.HttpOutput/Microsoft.Diagnostics.EventFlow.Outputs.HttpOutput.csproj b/...cs.EventFlow.Outputs.HttpOutput/Microsoft.Diagnostics.EventFlow.Outputs.HttpOutput.csproj
@@ -9,7 +9,7 @@
     <AssemblyName>Microsoft.Diagnostics.EventFlow.Outputs.HttpOutput</AssemblyName>
     <AssemblyOriginatorKeyFile>../../PublicKey.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
-    <VersionPrefix>1.1.1</VersionPrefix>
+    <VersionPrefix>1.1.2</VersionPrefix>
     <PublicSign Condition=" '$(OS)' != 'Windows_NT' ">true</PublicSign>
     <PackageId>Microsoft.Diagnostics.EventFlow.Outputs.HttpOutput</PackageId>
     <PackageTags>Microsoft;Diagnostics;EventFlow;Outputs;HttpOutput</PackageTags>

diff --git a/...soft.Diagnostics.EventFlow.Outputs.Oms/Microsoft.Diagnostics.EventFlow.Outputs.Oms.csproj b/...soft.Diagnostics.EventFlow.Outputs.Oms/Microsoft.Diagnostics.EventFlow.Outputs.Oms.csproj
@@ -9,7 +9,7 @@
     <AssemblyName>Microsoft.Diagnostics.EventFlow.Outputs.Oms</AssemblyName>
     <AssemblyOriginatorKeyFile>../../PublicKey.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
-    <VersionPrefix>1.1.1</VersionPrefix>
+    <VersionPrefix>1.1.2</VersionPrefix>
     <PublicSign Condition=" '$(OS)' != 'Windows_NT' ">true</PublicSign>
     <PackageId>Microsoft.Diagnostics.EventFlow.Outputs.Oms</PackageId>
     <PackageTags>Microsoft;Diagnostics;EventFlow;Outputs;Microsoft Operations Management</PackageTags>

diff --git a/src/Microsoft.Diagnostics.EventFlow.Outputs.Oms/OmsOutput.cs b/src/Microsoft.Diagnostics.EventFlow.Outputs.Oms/OmsOutput.cs
@@ -5,6 +5,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Security.Cryptography;
@@ -16,7 +17,7 @@
 using Validation;
 
 using Microsoft.Diagnostics.EventFlow.Configuration;
-using System.Diagnostics;
+using Microsoft.Diagnostics.EventFlow.Utilities;
 
 namespace Microsoft.Diagnostics.EventFlow.Outputs
 {
@@ -137,7 +138,11 @@ public async Task SendEventsAsync(IReadOnlyCollection<EventData> events, long tr
             }
             catch (Exception e)
             {
-                this.healthReporter.ReportProblem("An error occurred while sending data to OMS: " + e.ToString());
+                ErrorHandlingPolicies.HandleOutputTaskError(e, () => 
+                {
+                    string errorMessage = nameof(OmsOutput) + ": an error occurred while sending data to OMS: " + Environment.NewLine + e.ToString();
+                    this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
+                });
             }
         }
 

diff --git a/...ventFlow.Outputs.TableStorage/Microsoft.Diagnostics.EventFlow.Outputs.TableStorage.csproj b/...ventFlow.Outputs.TableStorage/Microsoft.Diagnostics.EventFlow.Outputs.TableStorage.csproj
@@ -9,7 +9,7 @@
     <AssemblyName>Microsoft.Diagnostics.EventFlow.Outputs.TableStorage</AssemblyName>
     <AssemblyOriginatorKeyFile>../../PublicKey.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
-    <VersionPrefix>1.1.0</VersionPrefix>
+    <VersionPrefix>1.1.1</VersionPrefix>
     <PublicSign Condition=" '$(OS)' != 'Windows_NT' ">true</PublicSign>    
     <NetStandardImplicitPackageVersion>1.6.1</NetStandardImplicitPackageVersion>
     <GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>

diff --git a/src/Microsoft.Diagnostics.EventFlow.Outputs.TableStorage/TableStorageSender.cs b/src/Microsoft.Diagnostics.EventFlow.Outputs.TableStorage/TableStorageSender.cs
@@ -14,6 +14,8 @@
 using Microsoft.WindowsAzure.Storage.Table;
 using Validation;
 
+using Microsoft.Diagnostics.EventFlow.Utilities;
+
 namespace Microsoft.Diagnostics.EventFlow
 {
     public class TableStorageSender : IOutput
@@ -80,8 +82,11 @@ public async Task SendEventsAsync(IReadOnlyCollection<EventData> events, long tr
             }
             catch (Exception e)
             {
-                string errorMessage = nameof(TableStorageSender) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
-                this.healthReporter.ReportProblem(errorMessage);
+                ErrorHandlingPolicies.HandleOutputTaskError(e, () => 
+                {
+                    string errorMessage = nameof(TableStorageSender) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
+                    this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
+                });
             }
         }