Merge pull request #158 from North-Two-Five/issue#154-Catch-PostAsync…

…-Network-exceptions-and-Retry

Issue#154 catch post async network exceptions and retry

Analytics/Request/BlockingRequestHandler.cs

@@ -34,7 +34,7 @@ protected override WebRequest GetWebRequest(Uri address)
         {
             WebRequest w = base.GetWebRequest(address);
             if (Timeout.Milliseconds != 0)
-                w.Timeout = Timeout.Milliseconds;
+                w.Timeout = Convert.ToInt32(Timeout.Milliseconds);
             return w;
         }
     }
@@ -220,24 +220,22 @@ public async Task MakeRequest(Batch batch)
                         watch.Stop();
 
                         var response = (HttpWebResponse)ex.Response;
-                        if (response != null)
+                        statusCode = (response != null) ? (int)response.StatusCode : 0;
+                        if ((statusCode >= 500 && statusCode <= 600) || statusCode == 429 || statusCode == 0)
                         {
-                            statusCode = (int)response.StatusCode;
-                            if ((statusCode >= 500 && statusCode <= 600) || statusCode == 429)
-                            {
-                                // If status code is greater than 500 and less than 600, it indicates server error
-                                // Error code 429 indicates rate limited.
-                                // Retry uploading in these cases.
-                                Thread.Sleep(_backo.AttemptTime());
-                                continue;
-                            }
-                            else if (statusCode >= 400)
-                            {
-                                responseStr = String.Format("Status Code {0}. ", statusCode);
-                                responseStr += ex.Message;
-                                break;
-                            }
+                            // If status code is greater than 500 and less than 600, it indicates server error
+                            // Error code 429 indicates rate limited.
+                            // Retry uploading in these cases.
+                            Thread.Sleep(_backo.AttemptTime());
+                            continue;
                         }
+                        else if (statusCode >= 400)
+                        {
+                            responseStr = String.Format("Status Code {0}. ", statusCode);
+                            responseStr += ex.Message;
+                            break;
+                        }
+
                     }
 
 #else
@@ -248,19 +246,32 @@ public async Task MakeRequest(Batch batch)
                     if (_client.Config.Gzip)
                         content.Headers.ContentEncoding.Add("gzip");
 
-                    var response = await _httpClient.PostAsync(uri, content).ConfigureAwait(false);
+                    HttpResponseMessage response = null;
+                    bool retry = false;
+                    try
+                    {
+                        response = await _httpClient.PostAsync(uri, content).ConfigureAwait(false);
+                    }
+                    catch (TaskCanceledException)
+                    {
+                        retry = true;
+                    }
+                    catch (HttpRequestException)
+                    {
+                        retry = true;
+                    }
 
                     watch.Stop();
-                    statusCode = (int)response.StatusCode;
+                    statusCode = response != null ? (int)response.StatusCode : 0;
 
-                    if (statusCode == (int)HttpStatusCode.OK)
+                    if (response != null && response.StatusCode == HttpStatusCode.OK)
                     {
                         Succeed(batch, watch.ElapsedMilliseconds);
                         break;
                     }
                     else
                     {
-                        if ((statusCode >= 500 && statusCode <= 600) || statusCode == 429)
+                        if ((statusCode >= 500 && statusCode <= 600) || statusCode == 429 || retry)
                         {
                             // If status code is greater than 500 and less than 600, it indicates server error
                             // Error code 429 indicates rate limited.
@@ -281,6 +292,10 @@ public async Task MakeRequest(Batch batch)
                 if (_backo.HasReachedMax || statusCode != (int)HttpStatusCode.OK)
                 {
                     Fail(batch, new APIException("Unexpected Status Code", responseStr), watch.ElapsedMilliseconds);
+                    if (_backo.HasReachedMax)
+                    {
+                        _backo.Reset();
+                    }
                 }
             }
             catch (System.Exception e)

Test.Net35/ConnectionTests.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.Net;
 using System.Text;
 using Moq;
 using NUnit.Framework;
@@ -13,6 +15,14 @@ public class ConnectionTests
     {
         private Mock<IRequestHandler> _mockRequestHandler;
 
+        class RetryErrorTestCase
+        {
+            public HttpStatusCode ResponseCode;
+            public string ErrorMessage;
+            public int Timeout;
+            public bool ShouldRetry;
+        }
+
         [SetUp]
         public void Init()
         {
@@ -34,6 +44,113 @@ public void CleanUp()
             Logger.Handlers -= LoggingHandler;
         }
 
+        [Test()]
+        public void RetryErrorTestNet35()
+        {
+            Stopwatch watch = new Stopwatch();
+
+            // Set invalid host address and make timeout to 1s
+            var config = new Config().SetAsync(false);
+            config.SetHost("https://fake.segment-server.com");
+            config.SetTimeout(new TimeSpan(0, 0, 1));
+            Analytics.Initialize(Constants.WRITE_KEY, config);
+
+            // Calculate working time for Identiy message with invalid host address
+            watch.Start();
+            Actions.Identify(Analytics.Client);
+            watch.Stop();
+
+            Assert.AreEqual(1, Analytics.Client.Statistics.Submitted);
+            Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded);
+            Assert.AreEqual(1, Analytics.Client.Statistics.Failed);
+
+            // Handling Identify message will take more than 10s even though the timeout is 1s.
+            // That's because it retries submit when it's failed.
+            Assert.AreEqual(true, watch.ElapsedMilliseconds > 10000);
+        }
+
+        [Test()]
+        public void RetryServerErrorTestNet35()
+        {
+            Stopwatch watch = new Stopwatch();
+
+            string DummyServerUrl = "http://localhost:9696";
+            using (var DummyServer = new WebServer(DummyServerUrl))
+            {
+                DummyServer.RunAsync();
+
+                // Set invalid host address and make timeout to 1s
+                var config = new Config().SetAsync(false);
+                config.SetHost(DummyServerUrl);
+                config.SetTimeout(new TimeSpan(0, 0, 1));
+                Analytics.Initialize(Constants.WRITE_KEY, config);
+
+                var TestCases = new RetryErrorTestCase[]
+                {
+                    // The errors (500 > code >= 400) doesn't require retry
+                    new RetryErrorTestCase()
+                    {
+                        ErrorMessage = "Server Gone",
+                        ResponseCode = HttpStatusCode.Gone,
+                        ShouldRetry = false,
+                        Timeout = 10000
+                    },
+                    // 429 error requires retry
+                    new RetryErrorTestCase()
+                    {
+                        ErrorMessage = "Too many requests",
+                        ResponseCode = (HttpStatusCode)429,
+                        ShouldRetry = true,
+                        Timeout = 10000
+                    },
+                    // Server errors require retry
+                    new RetryErrorTestCase()
+                    {
+                        ErrorMessage = "Bad Gateway",
+                        ResponseCode = HttpStatusCode.BadGateway,
+                        ShouldRetry = true,
+                        Timeout = 10000
+                    }
+                };
+
+                foreach (var testCase in TestCases)
+                {
+                    // Setup fallback module which returns error code
+                    DummyServer.RequestHandler = ((req, res) =>
+                    {
+                        string pageData = "{ ErrorMessage: '" + testCase.ErrorMessage + "' }";
+                        byte[] data = Encoding.UTF8.GetBytes(pageData);
+
+                        res.StatusCode = (int)testCase.ResponseCode;
+                        res.ContentType = "application/json";
+                        res.ContentEncoding = Encoding.UTF8;
+                        res.ContentLength64 = data.LongLength;
+
+                        res.OutputStream.Write(data, 0, data.Length);
+                        res.Close();
+                    });
+
+                    // Calculate working time for Identiy message with invalid host address
+                    watch.Start();
+                    Actions.Identify(Analytics.Client);
+                    watch.Stop();
+
+                    DummyServer.RequestHandler = null;
+
+                    Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded);
+
+                    // Handling Identify message will less than 10s because the server returns GONE message.
+                    // That's because it retries submit when it's failed.
+                    if (testCase.ShouldRetry)
+                        Assert.IsTrue(watch.ElapsedMilliseconds > testCase.Timeout);
+                    else
+                        Assert.IsFalse(watch.ElapsedMilliseconds > testCase.Timeout);
+                }
+            }
+        }
+
+
+
         [Test()]
         public void ProxyTestNet35()
         {

Test.Net35/Test.Net35.csproj

@@ -67,6 +67,7 @@
     <Compile Include="RequestHandlerTest.cs" />
     <Compile Include="Request\BlockingRequestHandlerTests.cs" />
     <Compile Include="Stats\StatisticsTests.cs" />
+    <Compile Include="WebServer.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />

Test.Net35/WebServer.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading;
+
+namespace Segment.Test
+{
+    class WebServer :IDisposable
+    {
+        private HttpListener listener;
+        private Thread runningThread;
+        private bool runServer;
+        private string address;
+
+        public delegate void HandleRequest(HttpListenerRequest req, HttpListenerResponse res);
+        public HandleRequest RequestHandler = null;
+
+        public WebServer(string url)
+        {
+            // Only string ends with '/' is acceptable for web server url
+            address = url;
+            if (!address.EndsWith("/"))
+                address += "/";
+        }
+
+        public void HandleIncomingConnections()
+        {
+            while (runServer)
+            {
+                try
+                {
+                    // Will wait here until we hear from a connection
+                    HttpListenerContext ctx = listener.GetContext();
+
+                    // Peel out the requests and response objects
+                    HttpListenerRequest req = ctx.Request;
+                    HttpListenerResponse res = ctx.Response;
+
+                    if (RequestHandler != null)
+                    {
+                        RequestHandler(req, res);
+                    }
+                    else
+                    {
+                        // Write the response info
+                        string pageData = "{}";
+                        byte[] data = Encoding.UTF8.GetBytes(pageData);
+                        res.ContentType = "application/json";
+                        res.ContentEncoding = Encoding.UTF8;
+                        res.ContentLength64 = data.LongLength;
+
+                        // Write out to the response stream (asynchronously), then close it
+                        res.OutputStream.Write(data, 0, data.Length);
+                        res.Close();
+                    }
+                }
+                catch (System.Exception)
+                {
+                    runServer = false;
+                    return;
+                }
+            }
+        }
+
+        public void RunAsync()
+        {
+            // Stop already running server
+            Stop();
+
+            // Create new listener
+            listener = new HttpListener();
+            listener.Prefixes.Add(address);
+            listener.Start();
+
+            // Start listening requests
+            runServer = true;
+            runningThread = new Thread(HandleIncomingConnections);
+            runningThread.Start();
+        }
+
+        public void Stop()
+        {
+            runServer = false;
+
+            if (listener != null)
+            {
+                listener.Close();
+                listener = null;
+            }
+            if (runningThread != null)
+            {
+                runningThread.Join();
+                runningThread = null;
+            }
+        }
+
+        #region IDisposable Support
+        private bool disposedValue = false; // To detect redundant calls
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!disposedValue)
+            {
+                if (disposing)
+                {
+                    Stop();
+                }
+
+                disposedValue = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+        #endregion
+    }
+}