diff --git a/Analytics/Request/BlockingRequestHandler.cs b/Analytics/Request/BlockingRequestHandler.cs index 8c145667..e6f22887 100644 --- a/Analytics/Request/BlockingRequestHandler.cs +++ b/Analytics/Request/BlockingRequestHandler.cs @@ -34,7 +34,7 @@ protected override WebRequest GetWebRequest(Uri address) { WebRequest w = base.GetWebRequest(address); if (Timeout.Milliseconds != 0) - w.Timeout = Timeout.Milliseconds; + w.Timeout = Convert.ToInt32(Timeout.Milliseconds); return w; } } @@ -220,24 +220,22 @@ public async Task MakeRequest(Batch batch) watch.Stop(); var response = (HttpWebResponse)ex.Response; - if (response != null) + statusCode = (response != null) ? (int)response.StatusCode : 0; + if ((statusCode >= 500 && statusCode <= 600) || statusCode == 429 || statusCode == 0) { - statusCode = (int)response.StatusCode; - if ((statusCode >= 500 && statusCode <= 600) || statusCode == 429) - { - // If status code is greater than 500 and less than 600, it indicates server error - // Error code 429 indicates rate limited. - // Retry uploading in these cases. - Thread.Sleep(_backo.AttemptTime()); - continue; - } - else if (statusCode >= 400) - { - responseStr = String.Format("Status Code {0}. ", statusCode); - responseStr += ex.Message; - break; - } + // If status code is greater than 500 and less than 600, it indicates server error + // Error code 429 indicates rate limited. + // Retry uploading in these cases. + Thread.Sleep(_backo.AttemptTime()); + continue; } + else if (statusCode >= 400) + { + responseStr = String.Format("Status Code {0}. ", statusCode); + responseStr += ex.Message; + break; + } + } #else @@ -248,19 +246,32 @@ public async Task MakeRequest(Batch batch) if (_client.Config.Gzip) content.Headers.ContentEncoding.Add("gzip"); - var response = await _httpClient.PostAsync(uri, content).ConfigureAwait(false); + HttpResponseMessage response = null; + bool retry = false; + try + { + response = await _httpClient.PostAsync(uri, content).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + retry = true; + } + catch (HttpRequestException) + { + retry = true; + } watch.Stop(); - statusCode = (int)response.StatusCode; + statusCode = response != null ? (int)response.StatusCode : 0; - if (statusCode == (int)HttpStatusCode.OK) + if (response != null && response.StatusCode == HttpStatusCode.OK) { Succeed(batch, watch.ElapsedMilliseconds); break; } else { - if ((statusCode >= 500 && statusCode <= 600) || statusCode == 429) + if ((statusCode >= 500 && statusCode <= 600) || statusCode == 429 || retry) { // If status code is greater than 500 and less than 600, it indicates server error // Error code 429 indicates rate limited. @@ -281,6 +292,10 @@ public async Task MakeRequest(Batch batch) if (_backo.HasReachedMax || statusCode != (int)HttpStatusCode.OK) { Fail(batch, new APIException("Unexpected Status Code", responseStr), watch.ElapsedMilliseconds); + if (_backo.HasReachedMax) + { + _backo.Reset(); + } } } catch (System.Exception e) diff --git a/Test.Net35/ConnectionTests.cs b/Test.Net35/ConnectionTests.cs index a8bde2d5..8e4b3d80 100644 --- a/Test.Net35/ConnectionTests.cs +++ b/Test.Net35/ConnectionTests.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Net; using System.Text; using Moq; using NUnit.Framework; @@ -13,6 +15,14 @@ public class ConnectionTests { private Mock _mockRequestHandler; + class RetryErrorTestCase + { + public HttpStatusCode ResponseCode; + public string ErrorMessage; + public int Timeout; + public bool ShouldRetry; + } + [SetUp] public void Init() { @@ -34,6 +44,113 @@ public void CleanUp() Logger.Handlers -= LoggingHandler; } + [Test()] + public void RetryErrorTestNet35() + { + Stopwatch watch = new Stopwatch(); + + // Set invalid host address and make timeout to 1s + var config = new Config().SetAsync(false); + config.SetHost("https://fake.segment-server.com"); + config.SetTimeout(new TimeSpan(0, 0, 1)); + Analytics.Initialize(Constants.WRITE_KEY, config); + + // Calculate working time for Identiy message with invalid host address + watch.Start(); + Actions.Identify(Analytics.Client); + watch.Stop(); + + Assert.AreEqual(1, Analytics.Client.Statistics.Submitted); + Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded); + Assert.AreEqual(1, Analytics.Client.Statistics.Failed); + + // Handling Identify message will take more than 10s even though the timeout is 1s. + // That's because it retries submit when it's failed. + Assert.AreEqual(true, watch.ElapsedMilliseconds > 10000); + } + + [Test()] + public void RetryServerErrorTestNet35() + { + Stopwatch watch = new Stopwatch(); + + string DummyServerUrl = "http://localhost:9696"; + using (var DummyServer = new WebServer(DummyServerUrl)) + { + DummyServer.RunAsync(); + + // Set invalid host address and make timeout to 1s + var config = new Config().SetAsync(false); + config.SetHost(DummyServerUrl); + config.SetTimeout(new TimeSpan(0, 0, 1)); + Analytics.Initialize(Constants.WRITE_KEY, config); + + var TestCases = new RetryErrorTestCase[] + { + // The errors (500 > code >= 400) doesn't require retry + new RetryErrorTestCase() + { + ErrorMessage = "Server Gone", + ResponseCode = HttpStatusCode.Gone, + ShouldRetry = false, + Timeout = 10000 + }, + // 429 error requires retry + new RetryErrorTestCase() + { + ErrorMessage = "Too many requests", + ResponseCode = (HttpStatusCode)429, + ShouldRetry = true, + Timeout = 10000 + }, + // Server errors require retry + new RetryErrorTestCase() + { + ErrorMessage = "Bad Gateway", + ResponseCode = HttpStatusCode.BadGateway, + ShouldRetry = true, + Timeout = 10000 + } + }; + + foreach (var testCase in TestCases) + { + // Setup fallback module which returns error code + DummyServer.RequestHandler = ((req, res) => + { + string pageData = "{ ErrorMessage: '" + testCase.ErrorMessage + "' }"; + byte[] data = Encoding.UTF8.GetBytes(pageData); + + res.StatusCode = (int)testCase.ResponseCode; + res.ContentType = "application/json"; + res.ContentEncoding = Encoding.UTF8; + res.ContentLength64 = data.LongLength; + + res.OutputStream.Write(data, 0, data.Length); + res.Close(); + }); + + // Calculate working time for Identiy message with invalid host address + watch.Start(); + Actions.Identify(Analytics.Client); + watch.Stop(); + + DummyServer.RequestHandler = null; + + Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded); + + // Handling Identify message will less than 10s because the server returns GONE message. + // That's because it retries submit when it's failed. + if (testCase.ShouldRetry) + Assert.IsTrue(watch.ElapsedMilliseconds > testCase.Timeout); + else + Assert.IsFalse(watch.ElapsedMilliseconds > testCase.Timeout); + } + } + } + + + [Test()] public void ProxyTestNet35() { diff --git a/Test.Net35/Test.Net35.csproj b/Test.Net35/Test.Net35.csproj index 85902f64..80c9384d 100644 --- a/Test.Net35/Test.Net35.csproj +++ b/Test.Net35/Test.Net35.csproj @@ -67,6 +67,7 @@ + diff --git a/Test.Net35/WebServer.cs b/Test.Net35/WebServer.cs new file mode 100644 index 00000000..9bc577e1 --- /dev/null +++ b/Test.Net35/WebServer.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading; + +namespace Segment.Test +{ + class WebServer :IDisposable + { + private HttpListener listener; + private Thread runningThread; + private bool runServer; + private string address; + + public delegate void HandleRequest(HttpListenerRequest req, HttpListenerResponse res); + public HandleRequest RequestHandler = null; + + public WebServer(string url) + { + // Only string ends with '/' is acceptable for web server url + address = url; + if (!address.EndsWith("/")) + address += "/"; + } + + public void HandleIncomingConnections() + { + while (runServer) + { + try + { + // Will wait here until we hear from a connection + HttpListenerContext ctx = listener.GetContext(); + + // Peel out the requests and response objects + HttpListenerRequest req = ctx.Request; + HttpListenerResponse res = ctx.Response; + + if (RequestHandler != null) + { + RequestHandler(req, res); + } + else + { + // Write the response info + string pageData = "{}"; + byte[] data = Encoding.UTF8.GetBytes(pageData); + res.ContentType = "application/json"; + res.ContentEncoding = Encoding.UTF8; + res.ContentLength64 = data.LongLength; + + // Write out to the response stream (asynchronously), then close it + res.OutputStream.Write(data, 0, data.Length); + res.Close(); + } + } + catch (System.Exception) + { + runServer = false; + return; + } + } + } + + public void RunAsync() + { + // Stop already running server + Stop(); + + // Create new listener + listener = new HttpListener(); + listener.Prefixes.Add(address); + listener.Start(); + + // Start listening requests + runServer = true; + runningThread = new Thread(HandleIncomingConnections); + runningThread.Start(); + } + + public void Stop() + { + runServer = false; + + if (listener != null) + { + listener.Close(); + listener = null; + } + if (runningThread != null) + { + runningThread.Join(); + runningThread = null; + } + } + + #region IDisposable Support + private bool disposedValue = false; // To detect redundant calls + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + Stop(); + } + + disposedValue = true; + } + } + + public void Dispose() + { + Dispose(true); + } + #endregion + } +} diff --git a/Test.Net45/ConnectionTests.cs b/Test.Net45/ConnectionTests.cs index 033a8331..bde36a57 100644 --- a/Test.Net45/ConnectionTests.cs +++ b/Test.Net45/ConnectionTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Text; using System.Threading.Tasks; using Moq; @@ -36,6 +37,30 @@ public void CleanUp() Analytics.Dispose(); Logger.Handlers -= LoggingHandler; } + [Test()] + public void RetryErrorTestNet45() + { + Stopwatch watch = new Stopwatch(); + + // Set invalid host address and make timeout to 1s + var config = new Config().SetAsync(false); + config.SetHost("https://fake.segment-server.com"); + config.SetTimeout(new TimeSpan(0, 0, 1)); + Analytics.Initialize(Constants.WRITE_KEY, config); + + // Calculate working time for Identiy message with invalid host address + watch.Start(); + Actions.Identify(Analytics.Client); + watch.Stop(); + + Assert.AreEqual(1, Analytics.Client.Statistics.Submitted); + Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded); + Assert.AreEqual(1, Analytics.Client.Statistics.Failed); + + // Handling Identify message will take more than 10s even though the timeout is 1s. + // That's because it retries submit when it's failed. + Assert.AreEqual(true, watch.ElapsedMilliseconds > 10000); + } [Test()] public void ProxyTestNet45() diff --git a/Test.NetStandard20/ConnectionTests.cs b/Test.NetStandard20/ConnectionTests.cs index 9b488707..52c702ae 100644 --- a/Test.NetStandard20/ConnectionTests.cs +++ b/Test.NetStandard20/ConnectionTests.cs @@ -1,7 +1,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Net; using System.Text; using System.Threading.Tasks; +using EmbedIO; +using EmbedIO.Actions; using Moq; using NUnit.Framework; using Segment.Model; @@ -36,6 +40,119 @@ public void CleanUp() Logger.Handlers -= LoggingHandler; } + [Test()] + public void RetryConnectionErrorTestNetStandard20() + { + Stopwatch watch = new Stopwatch(); + + // Set invalid host address and make timeout to 1s + var config = new Config().SetAsync(false); + config.SetHost("https://fake.segment-server.com"); + config.SetTimeout(new TimeSpan(0, 0, 1)); + Analytics.Initialize(Constants.WRITE_KEY, config); + + // Calculate working time for Identiy message with invalid host address + watch.Start(); + Actions.Identify(Analytics.Client); + watch.Stop(); + + Assert.AreEqual(1, Analytics.Client.Statistics.Submitted); + Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded); + Assert.AreEqual(1, Analytics.Client.Statistics.Failed); + + // Handling Identify message will take more than 10s even though the timeout is 1s. + // That's because it retries submit when it's failed. + Assert.AreEqual(true, watch.ElapsedMilliseconds > 10000); + } + + class RetryErrorTestCase + { + public HttpStatusCode ResponseCode; + public string ErrorMessage; + public int Timeout; + public bool ShouldRetry; + public string BaseActionUrl; + } + + [Test()] + public void RetryServerErrorTestNetStandard20() + { + + Stopwatch watch = new Stopwatch(); + string DummyServerUrl = "http://localhost:9696"; + using (var DummyServer = new WebServer(DummyServerUrl)) + { + + // Set invalid host address and make timeout to 1s + var config = new Config().SetAsync(false); + config.SetHost(DummyServerUrl); + config.SetTimeout(new TimeSpan(0, 0, 1)); + Analytics.Initialize(Constants.WRITE_KEY, config); + + var TestCases = new RetryErrorTestCase[] + { + // The errors (500 > code >= 400) doesn't require retry + new RetryErrorTestCase() + { + ErrorMessage = "Server Gone", + ResponseCode = HttpStatusCode.Gone, + ShouldRetry = false, + Timeout = 10000, + BaseActionUrl = "/ServerGone" + }, + // 429 error requires retry + new RetryErrorTestCase() + { + ErrorMessage = "Too many requests", + ResponseCode = (HttpStatusCode)429, + ShouldRetry = true, + Timeout = 10000, + BaseActionUrl = "/TooManyRequests" + }, + // Server errors require retry + new RetryErrorTestCase() + { + ErrorMessage = "Bad Gateway", + ResponseCode = HttpStatusCode.BadGateway, + ShouldRetry = true, + Timeout = 10000, + BaseActionUrl = "/BadGateWay" + } + }; + + foreach (var testCase in TestCases) + { + // Setup Action module which returns error code + var actionModule = new ActionModule(testCase.BaseActionUrl, HttpVerbs.Any,(ctx) => + { + return ctx.SendStandardHtmlAsync((int)testCase.ResponseCode); + }); + DummyServer.WithModule(actionModule); + } + + DummyServer.RunAsync(); + + foreach (var testCase in TestCases) + { + Analytics.Client.Config.SetHost(DummyServerUrl + testCase.BaseActionUrl); + // Calculate working time for Identiy message with invalid host address + watch.Reset(); + watch.Start(); + Actions.Identify(Analytics.Client); + watch.Stop(); + + Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded); + + // Handling Identify message will less than 10s because the server returns GONE message. + // That's because it retries submit when it's failed. + if (testCase.ShouldRetry) + Assert.IsTrue(watch.ElapsedMilliseconds > testCase.Timeout); + else + Assert.IsFalse(watch.ElapsedMilliseconds > testCase.Timeout); + } + } + } + [Test()] public void ProxyTestNetStanard20() { diff --git a/Test.NetStandard20/Test.NetStandard20.csproj b/Test.NetStandard20/Test.NetStandard20.csproj index 8fa73a7f..e9aa8871 100644 --- a/Test.NetStandard20/Test.NetStandard20.csproj +++ b/Test.NetStandard20/Test.NetStandard20.csproj @@ -10,6 +10,7 @@ + diff --git a/Test.UniversalApp/ConnectionTests.cs b/Test.UniversalApp/ConnectionTests.cs index 1de17443..d6538438 100644 --- a/Test.UniversalApp/ConnectionTests.cs +++ b/Test.UniversalApp/ConnectionTests.cs @@ -37,6 +37,31 @@ public void CleanUp() Logger.Handlers -= LoggingHandler; } + [TestMethod] + public void RetryErrorTestPortable() + { + Stopwatch watch = new Stopwatch(); + + // Set invalid host address and make timeout to 1s + var config = new Config().SetAsync(false); + config.SetHost("https://fake.segment-server.com"); + config.SetTimeout(new TimeSpan(0, 0, 1)); + Analytics.Initialize(Constants.WRITE_KEY, config); + + // Calculate working time for Identiy message with invalid host address + watch.Start(); + Actions.Identify(Analytics.Client); + watch.Stop(); + + Assert.AreEqual(1, Analytics.Client.Statistics.Submitted); + Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded); + Assert.AreEqual(1, Analytics.Client.Statistics.Failed); + + // Handling Identify message will take more than 10s even though the timeout is 1s. + // That's because it retries submit when it's failed. + Assert.AreEqual(true, watch.ElapsedMilliseconds > 10000); + } + [TestMethod] public void ProxyTestNetPortable() { diff --git a/Test/ConnectionTests.cs b/Test/ConnectionTests.cs index 78fceaaa..d2c579af 100644 --- a/Test/ConnectionTests.cs +++ b/Test/ConnectionTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Text; using System.Threading.Tasks; using Moq; @@ -36,6 +37,31 @@ public void CleanUp() Logger.Handlers -= LoggingHandler; } + [Test()] + public void RetryErrorTest() + { + Stopwatch watch = new Stopwatch(); + + // Set invalid host address and make timeout to 1s + var config = new Config().SetAsync(false); + config.SetHost("https://fake.segment-server.com"); + config.SetTimeout(new TimeSpan(0, 0, 1)); + Analytics.Initialize(Constants.WRITE_KEY, config); + + // Calculate working time for Identiy message with invalid host address + watch.Start(); + Actions.Identify(Analytics.Client); + watch.Stop(); + + Assert.AreEqual(1, Analytics.Client.Statistics.Submitted); + Assert.AreEqual(0, Analytics.Client.Statistics.Succeeded); + Assert.AreEqual(1, Analytics.Client.Statistics.Failed); + + // Handling Identify message will take more than 10s even though the timeout is 1s. + // That's because it retries submit when it's failed. + Assert.AreEqual(true, watch.ElapsedMilliseconds > 10000); + } + [Test()] public void ProxyTest() {