Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add TelemetrySource to ExecutionRejectedException #2346

Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ public ValueTask IsolateCircuitAsync(ResilienceContext context)

lock (_lock)
{
SetLastHandledOutcome_NeedsLock(Outcome.FromException<T>(new IsolatedCircuitException()));
var exception = new IsolatedCircuitException();
_telemetry.UpdateTelemetrySource(exception);
SetLastHandledOutcome_NeedsLock(Outcome.FromException<T>(exception));
OpenCircuitFor_NeedsLock(Outcome.FromResult<T>(default), TimeSpan.MaxValue, manual: true, context, out task);
_circuitState = CircuitState.Isolated;
}
Expand Down Expand Up @@ -123,7 +125,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)
{
EnsureNotDisposed();

Exception? exception = null;
BrokenCircuitException? exception = null;
bool isHalfOpen = false;

Task? task = null;
Expand Down Expand Up @@ -157,6 +159,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)

if (exception is not null)
{
_telemetry.UpdateTelemetrySource(exception);
return Outcome.FromException<T>(exception);
}

Expand Down Expand Up @@ -308,11 +311,13 @@ private void SetLastHandledOutcome_NeedsLock(Outcome<T> outcome)
private BrokenCircuitException CreateBrokenCircuitException()
{
TimeSpan retryAfter = _blockedUntil - _timeProvider.GetUtcNow();
return _breakingException switch
BrokenCircuitException exception = _breakingException switch
martincostello marked this conversation as resolved.
Show resolved Hide resolved
{
Exception exception => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter, exception),
Exception ex => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter, ex),
_ => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter)
};
_telemetry.UpdateTelemetrySource(exception);
return exception;
}

private void OpenCircuit_NeedsLock(Outcome<T> outcome, bool manual, ResilienceContext context, out Task? scheduledTask)
Expand Down
8 changes: 8 additions & 0 deletions src/Polly.Core/ExecutionRejectedException.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
using System.Runtime.Serialization;
#endif

using Polly.Telemetry;

namespace Polly;

/// <summary>
Expand Down Expand Up @@ -49,4 +51,10 @@ protected ExecutionRejectedException(SerializationInfo info, StreamingContext co
}
#endif
#pragma warning restore RS0016 // Add public types and members to the declared API

/// <summary>
/// Gets the source of the strategy which has thrown the exception, if known.
/// </summary>
public virtual ResilienceTelemetrySource? TelemetrySource { get; internal set; }

martincostello marked this conversation as resolved.
Show resolved Hide resolved
}
2 changes: 2 additions & 0 deletions src/Polly.Core/PublicAPI.Unshipped.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@ Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(string! messa
Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(string! message, System.TimeSpan retryAfter, System.Exception! inner) -> void
Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(System.TimeSpan retryAfter) -> void
Polly.CircuitBreaker.BrokenCircuitException.RetryAfter.get -> System.TimeSpan?
virtual Polly.ExecutionRejectedException.TelemetrySource.get -> Polly.Telemetry.ResilienceTelemetrySource?
Polly.Telemetry.ResilienceStrategyTelemetry.UpdateTelemetrySource(Polly.ExecutionRejectedException! exception) -> void
14 changes: 14 additions & 0 deletions src/Polly.Core/Telemetry/ResilienceStrategyTelemetry.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using System.ComponentModel;

namespace Polly.Telemetry;

/// <summary>
Expand All @@ -21,6 +23,18 @@ internal ResilienceStrategyTelemetry(ResilienceTelemetrySource source, Telemetry

internal ResilienceTelemetrySource TelemetrySource { get; }

/// <summary>
/// Updates the source of the telemetry on the provided exception.
/// </summary>
/// <param name="exception">The to-be-updated exception.</param>
peter-csala marked this conversation as resolved.
Show resolved Hide resolved
[EditorBrowsable(EditorBrowsableState.Never)]
public void UpdateTelemetrySource(ExecutionRejectedException exception)
{
Guard.NotNull(exception);

exception.TelemetrySource = TelemetrySource;
}
martincostello marked this conversation as resolved.
Show resolved Hide resolved

/// <summary>
/// Reports an event that occurred in a resilience strategy.
/// </summary>
Expand Down
1 change: 1 addition & 0 deletions src/Polly.Core/Timeout/TimeoutResilienceStrategy.cs
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ protected internal override async ValueTask<Outcome<TResult>> ExecuteCore<TResul
timeout,
e);

_telemetry.UpdateTelemetrySource(timeoutException);
return Outcome.FromException<TResult>(timeoutException.TrySetStackTrace());
}

Expand Down
21 changes: 8 additions & 13 deletions src/Polly.RateLimiting/RateLimiterRejectedException.cs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ public RateLimiterRejectedException(string message)
/// <param name="message">The message that describes the error.</param>
/// <param name="retryAfter">The retry after value.</param>
public RateLimiterRejectedException(string message, TimeSpan retryAfter)
: base(message) => RetryAfter = retryAfter;
: base(message)
=> RetryAfter = retryAfter;

/// <summary>
/// Initializes a new instance of the <see cref="RateLimiterRejectedException"/> class.
Expand All @@ -63,7 +64,8 @@ public RateLimiterRejectedException(string message, Exception inner)
/// <param name="retryAfter">The retry after value.</param>
/// <param name="inner">The inner exception.</param>
public RateLimiterRejectedException(string message, TimeSpan retryAfter, Exception inner)
: base(message, inner) => RetryAfter = retryAfter;
: base(message, inner)
=> RetryAfter = retryAfter;

/// <summary>
/// Gets the amount of time to wait before retrying again.
Expand All @@ -84,10 +86,10 @@ public RateLimiterRejectedException(string message, TimeSpan retryAfter, Excepti
private RateLimiterRejectedException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
var value = info.GetDouble("RetryAfter");
if (value >= 0.0)
var retryAfter = info.GetDouble(nameof(RetryAfter));
if (retryAfter >= 0.0)
martincostello marked this conversation as resolved.
Show resolved Hide resolved
{
RetryAfter = TimeSpan.FromSeconds(value);
RetryAfter = TimeSpan.FromSeconds(retryAfter);
}
}

Expand All @@ -96,14 +98,7 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont
{
Guard.NotNull(info);

if (RetryAfter.HasValue)
{
info.AddValue("RetryAfter", RetryAfter.Value.TotalSeconds);
}
else
{
info.AddValue("RetryAfter", -1.0);
}
info.AddValue(nameof(RetryAfter), RetryAfter.HasValue ? RetryAfter.Value.TotalSeconds : -1.0);
martincostello marked this conversation as resolved.
Show resolved Hide resolved

base.GetObjectData(info, context);
}
Expand Down
6 changes: 5 additions & 1 deletion src/Polly.RateLimiting/RateLimiterResilienceStrategy.cs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,11 @@ protected override async ValueTask<Outcome<TResult>> ExecuteCore<TResult, TState
await OnLeaseRejected(new OnRateLimiterRejectedArguments(context, lease)).ConfigureAwait(context.ContinueOnCapturedContext);
}

var exception = retryAfter.HasValue ? new RateLimiterRejectedException(retryAfter.Value) : new RateLimiterRejectedException();
var exception = retryAfter.HasValue
? new RateLimiterRejectedException(retryAfter.Value)
: new RateLimiterRejectedException();
martincostello marked this conversation as resolved.
Show resolved Hide resolved

_telemetry.UpdateTelemetrySource(exception);

return Outcome.FromException<TResult>(exception.TrySetStackTrace());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ public async Task IsolateAsync_Ok()
called.Should().BeTrue();

var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
outcome.Value.Exception.Should().BeOfType<IsolatedCircuitException>()
.And.Subject.As<IsolatedCircuitException>().RetryAfter.Should().BeNull();
var exception = outcome.Value.Exception.Should().BeOfType<IsolatedCircuitException>().Subject;
exception.RetryAfter.Should().BeNull();
exception.TelemetrySource.Should().NotBeNull();

// now close it
await controller.CloseCircuitAsync(ResilienceContextPool.Shared.Get());
Expand Down Expand Up @@ -119,8 +120,9 @@ public async Task OnActionPreExecute_CircuitOpenedByValue()
using var controller = CreateController();

await OpenCircuit(controller, Outcome.FromResult(99));
var error = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
error.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
var exception = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
exception.RetryAfter.Should().NotBeNull();
exception.TelemetrySource.Should().NotBeNull();

GetBlockedTill(controller).Should().Be(_timeProvider.GetUtcNow() + _options.BreakDuration);
}
Expand Down Expand Up @@ -149,6 +151,7 @@ await OpenCircuit(
stacks.Add(e.StackTrace!);
e.Message.Should().Be("The circuit is now open and is not allowing calls.");
e.RetryAfter.Should().NotBeNull();
e.TelemetrySource.Should().NotBeNull();

if (innerException)
{
Expand Down Expand Up @@ -206,9 +209,10 @@ public async Task OnActionPreExecute_CircuitOpenedByException()
using var controller = CreateController();

await OpenCircuit(controller, Outcome.FromException<int>(new InvalidOperationException()));
var error = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
error.InnerException.Should().BeOfType<InvalidOperationException>();
error.RetryAfter.Should().NotBeNull();
var exception = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
exception.InnerException.Should().BeOfType<InvalidOperationException>();
exception.RetryAfter.Should().NotBeNull();
exception.TelemetrySource.Should().NotBeNull();
}

[Fact]
Expand Down Expand Up @@ -261,9 +265,11 @@ public async Task OnActionPreExecute_HalfOpen()
// act
await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
var error = (await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception;
error.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();

// assert
var exception = error.Should().BeOfType<BrokenCircuitException>().Subject;
exception.RetryAfter.Should().NotBeNull();
exception.TelemetrySource.Should().NotBeNull();
controller.CircuitState.Should().Be(CircuitState.HalfOpen);
called.Should().BeTrue();
}
Expand Down Expand Up @@ -465,7 +471,9 @@ public async Task OnActionFailureAsync_VoidResult_EnsureBreakingExceptionNotSet(
// assert
controller.LastException.Should().BeNull();
var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
var exception = outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().Subject;
exception.RetryAfter.Should().NotBeNull();
exception.TelemetrySource.Should().NotBeNull();
}

[Fact]
Expand Down Expand Up @@ -501,8 +509,9 @@ public async Task Flow_Closed_HalfOpen_Open_HalfOpen_Closed()
TimeSpan advanceTimeRejected = TimeSpan.FromMilliseconds(1);
AdvanceTime(advanceTimeRejected);
var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>()
.And.Subject.As<BrokenCircuitException>().RetryAfter.Should().Be(_options.BreakDuration - advanceTimeRejected);
var exception = outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().Subject;
exception.RetryAfter.Should().Be(_options.BreakDuration - advanceTimeRejected);
exception.TelemetrySource.Should().NotBeNull();

// wait and try, transition to half open
AdvanceTime(_options.BreakDuration + _options.BreakDuration);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Polly.Telemetry;
using Polly.Timeout;

namespace Polly.Core.Tests.Telemetry;

Expand Down Expand Up @@ -94,4 +95,25 @@ public void Report_NoListener_ShouldNotThrow()
.Should()
.NotThrow();
}

[Fact]
public void UpdateTelemetrySource_Ok()
{
var sut = new ResilienceStrategyTelemetry(_source, null);
var exception = new TimeoutRejectedException();

sut.UpdateTelemetrySource(exception);

exception.TelemetrySource.Should().Be(_source);
}

[Fact]
public void UpdateTelemetrySource_ShouldThrow()
{
ExecutionRejectedException? exception = null;

_sut.Invoking(s => s.UpdateTelemetrySource(exception!))
.Should()
.Throw<ArgumentNullException>();
}
}
21 changes: 21 additions & 0 deletions test/Polly.Core.Tests/Timeout/TimeoutResilienceStrategyTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,27 @@ public async Task Execute_Timeout_EnsureStackTrace()
}
}

[Fact]
public async Task Execute_Timeout_EnsureTelemetrySource()
{
SetTimeout(TimeSpan.FromSeconds(2));
var sut = CreateSut();

var outcome = await sut.ExecuteOutcomeAsync(
async (c, _) =>
{
var delay = _timeProvider.Delay(TimeSpan.FromSeconds(4), c.CancellationToken);
_timeProvider.Advance(TimeSpan.FromSeconds(2));
await delay;

return Outcome.FromResult("dummy");
},
ResilienceContextPool.Shared.Get(),
"state");

outcome.Exception.Should().BeOfType<TimeoutRejectedException>().Subject.TelemetrySource.Should().NotBeNull();
}

[Fact]
public async Task Execute_Cancelled_EnsureNoTimeout()
{
Expand Down
66 changes: 57 additions & 9 deletions test/Polly.RateLimiting.Tests/RateLimiterRejectedExceptionTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,67 @@ namespace Polly.Core.Tests.Timeout;

public class RateLimiterRejectedExceptionTests
{
private readonly string _message = "dummy";
private readonly TimeSpan _retryAfter = TimeSpan.FromSeconds(4);

[Fact]
public void Ctor_Ok()
{
var retryAfter = TimeSpan.FromSeconds(4);
var exception = new RateLimiterRejectedException();
exception.InnerException.Should().BeNull();
exception.Message.Should().Be("The operation could not be executed because it was rejected by the rate limiter.");
exception.RetryAfter.Should().BeNull();
exception.TelemetrySource.Should().BeNull();
}

[Fact]
public void Ctor_RetryAfter_Ok()
{
var exception = new RateLimiterRejectedException(_retryAfter);
exception.InnerException.Should().BeNull();
exception.Message.Should().Be($"The operation could not be executed because it was rejected by the rate limiter. It can be retried after '00:00:04'.");
exception.RetryAfter.Should().Be(_retryAfter);
exception.TelemetrySource.Should().BeNull();
}

[Fact]
public void Ctor_Message_Ok()
{
var exception = new RateLimiterRejectedException(_message);
exception.InnerException.Should().BeNull();
exception.Message.Should().Be(_message);
exception.RetryAfter.Should().BeNull();
exception.TelemetrySource.Should().BeNull();
}

[Fact]
public void Ctor_Message_RetryAfter_Ok()
{
var exception = new RateLimiterRejectedException(_message, _retryAfter);
exception.InnerException.Should().BeNull();
exception.Message.Should().Be(_message);
exception.RetryAfter.Should().Be(_retryAfter);
exception.TelemetrySource.Should().BeNull();
}

[Fact]
public void Ctor_Message_InnerException_Ok()
{
var exception = new RateLimiterRejectedException(_message, new InvalidOperationException());
exception.InnerException.Should().BeOfType<InvalidOperationException>();
exception.Message.Should().Be(_message);
exception.RetryAfter.Should().BeNull();
exception.TelemetrySource.Should().BeNull();
}

new RateLimiterRejectedException().Message.Should().Be("The operation could not be executed because it was rejected by the rate limiter.");
new RateLimiterRejectedException().RetryAfter.Should().BeNull();
new RateLimiterRejectedException("dummy").Message.Should().Be("dummy");
new RateLimiterRejectedException("dummy", new InvalidOperationException()).Message.Should().Be("dummy");
new RateLimiterRejectedException(retryAfter).RetryAfter.Should().Be(retryAfter);
new RateLimiterRejectedException(retryAfter).Message.Should().Be($"The operation could not be executed because it was rejected by the rate limiter. It can be retried after '{retryAfter}'.");
new RateLimiterRejectedException("dummy", retryAfter).RetryAfter.Should().Be(retryAfter);
new RateLimiterRejectedException("dummy", retryAfter, new InvalidOperationException()).RetryAfter.Should().Be(retryAfter);
[Fact]
public void Ctor_Message_RetryAfter_InnerException_Ok()
{
var exception = new RateLimiterRejectedException(_message, _retryAfter, new InvalidOperationException());
exception.InnerException.Should().BeOfType<InvalidOperationException>();
exception.Message.Should().Be(_message);
exception.RetryAfter.Should().Be(_retryAfter);
exception.TelemetrySource.Should().BeNull();
}

#if !NETCOREAPP
Expand Down
Loading