diff --git a/src/TagzApp.AppHost/DatabaseService.cs b/src/TagzApp.AppHost/DatabaseService.cs index b3b337c3..6b621b34 100644 --- a/src/TagzApp.AppHost/DatabaseService.cs +++ b/src/TagzApp.AppHost/DatabaseService.cs @@ -12,7 +12,8 @@ public static class DatabaseConfig public static IDistributedApplicationBuilder AddDatabase( this IDistributedApplicationBuilder builder , out IResourceBuilder db - , out IResourceBuilder securityDb) + , out IResourceBuilder securityDb + , out IResourceBuilder migration) { var dbPassword = builder.AddParameter("dbPassword", false); @@ -25,6 +26,13 @@ this IDistributedApplicationBuilder builder securityDb = dbServer.AddDatabase("securitydb"); + + migration = builder.AddProject("db-migrations") + .WaitFor(db) + .WaitFor(securityDb) + .WithReference(db) + .WithReference(securityDb); + return builder; } diff --git a/src/TagzApp.AppHost/Program.cs b/src/TagzApp.AppHost/Program.cs index 68be5808..f1566bf5 100644 --- a/src/TagzApp.AppHost/Program.cs +++ b/src/TagzApp.AppHost/Program.cs @@ -3,7 +3,10 @@ var builder = DistributedApplication.CreateBuilder(args); -builder.AddDatabase(out var db, out var securityDb); +builder.AddDatabase( + out var db, + out var securityDb, + out var migration); var twitchCache = builder.AddRedis("twitchCache"); var twitchRelay = builder.AddExecutable("twitchrelay", @@ -15,6 +18,7 @@ #region Website var tagzAppWeb = builder.AddProject("web", "https") + .WaitForCompletion(migration) .WithReference(db) .WithReference(securityDb) .WithEnvironment("TwitchRelayUri", "http://localhost:7082"); diff --git a/src/TagzApp.AppHost/TagzApp.AppHost.csproj b/src/TagzApp.AppHost/TagzApp.AppHost.csproj index 02ca5922..c300cb0d 100644 --- a/src/TagzApp.AppHost/TagzApp.AppHost.csproj +++ b/src/TagzApp.AppHost/TagzApp.AppHost.csproj @@ -18,7 +18,9 @@ + + diff --git a/src/TagzApp.AppHostExtensions/HealthCheckAnnotation.cs b/src/TagzApp.AppHostExtensions/HealthCheckAnnotation.cs new file mode 100644 index 00000000..2e644ac6 --- /dev/null +++ b/src/TagzApp.AppHostExtensions/HealthCheckAnnotation.cs @@ -0,0 +1,32 @@ +using System; +using Aspire.Hosting.ApplicationModel; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Aspire.Hosting; + +/// +/// An annotation that associates a health check factory with a resource +/// +/// A function that creates the health check +public class HealthCheckAnnotation(Func> healthCheckFactory) : IResourceAnnotation +{ + public Func> HealthCheckFactory { get; } = healthCheckFactory; + + public static HealthCheckAnnotation Create(Func connectionStringFactory) + { + return new(async (resource, token) => + { + if (resource is not IResourceWithConnectionString c) + { + return null; + } + + if (await c.GetConnectionStringAsync(token) is not string cs) + { + return null; + } + + return connectionStringFactory(cs); + }); + } +} \ No newline at end of file diff --git a/src/TagzApp.AppHostExtensions/PostgreSqlHealthCheckExtensions.cs b/src/TagzApp.AppHostExtensions/PostgreSqlHealthCheckExtensions.cs new file mode 100644 index 00000000..057f6f84 --- /dev/null +++ b/src/TagzApp.AppHostExtensions/PostgreSqlHealthCheckExtensions.cs @@ -0,0 +1,23 @@ +using Aspire.Hosting.ApplicationModel; +using HealthChecks.NpgSql; + +namespace Aspire.Hosting; + +public static class PostgreSqlHealthCheckExtensions +{ + /// + /// Adds a health check to the PostgreSQL server resource. + /// + public static IResourceBuilder WithHealthCheck(this IResourceBuilder builder) + { + return builder.WithAnnotation(HealthCheckAnnotation.Create(cs => new NpgSqlHealthCheck(new NpgSqlHealthCheckOptions(cs)))); + } + + /// + /// Adds a health check to the PostgreSQL database resource. + /// + public static IResourceBuilder WithHealthCheck(this IResourceBuilder builder) + { + return builder.WithAnnotation(HealthCheckAnnotation.Create(cs => new NpgSqlHealthCheck(new NpgSqlHealthCheckOptions(cs)))); + } +} diff --git a/src/TagzApp.AppHostExtensions/RedisResourceHealthCheckExtensions.cs b/src/TagzApp.AppHostExtensions/RedisResourceHealthCheckExtensions.cs new file mode 100644 index 00000000..9e8fbb00 --- /dev/null +++ b/src/TagzApp.AppHostExtensions/RedisResourceHealthCheckExtensions.cs @@ -0,0 +1,15 @@ +using Aspire.Hosting.ApplicationModel; +using HealthChecks.Redis; + +namespace Aspire.Hosting; + +public static class RedisResourceHealthCheckExtensions +{ + /// + /// Adds a health check to the Redis server resource. + /// + public static IResourceBuilder WithHealthCheck(this IResourceBuilder builder) + { + return builder.WithAnnotation(HealthCheckAnnotation.Create(cs => new RedisHealthCheck(cs))); + } +} diff --git a/src/TagzApp.AppHostExtensions/TagzApp.AppHostExtensions.csproj b/src/TagzApp.AppHostExtensions/TagzApp.AppHostExtensions.csproj new file mode 100644 index 00000000..e03103bc --- /dev/null +++ b/src/TagzApp.AppHostExtensions/TagzApp.AppHostExtensions.csproj @@ -0,0 +1,18 @@ + + + + net8.0 + enable + enable + + + + + + + + + + + + diff --git a/src/TagzApp.AppHostExtensions/WaitForDependenciesExtensions.cs b/src/TagzApp.AppHostExtensions/WaitForDependenciesExtensions.cs new file mode 100644 index 00000000..0e02d57a --- /dev/null +++ b/src/TagzApp.AppHostExtensions/WaitForDependenciesExtensions.cs @@ -0,0 +1,301 @@ +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Lifecycle; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Polly.Retry; +using Polly; +using System.Collections.Concurrent; +using System.Runtime.ExceptionServices; + +namespace Aspire.Hosting; + +public static class WaitForDependenciesExtensions +{ + /// + /// Wait for a resource to be running before starting another resource. + /// + /// The resource type. + /// The resource builder. + /// The resource to wait for. + public static IResourceBuilder WaitFor(this IResourceBuilder builder, IResourceBuilder other) + where T : IResource + { + builder.ApplicationBuilder.AddWaitForDependencies(); + return builder.WithAnnotation(new WaitOnAnnotation(other.Resource)); + } + + /// + /// Wait for a resource to run to completion before starting another resource. + /// + /// The resource type. + /// The resource builder. + /// The resource to wait for. + public static IResourceBuilder WaitForCompletion(this IResourceBuilder builder, IResourceBuilder other) + where T : IResource + { + builder.ApplicationBuilder.AddWaitForDependencies(); + return builder.WithAnnotation(new WaitOnAnnotation(other.Resource) { WaitUntilCompleted = true }); + } + + /// + /// Adds a lifecycle hook that waits for all dependencies to be "running" before starting resources. If that resource + /// has a health check, it will be executed before the resource is considered "running". + /// + /// The . + private static IDistributedApplicationBuilder AddWaitForDependencies(this IDistributedApplicationBuilder builder) + { + builder.Services.TryAddLifecycleHook(); + return builder; + } + + private class WaitOnAnnotation(IResource resource) : IResourceAnnotation + { + public IResource Resource { get; } = resource; + + public string[]? States { get; set; } + + public bool WaitUntilCompleted { get; set; } + } + + private class WaitForDependenciesRunningHook(DistributedApplicationExecutionContext executionContext, + ResourceNotificationService resourceNotificationService) : + IDistributedApplicationLifecycleHook, + IAsyncDisposable + { + private readonly CancellationTokenSource _cts = new(); + + public Task BeforeStartAsync(DistributedApplicationModel appModel, CancellationToken cancellationToken = default) + { + // We don't need to execute any of this logic in publish mode + if (executionContext.IsPublishMode) + { + return Task.CompletedTask; + } + + // The global list of resources being waited on + var waitingResources = new ConcurrentDictionary>(); + + // For each resource, add an environment callback that waits for dependencies to be running + foreach (var r in appModel.Resources) + { + var resourcesToWaitOn = r.Annotations.OfType().ToLookup(a => a.Resource); + + if (resourcesToWaitOn.Count == 0) + { + continue; + } + + // Abuse the environment callback to wait for dependencies to be running + + r.Annotations.Add(new EnvironmentCallbackAnnotation(async context => + { + var dependencies = new List(); + + // Find connection strings and endpoint references and get the resource they point to + foreach (var group in resourcesToWaitOn) + { + var resource = group.Key; + + // REVIEW: This logic does not handle cycles in the dependency graph (that would result in a deadlock) + + // Don't wait for yourself + if (resource != r && resource is not null) + { + var pendingAnnotations = waitingResources.GetOrAdd(resource, _ => new()); + + foreach (var waitOn in group) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + async Task Wait() + { + context.Logger?.LogInformation("Waiting for {Resource}.", waitOn.Resource.Name); + + await tcs.Task; + + context.Logger?.LogInformation("Waiting for {Resource} completed.", waitOn.Resource.Name); + } + + pendingAnnotations[waitOn] = tcs; + + dependencies.Add(Wait()); + } + } + } + + await resourceNotificationService.PublishUpdateAsync(r, s => s with + { + State = new("Waiting", KnownResourceStateStyles.Info) + }); + + await Task.WhenAll(dependencies).WaitAsync(context.CancellationToken); + })); + } + + _ = Task.Run(async () => + { + var stoppingToken = _cts.Token; + + // These states are terminal but we need a better way to detect that + static bool IsKnownTerminalState(CustomResourceSnapshot snapshot) => + snapshot.State == "FailedToStart" || + snapshot.State == "Exited" || + snapshot.ExitCode is not null; + + // Watch for global resource state changes + await foreach (var resourceEvent in resourceNotificationService.WatchAsync(stoppingToken)) + { + if (waitingResources.TryGetValue(resourceEvent.Resource, out var pendingAnnotations)) + { + foreach (var (waitOn, tcs) in pendingAnnotations) + { + if (waitOn.States is string[] states && states.Contains(resourceEvent.Snapshot.State?.Text, StringComparer.Ordinal)) + { + pendingAnnotations.TryRemove(waitOn, out _); + + _ = DoTheHealthCheck(resourceEvent, tcs); + } + else if (waitOn.WaitUntilCompleted) + { + if (IsKnownTerminalState(resourceEvent.Snapshot)) + { + pendingAnnotations.TryRemove(waitOn, out _); + + _ = DoTheHealthCheck(resourceEvent, tcs); + } + } + else if (waitOn.States is null) + { + if (resourceEvent.Snapshot.State == "Running") + { + pendingAnnotations.TryRemove(waitOn, out _); + + _ = DoTheHealthCheck(resourceEvent, tcs); + } + else if (IsKnownTerminalState(resourceEvent.Snapshot)) + { + pendingAnnotations.TryRemove(waitOn, out _); + + tcs.TrySetException(new Exception($"Dependency {waitOn.Resource.Name} failed to start")); + } + } + } + } + } + }, + cancellationToken); + + return Task.CompletedTask; + } + + private async Task DoTheHealthCheck(ResourceEvent resourceEvent, TaskCompletionSource tcs) + { + var resource = resourceEvent.Resource; + + // REVIEW: Right now, every resource does an independent health check, we could instead cache + // the health check result and reuse it for all resources that depend on the same resource + + + HealthCheckAnnotation? healthCheckAnnotation = null; + + // Find the relevant health check annotation. If the resource has a parent, walk up the tree + // until we find the health check annotation. + while (true) + { + // If we find a health check annotation, break out of the loop + if (resource.TryGetLastAnnotation(out healthCheckAnnotation)) + { + break; + } + + // If the resource has a parent, walk up the tree + if (resource is IResourceWithParent parent) + { + resource = parent.Parent; + } + else + { + break; + } + } + + Func? operation = null; + + if (healthCheckAnnotation?.HealthCheckFactory is { } factory) + { + IHealthCheck? check; + + try + { + // TODO: Do need to pass a cancellation token here? + check = await factory(resource, default); + + if (check is not null) + { + var context = new HealthCheckContext() + { + Registration = new HealthCheckRegistration("", check, HealthStatus.Unhealthy, []) + }; + + operation = async (cancellationToken) => + { + var result = await check.CheckHealthAsync(context, cancellationToken); + + if (result.Exception is not null) + { + ExceptionDispatchInfo.Throw(result.Exception); + } + + if (result.Status != HealthStatus.Healthy) + { + throw new Exception("Health check failed"); + } + }; + } + } + catch (Exception ex) + { + tcs.TrySetException(ex); + + return; + } + } + + try + { + if (operation is not null) + { + var pipeline = CreateResiliencyPipeline(); + + await pipeline.ExecuteAsync(operation); + } + + tcs.TrySetResult(); + } + catch (Exception ex) + { + tcs.TrySetException(ex); + } + } + + private static ResiliencePipeline CreateResiliencyPipeline() + { + var retryUntilCancelled = new RetryStrategyOptions() + { + ShouldHandle = new PredicateBuilder().Handle(), + BackoffType = DelayBackoffType.Exponential, + MaxRetryAttempts = 5, + UseJitter = true, + MaxDelay = TimeSpan.FromSeconds(30) + }; + + return new ResiliencePipelineBuilder().AddRetry(retryUntilCancelled).Build(); + } + + public ValueTask DisposeAsync() + { + _cts.Cancel(); + return default; + } + } +} diff --git a/src/TagzApp.DatabaseMigration/Program.cs b/src/TagzApp.DatabaseMigration/Program.cs index 24e8cf91..5c5c519d 100644 --- a/src/TagzApp.DatabaseMigration/Program.cs +++ b/src/TagzApp.DatabaseMigration/Program.cs @@ -1,14 +1,22 @@ using Aspireify.Data.MigrationService; -using Aspireify.Data.Security; using Microsoft.EntityFrameworkCore; +using TagzApp.Security; +using TagzApp.Storage.Postgres; +using TagzApp.Storage.Postgres.Security.Migrations; var builder = Host.CreateApplicationBuilder(args); builder.AddServiceDefaults(); builder.Services.AddHostedService(); -builder.Services.AddDbContext( - options => options.UseNpgsql(builder.Configuration.GetConnectionString("security")) +builder.Services.AddDbContext( + options => options.UseNpgsql(builder.Configuration.GetConnectionString("securitydb"), options => { + options.MigrationsAssembly(typeof(SecurityContextModelSnapshot).Assembly.FullName); + }) +); + +builder.Services.AddDbContext( + options => options.UseNpgsql(builder.Configuration.GetConnectionString("tagzappdb")) ); builder.Services.AddOpenTelemetry() diff --git a/src/TagzApp.DatabaseMigration/Aspireify.Data.MigrationService.csproj b/src/TagzApp.DatabaseMigration/TagzApp.MigrationService.csproj similarity index 74% rename from src/TagzApp.DatabaseMigration/Aspireify.Data.MigrationService.csproj rename to src/TagzApp.DatabaseMigration/TagzApp.MigrationService.csproj index 5433dd22..9566abe3 100644 --- a/src/TagzApp.DatabaseMigration/Aspireify.Data.MigrationService.csproj +++ b/src/TagzApp.DatabaseMigration/TagzApp.MigrationService.csproj @@ -18,7 +18,8 @@ - - - + + + + diff --git a/src/TagzApp.DatabaseMigration/Worker.cs b/src/TagzApp.DatabaseMigration/Worker.cs index 0981255d..7eaa5584 100644 --- a/src/TagzApp.DatabaseMigration/Worker.cs +++ b/src/TagzApp.DatabaseMigration/Worker.cs @@ -1,8 +1,8 @@ -using Aspireify.Data.Security; using Microsoft.EntityFrameworkCore; using OpenTelemetry.Trace; using System.Diagnostics; -using System.Threading; +using TagzApp.Security; +using TagzApp.Storage.Postgres; namespace Aspireify.Data.MigrationService; @@ -32,7 +32,23 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) try { using var scope = serviceProvider.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await dbContext.Database.MigrateAsync(stoppingToken); + + } + catch (Exception ex) + { + activity?.RecordException(ex); + throw; + } + + using var tagzActivity = s_activitySource.StartActivity("Migrating database", ActivityKind.Client); + + try + { + using var scope = serviceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); await dbContext.Database.MigrateAsync(stoppingToken); diff --git a/src/TagzApp.sln b/src/TagzApp.sln index 88f2dd21..20afc44e 100644 --- a/src/TagzApp.sln +++ b/src/TagzApp.sln @@ -69,6 +69,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TagzApp.AppHost", "TagzApp. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TagzApp.ServiceDefaults", "TagzApp.ServiceDefaults\TagzApp.ServiceDefaults.csproj", "{290F15DE-B3E8-4FCA-ABF5-C223D611BA6B}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TagzApp.MigrationService", "TagzApp.DatabaseMigration\TagzApp.MigrationService.csproj", "{F1F4F081-22D3-4F8A-AD34-8BD270848F8E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TagzApp.AppHostExtensions", "TagzApp.AppHostExtensions\TagzApp.AppHostExtensions.csproj", "{8A628B08-E52F-423D-BE84-FDB3F8A09DCF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -167,6 +171,14 @@ Global {290F15DE-B3E8-4FCA-ABF5-C223D611BA6B}.Debug|Any CPU.Build.0 = Debug|Any CPU {290F15DE-B3E8-4FCA-ABF5-C223D611BA6B}.Release|Any CPU.ActiveCfg = Release|Any CPU {290F15DE-B3E8-4FCA-ABF5-C223D611BA6B}.Release|Any CPU.Build.0 = Release|Any CPU + {F1F4F081-22D3-4F8A-AD34-8BD270848F8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F1F4F081-22D3-4F8A-AD34-8BD270848F8E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F1F4F081-22D3-4F8A-AD34-8BD270848F8E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F1F4F081-22D3-4F8A-AD34-8BD270848F8E}.Release|Any CPU.Build.0 = Release|Any CPU + {8A628B08-E52F-423D-BE84-FDB3F8A09DCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A628B08-E52F-423D-BE84-FDB3F8A09DCF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A628B08-E52F-423D-BE84-FDB3F8A09DCF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A628B08-E52F-423D-BE84-FDB3F8A09DCF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -195,6 +207,8 @@ Global {A64B6AB5-2F9E-4FBD-A622-49598B904E64} = {370455D5-6EA6-44C1-B315-B621F7B6A954} {70CA3D81-8CBB-4C63-9D8D-ADE58CCE5FE8} = {6B73C62C-6CC8-4C85-A24F-FA84489B3137} {290F15DE-B3E8-4FCA-ABF5-C223D611BA6B} = {6B73C62C-6CC8-4C85-A24F-FA84489B3137} + {F1F4F081-22D3-4F8A-AD34-8BD270848F8E} = {370455D5-6EA6-44C1-B315-B621F7B6A954} + {8A628B08-E52F-423D-BE84-FDB3F8A09DCF} = {6B73C62C-6CC8-4C85-A24F-FA84489B3137} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {F2AD77A5-5B2D-41FC-AD37-8EF8A4E54410}