Have lexbox API container check fw-headless health (#1192)

Since fw-headless container isn't exposed to outside, but we might want
to know if it's healthy, we'll add code into the lexbox-api health check
to call the fw-headless health check and report lexbox-api's status as
unhealthy if fw-headless is not healthy. (Except on local dev, where
lexbox-api's status would just be Degraded so that editing fw-headless
code doesn't bring down the whole API server).

backend/FwHeadless/Program.cs

@@ -25,6 +25,14 @@
 
 var app = builder.Build();
 
+// Add lexbox-version header to all requests
+app.Logger.LogInformation("FwHeadless version: {version}", AppVersionService.Version);
+app.Use(async (context, next) =>
+{
+    context.Response.Headers["lexbox-version"] = AppVersionService.Version;
+    await next();
+});
+
 // Configure the HTTP request pipeline.
 if (app.Environment.IsDevelopment())
 {

backend/FwHeadless/Services/AppVersionService.cs

@@ -0,0 +1,9 @@
+using System.Reflection;
+
+namespace FwHeadless;
+
+public static class AppVersionService
+{
+    public static readonly string Version = typeof(AppVersionService).Assembly
+        .GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion ?? "dev";
+}

backend/LexBoxApi/Config/HealthChecksConfig.cs

@@ -0,0 +1,7 @@
+namespace LexBoxApi.Config;
+
+public class HealthChecksConfig
+{
+    public bool RequireFwHeadlessContainerVersionMatch { get; init; } = true;
+    public bool RequireHealthyFwHeadlessContainer { get; init; } = true;
+}

backend/LexBoxApi/LexBoxKernel.cs

@@ -44,6 +44,10 @@ public static void AddLexBoxApi(this IServiceCollection services,
             .BindConfiguration("Tus")
             .ValidateDataAnnotations()
             .ValidateOnStart();
+        services.AddOptions<HealthChecksConfig>()
+            .BindConfiguration("HealthChecks")
+            .ValidateDataAnnotations()
+            .ValidateOnStart();
         services.AddHttpClient();
         services.AddHttpContextAccessor();
         services.AddMemoryCache();
@@ -57,6 +61,7 @@ public static void AddLexBoxApi(this IServiceCollection services,
         services.AddScoped<IHgService, HgService>();
         services.AddHostedService<HgService>();
         services.AddTransient<HgWebHealthCheck>();
+        services.AddTransient<FwHeadlessHealthCheck>();
         services.AddScoped<IIsLanguageForgeProjectDataLoader, IsLanguageForgeProjectDataLoader>();
         services.AddResiliencePipeline<string, IReadOnlyDictionary<string, bool>>(IsLanguageForgeProjectDataLoader.ResiliencePolicyName, (builder, context) =>
         {
@@ -69,7 +74,9 @@ public static void AddLexBoxApi(this IServiceCollection services,
         if (environment.IsDevelopment())
             services.AddHostedService<SwaggerValidationService>();
         services.AddScheduledTasks(configuration);
-        services.AddHealthChecks().AddCheck<HgWebHealthCheck>("hgweb", HealthStatus.Unhealthy, ["hg"], TimeSpan.FromSeconds(5));
+        services.AddHealthChecks()
+            .AddCheck<HgWebHealthCheck>("hgweb", HealthStatus.Unhealthy, ["hg"], TimeSpan.FromSeconds(5))
+            .AddCheck<FwHeadlessHealthCheck>("fw-headless", HealthStatus.Unhealthy, ["fw-headless"], TimeSpan.FromSeconds(5));
         services.AddSyncProxy();
         AuthKernel.AddLexBoxAuth(services, configuration, environment);
         services.AddLexGraphQL(environment);

backend/LexBoxApi/Services/FwHeadlessHealthCheck.cs

@@ -0,0 +1,37 @@
+using LexBoxApi.Config;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Options;
+
+namespace LexBoxApi.Services;
+
+public class FwHeadlessHealthCheck(IHttpClientFactory clientFactory, IOptions<HealthChecksConfig> healthCheckOptions) : IHealthCheck
+{
+    public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context,
+        CancellationToken cancellationToken = new())
+    {
+        var http = clientFactory.CreateClient();
+        var fwHeadlessResponse = await http.GetAsync("http://fw-headless/api/healthz");
+        if (!fwHeadlessResponse.IsSuccessStatusCode)
+        {
+            if (healthCheckOptions.Value.RequireHealthyFwHeadlessContainer)
+            {
+                return HealthCheckResult.Unhealthy("fw-headless not repsonding to health check");
+            }
+            else
+            {
+                return HealthCheckResult.Degraded("fw-headless not repsonding to health check");
+            }
+        }
+        var fwHeadlessVersion = fwHeadlessResponse.Headers.GetValues("lexbox-version").FirstOrDefault();
+        if (healthCheckOptions.Value.RequireFwHeadlessContainerVersionMatch && string.IsNullOrEmpty(fwHeadlessVersion))
+        {
+            return HealthCheckResult.Degraded("fw-headless version check failed to return a value");
+        }
+        if (healthCheckOptions.Value.RequireFwHeadlessContainerVersionMatch && fwHeadlessVersion != AppVersionService.Version)
+        {
+            return HealthCheckResult.Degraded(
+                $"api version: '{AppVersionService.Version}' fw-headless version: '{fwHeadlessVersion}' mismatch");
+        }
+        return HealthCheckResult.Healthy();
+    }
+}

backend/LexBoxApi/appsettings.Development.json

@@ -49,6 +49,10 @@
     "LfMergeTrustToken": "lf-merge-dev-trust-token",
     "AutoUpdateLexEntryCountOnSendReceive": true
   },
+  "HealthChecks": {
+    "RequireFwHeadlessContainerVersionMatch": false,
+    "RequireHealthyFwHeadlessContainer": false
+  },
   "Authentication": {
     "Jwt": {
       "Secret": "d5cf1adc-16e6-4064-8041-4cfa00174210"

deployment/develop/lexbox-deployment.patch.yaml

@@ -26,3 +26,5 @@ spec:
               value: "https://develop.lexbox.org"
             - name: HgConfig__RequireContainerVersionMatch
               value: "false"
+            - name: HealthChecksConfig__RequireFwHeadlessContainerVersionMatch
+              value: "false"

deployment/local-dev/lexbox-deployment.patch.yaml

@@ -36,6 +36,10 @@ spec:
             valueFrom:
           - name: CloudFlare__AllowDomain
             value: "mailinator.com"
+          - name: HealthChecksConfig__RequireFwHeadlessContainerVersionMatch
+            value: "false"
+          - name: HealthChecksConfig__RequireHealthyFwHeadlessContainer
+            value: "false"
           - name: Email__SmtpUser
             value: 'maildev'
             valueFrom: