From b46c26ebbc55bf2958c08a58404d79e964ebd107 Mon Sep 17 00:00:00 2001 From: napalm684 Date: Wed, 21 Feb 2024 20:39:30 -0500 Subject: [PATCH 1/9] Added OTEL for tracing, metrics, logging (Console and OTLP exporters) --- src/TagzApp.Blazor/OpenTelemetryExtensions.cs | 138 ++++++++++++++++++ src/TagzApp.Blazor/Program.cs | 6 + src/TagzApp.Blazor/TagzApp.Blazor.csproj | 7 + src/TagzApp.Blazor/appsettings.json | 10 ++ 4 files changed, 161 insertions(+) create mode 100644 src/TagzApp.Blazor/OpenTelemetryExtensions.cs diff --git a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs new file mode 100644 index 00000000..17a26fe6 --- /dev/null +++ b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs @@ -0,0 +1,138 @@ +using OpenTelemetry.Instrumentation.AspNetCore; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using System.Diagnostics.Metrics; + +namespace TagzApp.Blazor; + +public static class OpenTelemetryExtensions +{ + private static readonly Action _ConfigureResource = r => r.AddService( + serviceName: "tags-app-blazor", + serviceVersion: typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown", + serviceInstanceId: Environment.MachineName); + + public static IServiceCollection AddOpenTelemetryObservability(this IServiceCollection services, IConfiguration configuration) + { + services.AddOpenTelemetry() + .ConfigureResource(_ConfigureResource) + .WithTracing(builder => + { + builder.AddSource() + .SetSampler(new AlwaysOnSampler()) + .AddHttpClientInstrumentation() + .AddAspNetCoreInstrumentation(); + + services.Configure(configuration.GetSection("AspNetCoreInstrumentation")); + + builder.SetupTracingExporter(configuration); + }) + .WithMetrics(builder => + { + builder + //.AddMeter() //TODO: Maybe set one up to demonstrate functionality + .AddRuntimeInstrumentation() + .AddHttpClientInstrumentation() + .AddAspNetCoreInstrumentation(); + + builder.SetupMetricsView(configuration); + builder.SetupMetricsExporter(configuration); + }); + + return services; + } + + public static void AddOpenTelemetryLogging(this ILoggingBuilder builder, IConfiguration configuration) + { + var logExporter = configuration.GetValue("UseLogExporter", defaultValue: "console")!.ToLowerInvariant(); + + builder.ClearProviders(); + + builder.AddOpenTelemetry(options => + { + var resourceBuilder = ResourceBuilder.CreateDefault(); + _ConfigureResource(resourceBuilder); + options.SetResourceBuilder(resourceBuilder); + + options.SetupLogsExporter(logExporter, configuration); + }); + } + + private static void SetupTracingExporter(this TracerProviderBuilder builder, IConfiguration configuration) + { + var tracingExporter = configuration.GetValue("UseTracingExporter", defaultValue: "console")!.ToLowerInvariant(); + + switch (tracingExporter) + { + case "otlp": + builder.AddOtlpExporter(otlpOptions => + { + otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); + }); + break; + + default: + builder.AddConsoleExporter(); + break; + } + } + + private static void SetupMetricsExporter(this MeterProviderBuilder builder, IConfiguration configuration) + { + var metricsExporter = configuration.GetValue("UseMetricsExporter", defaultValue: "console")!.ToLowerInvariant(); + + switch (metricsExporter) + { + case "otlp": + builder.AddOtlpExporter(otlpOptions => + { + otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); + }); + break; + + default: + builder.AddConsoleExporter(); + break; + } + } + + private static void SetupMetricsView(this MeterProviderBuilder builder, IConfiguration configuration) + { + var histogramAggregation = configuration.GetValue("HistogramAggregation", defaultValue: "explicit")!.ToLowerInvariant(); + + switch (histogramAggregation) + { + case "exponential": + builder.AddView(instrument => + { + return instrument.GetType().GetGenericTypeDefinition() == typeof(Histogram<>) + ? new Base2ExponentialBucketHistogramConfiguration() + : null; + }); + break; + + default: + // Explicit bounds histogram is default, nothing to do. + break; + } + } + + private static void SetupLogsExporter(this OpenTelemetryLoggerOptions options, string logExporter, IConfiguration configuration) + { + switch (logExporter) + { + case "otlp": + options.AddOtlpExporter(otlpOptions => + { + otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); + }); + break; + + default: + options.AddConsoleExporter(); + break; + } + } +} diff --git a/src/TagzApp.Blazor/Program.cs b/src/TagzApp.Blazor/Program.cs index 51dccc28..7365784d 100644 --- a/src/TagzApp.Blazor/Program.cs +++ b/src/TagzApp.Blazor/Program.cs @@ -53,6 +53,9 @@ private static async Task StartWebsite(string[] args) var configure = ConfigureTagzAppFactory.Create(builder.Configuration, null); + // Add OpenTelemetry for tracing and metrics. + builder.Services.AddOpenTelemetryObservability(builder.Configuration); + var appConfig = await ApplicationConfiguration.LoadFromConfiguration(configure); builder.Services.AddSingleton(appConfig); @@ -95,6 +98,9 @@ private static async Task StartWebsite(string[] args) options.KnownProxies.Clear(); }); + // Add OpenTelemetry for logging. + builder.Logging.AddOpenTelemetryLogging(builder.Configuration); + var app = builder.Build(); // Configure the HTTP request pipeline. diff --git a/src/TagzApp.Blazor/TagzApp.Blazor.csproj b/src/TagzApp.Blazor/TagzApp.Blazor.csproj index a6f69a96..30431978 100644 --- a/src/TagzApp.Blazor/TagzApp.Blazor.csproj +++ b/src/TagzApp.Blazor/TagzApp.Blazor.csproj @@ -62,6 +62,13 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + + + + + + + diff --git a/src/TagzApp.Blazor/appsettings.json b/src/TagzApp.Blazor/appsettings.json index ec04bc12..03b9715a 100644 --- a/src/TagzApp.Blazor/appsettings.json +++ b/src/TagzApp.Blazor/appsettings.json @@ -5,5 +5,15 @@ "Microsoft.AspNetCore": "Warning" } }, + "UseTracingExporter": "otlp", + "UseMetricsExporter": "otlp", + "UseLogExporter": "otlp", + "HistogramAggregation": "explicit", + "AspNetCoreInstrumentation": { + "RecordException": "true" + }, + "Otlp": { + "Endpoint": "http://localhost:4317" + }, "AllowedHosts": "*" } \ No newline at end of file From 05f58f151e4897a2623b6c8de62c8aef2a3ff54c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 22 Feb 2024 01:41:01 +0000 Subject: [PATCH 2/9] Applying formatting changes through GitHub Actions --- src/TagzApp.Blazor/OpenTelemetryExtensions.cs | 276 +++++++++--------- 1 file changed, 138 insertions(+), 138 deletions(-) diff --git a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs index 17a26fe6..f267c451 100644 --- a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs +++ b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs @@ -1,138 +1,138 @@ -using OpenTelemetry.Instrumentation.AspNetCore; -using OpenTelemetry.Logs; -using OpenTelemetry.Metrics; -using OpenTelemetry.Resources; -using OpenTelemetry.Trace; -using System.Diagnostics.Metrics; - -namespace TagzApp.Blazor; - -public static class OpenTelemetryExtensions -{ - private static readonly Action _ConfigureResource = r => r.AddService( - serviceName: "tags-app-blazor", - serviceVersion: typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown", - serviceInstanceId: Environment.MachineName); - - public static IServiceCollection AddOpenTelemetryObservability(this IServiceCollection services, IConfiguration configuration) - { - services.AddOpenTelemetry() - .ConfigureResource(_ConfigureResource) - .WithTracing(builder => - { - builder.AddSource() - .SetSampler(new AlwaysOnSampler()) - .AddHttpClientInstrumentation() - .AddAspNetCoreInstrumentation(); - - services.Configure(configuration.GetSection("AspNetCoreInstrumentation")); - - builder.SetupTracingExporter(configuration); - }) - .WithMetrics(builder => - { - builder - //.AddMeter() //TODO: Maybe set one up to demonstrate functionality - .AddRuntimeInstrumentation() - .AddHttpClientInstrumentation() - .AddAspNetCoreInstrumentation(); - - builder.SetupMetricsView(configuration); - builder.SetupMetricsExporter(configuration); - }); - - return services; - } - - public static void AddOpenTelemetryLogging(this ILoggingBuilder builder, IConfiguration configuration) - { - var logExporter = configuration.GetValue("UseLogExporter", defaultValue: "console")!.ToLowerInvariant(); - - builder.ClearProviders(); - - builder.AddOpenTelemetry(options => - { - var resourceBuilder = ResourceBuilder.CreateDefault(); - _ConfigureResource(resourceBuilder); - options.SetResourceBuilder(resourceBuilder); - - options.SetupLogsExporter(logExporter, configuration); - }); - } - - private static void SetupTracingExporter(this TracerProviderBuilder builder, IConfiguration configuration) - { - var tracingExporter = configuration.GetValue("UseTracingExporter", defaultValue: "console")!.ToLowerInvariant(); - - switch (tracingExporter) - { - case "otlp": - builder.AddOtlpExporter(otlpOptions => - { - otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); - }); - break; - - default: - builder.AddConsoleExporter(); - break; - } - } - - private static void SetupMetricsExporter(this MeterProviderBuilder builder, IConfiguration configuration) - { - var metricsExporter = configuration.GetValue("UseMetricsExporter", defaultValue: "console")!.ToLowerInvariant(); - - switch (metricsExporter) - { - case "otlp": - builder.AddOtlpExporter(otlpOptions => - { - otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); - }); - break; - - default: - builder.AddConsoleExporter(); - break; - } - } - - private static void SetupMetricsView(this MeterProviderBuilder builder, IConfiguration configuration) - { - var histogramAggregation = configuration.GetValue("HistogramAggregation", defaultValue: "explicit")!.ToLowerInvariant(); - - switch (histogramAggregation) - { - case "exponential": - builder.AddView(instrument => - { - return instrument.GetType().GetGenericTypeDefinition() == typeof(Histogram<>) - ? new Base2ExponentialBucketHistogramConfiguration() - : null; - }); - break; - - default: - // Explicit bounds histogram is default, nothing to do. - break; - } - } - - private static void SetupLogsExporter(this OpenTelemetryLoggerOptions options, string logExporter, IConfiguration configuration) - { - switch (logExporter) - { - case "otlp": - options.AddOtlpExporter(otlpOptions => - { - otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); - }); - break; - - default: - options.AddConsoleExporter(); - break; - } - } -} +using OpenTelemetry.Instrumentation.AspNetCore; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using System.Diagnostics.Metrics; + +namespace TagzApp.Blazor; + +public static class OpenTelemetryExtensions +{ + private static readonly Action _ConfigureResource = r => r.AddService( + serviceName: "tags-app-blazor", + serviceVersion: typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown", + serviceInstanceId: Environment.MachineName); + + public static IServiceCollection AddOpenTelemetryObservability(this IServiceCollection services, IConfiguration configuration) + { + services.AddOpenTelemetry() + .ConfigureResource(_ConfigureResource) + .WithTracing(builder => + { + builder.AddSource() + .SetSampler(new AlwaysOnSampler()) + .AddHttpClientInstrumentation() + .AddAspNetCoreInstrumentation(); + + services.Configure(configuration.GetSection("AspNetCoreInstrumentation")); + + builder.SetupTracingExporter(configuration); + }) + .WithMetrics(builder => + { + builder + //.AddMeter() //TODO: Maybe set one up to demonstrate functionality + .AddRuntimeInstrumentation() + .AddHttpClientInstrumentation() + .AddAspNetCoreInstrumentation(); + + builder.SetupMetricsView(configuration); + builder.SetupMetricsExporter(configuration); + }); + + return services; + } + + public static void AddOpenTelemetryLogging(this ILoggingBuilder builder, IConfiguration configuration) + { + var logExporter = configuration.GetValue("UseLogExporter", defaultValue: "console")!.ToLowerInvariant(); + + builder.ClearProviders(); + + builder.AddOpenTelemetry(options => + { + var resourceBuilder = ResourceBuilder.CreateDefault(); + _ConfigureResource(resourceBuilder); + options.SetResourceBuilder(resourceBuilder); + + options.SetupLogsExporter(logExporter, configuration); + }); + } + + private static void SetupTracingExporter(this TracerProviderBuilder builder, IConfiguration configuration) + { + var tracingExporter = configuration.GetValue("UseTracingExporter", defaultValue: "console")!.ToLowerInvariant(); + + switch (tracingExporter) + { + case "otlp": + builder.AddOtlpExporter(otlpOptions => + { + otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); + }); + break; + + default: + builder.AddConsoleExporter(); + break; + } + } + + private static void SetupMetricsExporter(this MeterProviderBuilder builder, IConfiguration configuration) + { + var metricsExporter = configuration.GetValue("UseMetricsExporter", defaultValue: "console")!.ToLowerInvariant(); + + switch (metricsExporter) + { + case "otlp": + builder.AddOtlpExporter(otlpOptions => + { + otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); + }); + break; + + default: + builder.AddConsoleExporter(); + break; + } + } + + private static void SetupMetricsView(this MeterProviderBuilder builder, IConfiguration configuration) + { + var histogramAggregation = configuration.GetValue("HistogramAggregation", defaultValue: "explicit")!.ToLowerInvariant(); + + switch (histogramAggregation) + { + case "exponential": + builder.AddView(instrument => + { + return instrument.GetType().GetGenericTypeDefinition() == typeof(Histogram<>) + ? new Base2ExponentialBucketHistogramConfiguration() + : null; + }); + break; + + default: + // Explicit bounds histogram is default, nothing to do. + break; + } + } + + private static void SetupLogsExporter(this OpenTelemetryLoggerOptions options, string logExporter, IConfiguration configuration) + { + switch (logExporter) + { + case "otlp": + options.AddOtlpExporter(otlpOptions => + { + otlpOptions.Endpoint = new Uri(configuration.GetValue("Otlp:Endpoint", defaultValue: " http://localhost:4317")!); + }); + break; + + default: + options.AddConsoleExporter(); + break; + } + } +} From 7f5f63a014fe2318abcf4915d43de82abb4a0563 Mon Sep 17 00:00:00 2001 From: napalm684 Date: Fri, 23 Feb 2024 21:49:59 -0500 Subject: [PATCH 3/9] Add observability setup for grafana to docker compose --- .gitignore | 1 + docker-compose.local.yml | 95 +- .../dashboards/19924_rev4.json | 1345 +++++++++++++++++ .../dashboards/19925_rev3.json | 941 ++++++++++++ observability-config/grafana-dashboard.yaml | 12 + observability-config/grafana-datasources.yaml | 48 + observability-config/otelcol-config.yaml | 49 + observability-config/prometheus.yaml | 15 + observability-config/tempo.yaml | 57 + 9 files changed, 2550 insertions(+), 13 deletions(-) create mode 100644 observability-config/dashboards/19924_rev4.json create mode 100644 observability-config/dashboards/19925_rev3.json create mode 100644 observability-config/grafana-dashboard.yaml create mode 100644 observability-config/grafana-datasources.yaml create mode 100644 observability-config/otelcol-config.yaml create mode 100644 observability-config/prometheus.yaml create mode 100644 observability-config/tempo.yaml diff --git a/.gitignore b/.gitignore index dfdaca31..a15cfbbc 100644 --- a/.gitignore +++ b/.gitignore @@ -416,3 +416,4 @@ FodyWeavers.xsd *.db-wal /src/TagzApp.Providers.Twitter/Models/SampleTweets.json.gz +/observability-config/tempo-data diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 2d2b2e02..40a4e389 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -1,14 +1,83 @@ -version: "3.7" +version: "3.8" services: - web: - container_name: tagzapp.web - hostname: tagzappweb - build: - dockerfile: ./src/TagzApp.Blazor/dockerfile - context: . - ports: - - "80" # runs on a random local port. Change to "8080:80" if you want to run on localhost:8080 - env_file: - # when .env values are used in this docker-compose file like 'hostname: $hostname' for example it is strongly recommended to inject them by referencing them like 'docker compose up --env-file .env' or 'docker compose up --env-file .env.local' - - .env - - .env.local + # TODO: This was not implemented yet... + # web: + # container_name: tagzapp.web + # hostname: tagzappweb + # build: + # dockerfile: ./src/TagzApp.Blazor/dockerfile + # context: . + # ports: + # - "80" # runs on a random local port. Change to "8080:80" if you want to run on localhost:8080 + # env_file: + # # when .env values are used in this docker-compose file like 'hostname: $hostname' for example it is strongly recommended to inject them by referencing them like 'docker compose up --env-file .env' or 'docker compose up --env-file .env.local' + # - .env + # - .env.local + # depends_on: + # - otelcol + + grafana: + image: grafana/grafana:10.3.1 + environment: + - "GF_AUTH_DISABLE_LOGIN_FORM=true" + - "GF_AUTH_ANONYMOUS_ENABLED=true" + - "GF_AUTH_ANONYMOUS_ORG_ROLE=Admin" + - "GF_FEATURE_TOGGLES_ENABLE=traceToMetrics" + volumes: + - ./observability-config/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - ./observability-config/grafana-dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml + - ./observability-config/dashboards:/var/lib/grafana/dashboards + ports: + - "3000:3000" + depends_on: + - tempo + - loki + + otelcol: + image: otel/opentelemetry-collector-contrib:0.95.0 + deploy: + resources: + limits: + memory: 125M + restart: unless-stopped + command: [ "--config=/etc/otelcol-config.yaml" ] + volumes: + - ./observability-config/otelcol-config.yaml:/etc/otelcol-config.yaml + ports: + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver + + loki: + image: grafana/loki:2.9.4 + ports: + - "3100" + command: -config.file=/etc/loki/local-config.yaml + depends_on: + - otelcol + + tempo: + image: grafana/tempo:2.3.1 + command: [ "-config.file=/etc/tempo.yaml" ] + volumes: + - ./observability-config/tempo.yaml:/etc/tempo.yaml + - ./observability-config/tempo-data:/tmp/tempo + ports: + - "14268" # jaeger ingest + - "3200" # tempo + - "4317" # otlp grpc + - "4318" # otlp http + - "9411" # zipkin + depends_on: + - prometheus + - loki + + prometheus: + image: prom/prometheus:v2.50.0 + command: + - --config.file=/etc/prometheus.yaml + - --web.enable-remote-write-receiver + - --enable-feature=exemplar-storage + volumes: + - ./observability-config/prometheus.yaml:/etc/prometheus.yaml + ports: + - "9090:9090" diff --git a/observability-config/dashboards/19924_rev4.json b/observability-config/dashboards/19924_rev4.json new file mode 100644 index 00000000..4d9b14ad --- /dev/null +++ b/observability-config/dashboards/19924_rev4.json @@ -0,0 +1,1345 @@ +{ + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "ASP.NET Core metrics from OpenTelemetry", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 19924, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-green", + "mode": "continuous-GrYlRd", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 1, + "text": "0 ms" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "p50" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval])) by (le))", + "legendFormat": "p50", + "range": true, + "refId": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p75", + "range": true, + "refId": "p75" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p95", + "range": true, + "refId": "p95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.98, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p98", + "range": true, + "refId": "p98" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "p99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.999, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p99.9", + "range": true, + "refId": "p99.9" + } + ], + "title": "Requests Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 1, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "All" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4XX" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5XX" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_response_status_code=~\"4..|5..\"}[$__rate_interval]) or vector(0)) / sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval]))", + "legendFormat": "All", + "range": true, + "refId": "All" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_response_status_code=~\"4..\"}[$__rate_interval]) or vector(0)) / sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "4XX", + "range": true, + "refId": "4XX" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_response_status_code=~\"5..\"}[$__rate_interval]) or vector(0)) / sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "5XX", + "range": true, + "refId": "5XX" + } + ], + "title": "Errors Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kestrel_active_connections{job=\"$job\", instance=\"$instance\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Current Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(http_server_active_requests{job=\"$job\", instance=\"$instance\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Current Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 58, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-red", + "mode": "fixed" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 59, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", error_type!=\"\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Unhandled Exceptions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 60, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (url_scheme) (\r\n max_over_time(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\"}[$__rate_interval])\r\n )", + "legendFormat": "{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests Secured", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 42, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (method_route) (\r\n label_replace(max_over_time(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\"}[$__rate_interval]), \"method_route\", \"http/$1\", \"network_protocol_version\", \"(.*)\")\r\n )", + "legendFormat": "{{protocol}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests HTTP Protocol", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Endpoint" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": false, + "title": "Test", + "url": "/d/NagEsjE4z/asp-net-core-endpoint-details?var-route=${__data.fields.http_route}&var-method=${__data.fields.http_request_method}&${__url_time_range}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_route" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_request_method" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hideTimeOverride": false, + "id": 51, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": " topk(10,\r\n sum by (http_route, http_request_method, method_route) (\r\n label_join(max_over_time(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route!=\"\"}[$__rate_interval]), \"method_route\", \" \", \"http_request_method\", \"http_route\")\r\n ))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{route}}", + "range": false, + "refId": "A" + } + ], + "title": "Top 10 Requested Endpoints", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "method": false, + "route": false + }, + "indexByName": { + "Time": 0, + "Value": 4, + "method": 2, + "method_route": 3, + "route": 1 + }, + "renameByName": { + "Value": "Requests", + "method": "", + "method_route": "Endpoint", + "route": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-YlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Endpoint" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/NagEsjE4z/asp-net-core-endpoint-details?var-route=${__data.fields.http_route}&var-method=${__data.fields.http_request_method}&${__url_time_range}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_route" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_request_method" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hideTimeOverride": false, + "id": 54, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": " topk(10,\r\n sum by (http_route, http_request_method, method_route) (\r\n label_join(max_over_time(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route!=\"\", error_type!=\"\"}[$__rate_interval]), \"method_route\", \" \", \"http_request_method\", \"http_route\")\r\n ))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{route}}", + "range": false, + "refId": "A" + } + ], + "title": "Top 10 Unhandled Exception Endpoints", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "method": false + }, + "indexByName": { + "Time": 0, + "Value": 4, + "method": 2, + "method_route": 3, + "route": 1 + }, + "renameByName": { + "Value": "Requests", + "method": "", + "method_route": "Endpoint", + "route": "" + } + } + } + ], + "type": "table" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "tags": [ + "dotnet", + "prometheus", + "aspnetcore" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_active_requests,job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(http_server_active_requests,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_active_requests{job=~\"$job\"},instance)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(http_server_active_requests{job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "1s", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "ASP.NET Core", + "uid": "KdDACDp4z", + "version": 1, + "weekStart": "" +} diff --git a/observability-config/dashboards/19925_rev3.json b/observability-config/dashboards/19925_rev3.json new file mode 100644 index 00000000..49e730b8 --- /dev/null +++ b/observability-config/dashboards/19925_rev3.json @@ -0,0 +1,941 @@ +{ + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "ASP.NET Core endpoint metrics from OpenTelemetry", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "dashboard", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": " ASP.NET Core", + "tooltip": "", + "type": "link", + "url": "/d/KdDACDp4z/asp-net-core-metrics" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-green", + "mode": "continuous-GrYlRd", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0 ms" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "p50" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m])) by (le))", + "legendFormat": "p50", + "range": true, + "refId": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m])) by (le))", + "hide": false, + "legendFormat": "p75", + "range": true, + "refId": "p75" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m])) by (le))", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "p90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m])) by (le))", + "hide": false, + "legendFormat": "p95", + "range": true, + "refId": "p95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.98, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m])) by (le))", + "hide": false, + "legendFormat": "p98", + "range": true, + "refId": "p98" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m])) by (le))", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "p99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.999, sum(rate(http_server_request_duration_seconds_bucket{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m])) by (le))", + "hide": false, + "legendFormat": "p99.9", + "range": true, + "refId": "p99.9" + } + ], + "title": "Requests Duration - $method $route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "All" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4XX" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5XX" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\", http_response_status_code=~\"4..|5..\"}[5m]) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m]))", + "legendFormat": "All", + "range": true, + "refId": "All" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\", http_response_status_code=~\"4..\"}[5m]) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m]))", + "hide": false, + "legendFormat": "4XX", + "range": true, + "refId": "4XX" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\", http_response_status_code=~\"5..\"}[5m]) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[5m]))", + "hide": false, + "legendFormat": "5XX", + "range": true, + "refId": "5XX" + } + ], + "title": "Errors Rate - $method $route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Requests" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-YlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Route" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/NagEsjE4z/asp-net-core-endpoint-details?var-route=${__data.fields.Route}&var-method=${__data.fields.Method}&${__url_time_range}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hideTimeOverride": false, + "id": 44, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (error_type) (\r\n max_over_time(http_server_request_duration_seconds_count{http_route=\"$route\", http_request_method=\"$method\", error_type!=\"\"}[$__rate_interval])\r\n)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{route}}", + "range": false, + "refId": "A" + } + ], + "title": "Unhandled Exceptions", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "method": false + }, + "indexByName": { + "Time": 0, + "Value": 2, + "error_type": 1 + }, + "renameByName": { + "Value": "Requests", + "error_type": "Exception", + "http_request_method": "Method", + "http_route": "Route" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 42, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (http_response_status_code) (\r\n max_over_time(http_server_request_duration_seconds_count{http_route=\"$route\", http_request_method=\"$method\"}[$__rate_interval])\r\n )", + "legendFormat": "Status {{http_response_status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests HTTP Status Code", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 48, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (url_scheme) (\r\n max_over_time(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[$__rate_interval])\r\n )", + "legendFormat": "{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests Secured", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 50, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (method_route) (\r\n label_replace(max_over_time(http_server_request_duration_seconds_count{job=\"$job\", instance=\"$instance\", http_route=\"$route\", http_request_method=\"$method\"}[$__rate_interval]), \"method_route\", \"http/$1\", \"network_protocol_version\", \"(.*)\")\r\n )", + "legendFormat": "{{protocol}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests HTTP Protocol", + "type": "stat" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "tags": [ + "dotnet", + "prometheus", + "aspnetcore" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_active_requests,job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(http_server_active_requests,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_active_requests{job=~\"$job\"},instance)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(http_server_active_requests{job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_request_duration_seconds_count,http_route)", + "description": "Route", + "hide": 0, + "includeAll": false, + "label": "Route", + "multi": false, + "name": "route", + "options": [], + "query": { + "query": "label_values(http_server_request_duration_seconds_count,http_route)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(http_server_request_duration_seconds_count{http_route=~\"$route\"},http_request_method)", + "hide": 0, + "includeAll": false, + "label": "Method", + "multi": false, + "name": "method", + "options": [], + "query": { + "query": "label_values(http_server_request_duration_seconds_count{http_route=~\"$route\"},http_request_method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "1s", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "ASP.NET Core Endpoint", + "uid": "NagEsjE4z", + "version": 1, + "weekStart": "", + "gnetId": 19925 +} diff --git a/observability-config/grafana-dashboard.yaml b/observability-config/grafana-dashboard.yaml new file mode 100644 index 00000000..fd66a479 --- /dev/null +++ b/observability-config/grafana-dashboard.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: "Dashboard provider" + orgId: 1 + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true diff --git a/observability-config/grafana-datasources.yaml b/observability-config/grafana-datasources.yaml new file mode 100644 index 00000000..487cc5c8 --- /dev/null +++ b/observability-config/grafana-datasources.yaml @@ -0,0 +1,48 @@ +apiVersion: 1 + +datasources: +- name: Loki + type: loki + access: proxy + url: http://loki:3100 + +- name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo:3200 + basicAuth: false + isDefault: true + version: 1 + editable: true + apiVersion: 1 + uid: tempo + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: prometheus + tracesToLogsV2: + datasourceUid: Loki + tags: [] + filterByTraceID: false + filterBySpanID: false + customQuery: false + tracesToMetrics: + datasourceUid: prometheus + tags: [] + filterByTraceID: false + filterBySpanID: false + customQuery: false + +- name: Prometheus + type: prometheus + uid: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: false + version: 1 + editable: false + jsonData: + httpMethod: GET diff --git a/observability-config/otelcol-config.yaml b/observability-config/otelcol-config.yaml new file mode 100644 index 00000000..5a84ab2f --- /dev/null +++ b/observability-config/otelcol-config.yaml @@ -0,0 +1,49 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +processors: + batch: + resource: + attributes: + - action: insert + key: loki.resource.labels + value: service.name + +exporters: + logging: + loglevel: debug + + otlp: + endpoint: tempo:4317 + tls: + insecure: true + + prometheus: + endpoint: "0.0.0.0:8889" + + loki: + endpoint: "http://loki:3100/loki/api/v1/push" + tls: + insecure: true + default_labels_enabled: + exporter: true + job: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlp] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [logging, prometheus] + logs: + receivers: [otlp] + processors: [batch, resource] + exporters: [logging, loki] diff --git a/observability-config/prometheus.yaml b/observability-config/prometheus.yaml new file mode 100644 index 00000000..cdca1475 --- /dev/null +++ b/observability-config/prometheus.yaml @@ -0,0 +1,15 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: [ 'localhost:9090' ] + - job_name: 'tempo' + static_configs: + - targets: [ 'tempo:3200' ] + - job_name: 'otelcol' + scrape_interval: 1s + static_configs: + - targets: [ 'otelcol:8889' ] \ No newline at end of file diff --git a/observability-config/tempo.yaml b/observability-config/tempo.yaml new file mode 100644 index 00000000..ba143b7f --- /dev/null +++ b/observability-config/tempo.yaml @@ -0,0 +1,57 @@ +stream_over_http_enabled: true +server: + http_listen_port: 3200 + log_level: info + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + trace_by_id: + duration_slo: 5s + +distributor: + receivers: # this configuration will listen on all ports and protocols that tempo is capable of. + jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can + protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver + thrift_http: # + grpc: # for a production deployment you should only enable the receivers you need! + thrift_binary: + thrift_compact: + zipkin: + otlp: + protocols: + http: + grpc: + opencensus: + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + +compactor: + compaction: + block_retention: 1h # overall Tempo trace retention. set for demo purposes + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: docker-compose + storage: + path: /tmp/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /tmp/tempo/wal # where to store the the wal locally + local: + path: /tmp/tempo/blocks + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] # enables metrics generator From 12f1db5c6d2463234df6dd54a5c05d895e67962f Mon Sep 17 00:00:00 2001 From: napalm684 Date: Thu, 14 Mar 2024 17:45:58 -0400 Subject: [PATCH 4/9] Cause Zs are always more badass --- src/TagzApp.Blazor/OpenTelemetryExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs index f267c451..5c0ce39b 100644 --- a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs +++ b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs @@ -10,7 +10,7 @@ namespace TagzApp.Blazor; public static class OpenTelemetryExtensions { private static readonly Action _ConfigureResource = r => r.AddService( - serviceName: "tags-app-blazor", + serviceName: "tagz-app-blazor", serviceVersion: typeof(Program).Assembly.GetName().Version?.ToString() ?? "unknown", serviceInstanceId: Environment.MachineName); From 4473d3176fb7515749062947548c09ee81895a84 Mon Sep 17 00:00:00 2001 From: napalm684 Date: Thu, 14 Mar 2024 19:18:33 -0400 Subject: [PATCH 5/9] Instrument Mastodon provider with message counter metrics --- src/TagzApp.Blazor/OpenTelemetryExtensions.cs | 14 ++++++++++++-- src/TagzApp.Blazor/TagzApp.Blazor.csproj | 1 + .../MastodonInstrumentation.cs | 17 +++++++++++++++++ .../MastodonProvider.cs | 5 ++++- src/TagzApp.Providers.Mastodon/StartMastodon.cs | 1 + 5 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs diff --git a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs index 5c0ce39b..67c21288 100644 --- a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs +++ b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs @@ -20,7 +20,12 @@ public static IServiceCollection AddOpenTelemetryObservability(this IServiceColl .ConfigureResource(_ConfigureResource) .WithTracing(builder => { - builder.AddSource() + var resourceBuilder = ResourceBuilder.CreateDefault(); + _ConfigureResource(resourceBuilder); + + builder + .SetResourceBuilder(resourceBuilder) + .AddSource() .SetSampler(new AlwaysOnSampler()) .AddHttpClientInstrumentation() .AddAspNetCoreInstrumentation(); @@ -31,8 +36,13 @@ public static IServiceCollection AddOpenTelemetryObservability(this IServiceColl }) .WithMetrics(builder => { + var resourceBuilder = ResourceBuilder.CreateDefault(); + _ConfigureResource(resourceBuilder); + builder - //.AddMeter() //TODO: Maybe set one up to demonstrate functionality + .SetResourceBuilder(resourceBuilder) + .AddMeter("mastodon-metrics") + .AddProcessInstrumentation() .AddRuntimeInstrumentation() .AddHttpClientInstrumentation() .AddAspNetCoreInstrumentation(); diff --git a/src/TagzApp.Blazor/TagzApp.Blazor.csproj b/src/TagzApp.Blazor/TagzApp.Blazor.csproj index 02df2d0c..98575297 100644 --- a/src/TagzApp.Blazor/TagzApp.Blazor.csproj +++ b/src/TagzApp.Blazor/TagzApp.Blazor.csproj @@ -69,6 +69,7 @@ + diff --git a/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs b/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs new file mode 100644 index 00000000..f07e99c3 --- /dev/null +++ b/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs @@ -0,0 +1,17 @@ +using System.Diagnostics.Metrics; + +namespace TagzApp.Providers.Mastodon; + +public class MastodonInstrumentation +{ + private Counter MessagesReceivedCounter { get; } + + public MastodonInstrumentation(IMeterFactory meterFactory) + { + var meter = meterFactory.Create("mastodon-metrics"); + + MessagesReceivedCounter = meter.CreateCounter("mastodon-messages-received", "message", "Counter for Mastodon Messages Received"); + } + + public void AddMessages(int count) => MessagesReceivedCounter.Add(count); +} diff --git a/src/TagzApp.Providers.Mastodon/MastodonProvider.cs b/src/TagzApp.Providers.Mastodon/MastodonProvider.cs index 49cfe0ad..094d1ed7 100644 --- a/src/TagzApp.Providers.Mastodon/MastodonProvider.cs +++ b/src/TagzApp.Providers.Mastodon/MastodonProvider.cs @@ -10,14 +10,16 @@ public class MastodonProvider : ISocialMediaProvider, IHasNewestId private readonly HttpClient _HttpClient; private readonly ILogger _Logger; + private readonly MastodonInstrumentation _Instrumentation; private SocialMediaStatus _Status = SocialMediaStatus.Unhealthy; private string _StatusMessage = "Not started"; public MastodonProvider(IHttpClientFactory httpClientFactory, ILogger logger, - MastodonConfiguration configuration) + MastodonConfiguration configuration, MastodonInstrumentation instrumentation) { _HttpClient = httpClientFactory.CreateClient(nameof(MastodonProvider)); _Logger = logger; + _Instrumentation = instrumentation; Enabled = configuration.Enabled; if (!string.IsNullOrWhiteSpace(configuration.Description)) @@ -81,6 +83,7 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi } NewestId = messages!.OrderByDescending(m => m.id).First().id; + _Instrumentation.AddMessages(messages?.Length ?? 0); var baseServerAddress = _HttpClient.BaseAddress?.Host.ToString(); diff --git a/src/TagzApp.Providers.Mastodon/StartMastodon.cs b/src/TagzApp.Providers.Mastodon/StartMastodon.cs index 3d7787b7..72d6ef3f 100644 --- a/src/TagzApp.Providers.Mastodon/StartMastodon.cs +++ b/src/TagzApp.Providers.Mastodon/StartMastodon.cs @@ -14,6 +14,7 @@ public async Task RegisterServices(IServiceCollection servic _MastodonConfiguration = await ConfigureTagzAppFactory.Current.GetConfigurationById(MastodonConfiguration.AppSettingsSection); + services.AddSingleton(); services.AddSingleton(_MastodonConfiguration ?? new()); services.AddHttpClient(_MastodonConfiguration ?? new MastodonConfiguration()); services.AddTransient(); From 8b536fa0c42e2944bb21f4a0e1e88e69c4dff2e3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 14 Mar 2024 23:19:23 +0000 Subject: [PATCH 6/9] Applying formatting changes through GitHub Actions --- src/TagzApp.Blazor/OpenTelemetryExtensions.cs | 2 +- .../MastodonInstrumentation.cs | 34 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs index 67c21288..64117b3f 100644 --- a/src/TagzApp.Blazor/OpenTelemetryExtensions.cs +++ b/src/TagzApp.Blazor/OpenTelemetryExtensions.cs @@ -41,7 +41,7 @@ public static IServiceCollection AddOpenTelemetryObservability(this IServiceColl builder .SetResourceBuilder(resourceBuilder) - .AddMeter("mastodon-metrics") + .AddMeter("mastodon-metrics") .AddProcessInstrumentation() .AddRuntimeInstrumentation() .AddHttpClientInstrumentation() diff --git a/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs b/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs index f07e99c3..38e6e703 100644 --- a/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs +++ b/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs @@ -1,17 +1,17 @@ -using System.Diagnostics.Metrics; - -namespace TagzApp.Providers.Mastodon; - -public class MastodonInstrumentation -{ - private Counter MessagesReceivedCounter { get; } - - public MastodonInstrumentation(IMeterFactory meterFactory) - { - var meter = meterFactory.Create("mastodon-metrics"); - - MessagesReceivedCounter = meter.CreateCounter("mastodon-messages-received", "message", "Counter for Mastodon Messages Received"); - } - - public void AddMessages(int count) => MessagesReceivedCounter.Add(count); -} +using System.Diagnostics.Metrics; + +namespace TagzApp.Providers.Mastodon; + +public class MastodonInstrumentation +{ + private Counter MessagesReceivedCounter { get; } + + public MastodonInstrumentation(IMeterFactory meterFactory) + { + var meter = meterFactory.Create("mastodon-metrics"); + + MessagesReceivedCounter = meter.CreateCounter("mastodon-messages-received", "message", "Counter for Mastodon Messages Received"); + } + + public void AddMessages(int count) => MessagesReceivedCounter.Add(count); +} From 20e43665476404e589fc601162883cd6a8aa73cd Mon Sep 17 00:00:00 2001 From: napalm684 Date: Thu, 14 Mar 2024 19:25:09 -0400 Subject: [PATCH 7/9] Unit test fix --- .../GivenHashtag/WhenFetchingMessages.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/TagzApp.UnitTest/Mastodon/GivenHashtag/WhenFetchingMessages.cs b/src/TagzApp.UnitTest/Mastodon/GivenHashtag/WhenFetchingMessages.cs index 57ad465b..8cba7c30 100644 --- a/src/TagzApp.UnitTest/Mastodon/GivenHashtag/WhenFetchingMessages.cs +++ b/src/TagzApp.UnitTest/Mastodon/GivenHashtag/WhenFetchingMessages.cs @@ -1,6 +1,7 @@ // Ignore Spelling: Sut using Microsoft.Extensions.Logging.Abstractions; +using System.Diagnostics.Metrics; using TagzApp.Providers.Mastodon; using TagzApp.Providers.Mastodon.Configuration; using IHttpClientFactory = System.Net.Http.IHttpClientFactory; @@ -16,6 +17,7 @@ public class WhenFetchingMessages private MastodonProvider _Sut; private IHttpClientFactory _HttpClientFactory; + private MastodonInstrumentation _Instrumentation; public WhenFetchingMessages() { @@ -25,8 +27,9 @@ public WhenFetchingMessages() }; _HttpClientFactory = new StubHttpClientFactory(client); + _Instrumentation = new MastodonInstrumentation(new StubMeterFactory()); - _Sut = new MastodonProvider(_HttpClientFactory, NullLogger.Instance, new MastodonConfiguration()); + _Sut = new MastodonProvider(_HttpClientFactory, NullLogger.Instance, new MastodonConfiguration(), _Instrumentation); } [Fact] @@ -64,4 +67,16 @@ public HttpClient CreateClient(string name) return _Client; } } + + internal class StubMeterFactory : IMeterFactory + { + public Meter Create(MeterOptions options) + { + return new Meter(options); + } + + public void Dispose() + { + } + } } From 7e92c835d0730749d097328b200c66a6f7321f58 Mon Sep 17 00:00:00 2001 From: napalm684 Date: Mon, 18 Mar 2024 20:37:36 -0400 Subject: [PATCH 8/9] Initial TagzApp Dashboard and Mastodon Instrumentation --- .gitignore | 2 + docker-compose.local.yml | 4 + .../TagzApp Dashboard-1710808195996.json | 278 ++++++++++++++++++ observability-config/tempo.yaml | 7 - .../MastodonInstrumentation.cs | 3 + .../MastodonProvider.cs | 8 + 6 files changed, 295 insertions(+), 7 deletions(-) create mode 100644 observability-config/dashboards/TagzApp Dashboard-1710808195996.json diff --git a/.gitignore b/.gitignore index 8468a6ee..54a4b442 100644 --- a/.gitignore +++ b/.gitignore @@ -423,3 +423,5 @@ FodyWeavers.xsd /src/TagzApp.Providers.Twitter/Models/SampleTweets.json.gz /observability-config/tempo-data +/observability-config/loki-data +/observability-config/prometheus-data diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 40a4e389..698d83e1 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -52,6 +52,8 @@ services: ports: - "3100" command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./observability-config/loki-data:/loki depends_on: - otelcol @@ -77,7 +79,9 @@ services: - --config.file=/etc/prometheus.yaml - --web.enable-remote-write-receiver - --enable-feature=exemplar-storage + - --enable-feature=promql-experimental-functions volumes: - ./observability-config/prometheus.yaml:/etc/prometheus.yaml + - ./observability-config/prometheus-data:/prometheus ports: - "9090:9090" diff --git a/observability-config/dashboards/TagzApp Dashboard-1710808195996.json b/observability-config/dashboards/TagzApp Dashboard-1710808195996.json new file mode 100644 index 00000000..865645c0 --- /dev/null +++ b/observability-config/dashboards/TagzApp Dashboard-1710808195996.json @@ -0,0 +1,278 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "TagzApp monitoring and metrics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 11, + "w": 2, + "x": 7, + "y": 0 + }, + "id": 3, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

TagzApp Dashboard

\n
\n\"TagzApp\n
", + "mode": "html" + }, + "pluginVersion": "10.3.1", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Top 5 posters across all providers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 9, + "y": 0 + }, + "id": 1, + "options": { + "displayMode": "gradient", + "maxVizHeight": 80, + "minVizHeight": 50, + "minVizWidth": 8, + "namePlacement": "top", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(5, sum by (author) (sort_desc(max_over_time(mastodon_messages_by_author_message_total[$__range]))))", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Top 5 Posters", + "transformations": [], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Posts received across all providers.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-purple", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{exported_instance=\"SHAWNSDESKTOP\", exported_job=\"tagz-app-blazor\", instance=\"otelcol:8889\", job=\"otelcol\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 10, + "x": 7, + "y": 11 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false, + "sortBy": "Name", + "sortDesc": true + }, + "timezone": [ + "utc" + ], + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "round(increase(mastodon_messages_received_message_total[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Posts Received", + "transformations": [], + "type": "timeseries" + } + ], + "refresh": "auto", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "TagzApp Dashboard", + "uid": "fb737849-bd1f-45e3-9b20-3d87d4cc3e20", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/observability-config/tempo.yaml b/observability-config/tempo.yaml index ba143b7f..e1492242 100644 --- a/observability-config/tempo.yaml +++ b/observability-config/tempo.yaml @@ -25,13 +25,6 @@ distributor: grpc: opencensus: -ingester: - max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally - -compactor: - compaction: - block_retention: 1h # overall Tempo trace retention. set for demo purposes - metrics_generator: registry: external_labels: diff --git a/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs b/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs index 38e6e703..b0fbaa03 100644 --- a/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs +++ b/src/TagzApp.Providers.Mastodon/MastodonInstrumentation.cs @@ -5,13 +5,16 @@ namespace TagzApp.Providers.Mastodon; public class MastodonInstrumentation { private Counter MessagesReceivedCounter { get; } + private Counter MessagesByAuthorCounter { get; } public MastodonInstrumentation(IMeterFactory meterFactory) { var meter = meterFactory.Create("mastodon-metrics"); MessagesReceivedCounter = meter.CreateCounter("mastodon-messages-received", "message", "Counter for Mastodon Messages Received"); + MessagesByAuthorCounter = meter.CreateCounter("mastodon-messages-by-author", "message", "Counter for Mastodon Messages Received by Author"); } public void AddMessages(int count) => MessagesReceivedCounter.Add(count); + public void AddMessages(string author) => MessagesByAuthorCounter.Add(1, new KeyValuePair("author", author)); } diff --git a/src/TagzApp.Providers.Mastodon/MastodonProvider.cs b/src/TagzApp.Providers.Mastodon/MastodonProvider.cs index 094d1ed7..37cac99a 100644 --- a/src/TagzApp.Providers.Mastodon/MastodonProvider.cs +++ b/src/TagzApp.Providers.Mastodon/MastodonProvider.cs @@ -83,7 +83,15 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi } NewestId = messages!.OrderByDescending(m => m.id).First().id; + _Instrumentation.AddMessages(messages?.Length ?? 0); + foreach(var username in messages?.Select(x => x.account?.username)!) + { + if (!string.IsNullOrEmpty(username)) + { + _Instrumentation.AddMessages(username); + } + } var baseServerAddress = _HttpClient.BaseAddress?.Host.ToString(); From 64a43023afdd0a01233ef95733cbd34f705764a5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 19 Mar 2024 00:38:34 +0000 Subject: [PATCH 9/9] Applying formatting changes through GitHub Actions --- src/TagzApp.Providers.Mastodon/MastodonProvider.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/TagzApp.Providers.Mastodon/MastodonProvider.cs b/src/TagzApp.Providers.Mastodon/MastodonProvider.cs index 37cac99a..570b6bbc 100644 --- a/src/TagzApp.Providers.Mastodon/MastodonProvider.cs +++ b/src/TagzApp.Providers.Mastodon/MastodonProvider.cs @@ -85,12 +85,12 @@ public async Task> GetContentForHashtag(Hashtag tag, DateTi NewestId = messages!.OrderByDescending(m => m.id).First().id; _Instrumentation.AddMessages(messages?.Length ?? 0); - foreach(var username in messages?.Select(x => x.account?.username)!) + foreach (var username in messages?.Select(x => x.account?.username)!) { - if (!string.IsNullOrEmpty(username)) - { - _Instrumentation.AddMessages(username); - } + if (!string.IsNullOrEmpty(username)) + { + _Instrumentation.AddMessages(username); + } } var baseServerAddress = _HttpClient.BaseAddress?.Host.ToString();