From fc241e5b8148973a705944be5ecdc275dbb4fd24 Mon Sep 17 00:00:00 2001 From: Rohit Ranjan <90008725+RohitRanjanMS@users.noreply.github.com> Date: Thu, 2 Jan 2025 09:41:04 -0800 Subject: [PATCH 1/3] JIT optimization and reducing noise in telemetry. --- .../OpenTelemetryConfigurationExtensions.cs | 30 +++++++----- .../OpenTelemetry/TelemetryMode.cs | 3 +- .../ScriptHostBuilderExtensions.cs | 47 ++++++++++++------- .../StorageProvider/StorageClientProvider.cs | 5 +- 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs index 34dc8d5eda..3ca4f15a0c 100644 --- a/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs +++ b/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs @@ -3,8 +3,8 @@ using System; using System.ComponentModel; -using System.Diagnostics; using System.Diagnostics.Tracing; +using System.Linq; using Azure.Monitor.OpenTelemetry.Exporter; using Azure.Monitor.OpenTelemetry.LiveMetrics; using Microsoft.Extensions.Configuration; @@ -20,14 +20,24 @@ namespace Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry { internal static class OpenTelemetryConfigurationExtensions { + private static readonly string[] ExcludedRequestSubstrings = + [ + "azure-webjobs-hosts", + "azureFunctionsRpcMessages" + ]; + internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder, HostBuilderContext context) { - string azMonConnectionString = GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration); - bool enableOtlp = false; - if (!string.IsNullOrEmpty(GetConfigurationValue(EnvironmentSettingNames.OtlpEndpoint, context.Configuration))) - { - enableOtlp = true; - } + bool isPlaceholderMode = SystemEnvironment.Instance.IsPlaceholderModeEnabled(); + + // Initializing AppInsights and OTel services during placeholder mode as well to avoid the cost of JITting these objects during specialization. + // Azure Monitor Exporter requires a connection string to be initialized. Use placeholder connection string. + string azMonConnectionString = isPlaceholderMode + ? "InstrumentationKey=00000000-0000-0000-0000-000000000000;" + : GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration); + + bool enableOtlp = isPlaceholderMode || + !string.IsNullOrEmpty(GetConfigurationValue(EnvironmentSettingNames.OtlpEndpoint, context.Configuration)); loggingBuilder .AddOpenTelemetry(o => @@ -65,11 +75,7 @@ internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder, b.AddAspNetCoreInstrumentation(); b.AddHttpClientInstrumentation(o => { - o.FilterHttpRequestMessage = _ => - { - Activity activity = Activity.Current?.Parent; - return (activity == null || !activity.Source.Name.Equals("Azure.Core.Http")) ? true : false; - }; + o.FilterHttpRequestMessage = (httpRequestMessage) => httpRequestMessage.RequestUri?.AbsoluteUri is string uri && !ExcludedRequestSubstrings.Any(uri.Contains); }); if (enableOtlp) { diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/TelemetryMode.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/TelemetryMode.cs index 91fbd12f78..2aea48bfd3 100644 --- a/src/WebJobs.Script/Diagnostics/OpenTelemetry/TelemetryMode.cs +++ b/src/WebJobs.Script/Diagnostics/OpenTelemetry/TelemetryMode.cs @@ -7,6 +7,7 @@ internal enum TelemetryMode { None = 0, // or Default ApplicationInsights = 1, - OpenTelemetry = 2 + OpenTelemetry = 2, + Placeholder = 3 } } diff --git a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs index fbc20359ea..f87d8e9cb4 100644 --- a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs +++ b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs @@ -419,20 +419,9 @@ public static IHostBuilder SetAzureFunctionsConfigurationRoot(this IHostBuilder internal static void ConfigureTelemetry(this ILoggingBuilder loggingBuilder, HostBuilderContext context) { - TelemetryMode mode; - var telemetryModeSection = context.Configuration.GetSection(ConfigurationPath.Combine(ConfigurationSectionNames.JobHost, ConfigurationSectionNames.TelemetryMode)); - if (telemetryModeSection.Exists() && Enum.TryParse(telemetryModeSection.Value, true, out TelemetryMode telemetryMode)) - { - mode = telemetryMode; - } - else - { - // Default to ApplicationInsights. - mode = TelemetryMode.ApplicationInsights; - } + var telemetryMode = GetTelemetryMode(context); - // Use switch statement so any change to the enum results in a build error if we don't handle it. - switch (mode) + switch (telemetryMode) { case TelemetryMode.ApplicationInsights: case TelemetryMode.None: @@ -441,16 +430,23 @@ internal static void ConfigureTelemetry(this ILoggingBuilder loggingBuilder, Hos case TelemetryMode.OpenTelemetry: loggingBuilder.ConfigureOpenTelemetry(context); break; + case TelemetryMode.Placeholder: + loggingBuilder.ConfigureApplicationInsights(context); + loggingBuilder.ConfigureOpenTelemetry(context); + break; } } internal static void ConfigureApplicationInsights(this ILoggingBuilder builder, HostBuilderContext context) { string appInsightsInstrumentationKey = GetConfigurationValue(EnvironmentSettingNames.AppInsightsInstrumentationKey, context.Configuration); - string appInsightsConnectionString = GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration); - // Initializing AppInsights services during placeholder mode as well to avoid the cost of JITting these objects during specialization - if (!string.IsNullOrEmpty(appInsightsInstrumentationKey) || !string.IsNullOrEmpty(appInsightsConnectionString) || SystemEnvironment.Instance.IsPlaceholderModeEnabled()) + // Initializing AppInsights services during placeholder mode as well to avoid the cost of JITting these objects during specialization. + // Use placeholder connection. + string appInsightsConnectionString = SystemEnvironment.Instance.IsPlaceholderModeEnabled() ? "InstrumentationKey=00000000-0000-0000-0000-000000000000;" + : GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration); + + if (!string.IsNullOrEmpty(appInsightsInstrumentationKey) || !string.IsNullOrEmpty(appInsightsConnectionString)) { string eventLogLevel = GetConfigurationValue(EnvironmentSettingNames.AppInsightsEventListenerLogLevel, context.Configuration); string authString = GetConfigurationValue(EnvironmentSettingNames.AppInsightsAuthenticationString, context.Configuration); @@ -616,5 +612,24 @@ private static string GetConfigurationValue(string key, IConfiguration configura return null; } } + + private static TelemetryMode GetTelemetryMode(HostBuilderContext context) + { + var telemetryModeSection = context.Configuration.GetSection(ConfigurationPath.Combine(ConfigurationSectionNames.JobHost, ConfigurationSectionNames.TelemetryMode)); + + if (telemetryModeSection.Exists() && Enum.TryParse(telemetryModeSection.Value, true, out TelemetryMode telemetryMode)) + { + return telemetryMode; + } + + if (SystemEnvironment.Instance.IsPlaceholderModeEnabled()) + { + // Initialize AppInsights SDK and OTel services during placeholder mode to avoid JIT cost during specialization. + return TelemetryMode.Placeholder; + } + + // Default to ApplicationInsights. + return TelemetryMode.ApplicationInsights; + } } } \ No newline at end of file diff --git a/src/WebJobs.Script/StorageProvider/StorageClientProvider.cs b/src/WebJobs.Script/StorageProvider/StorageClientProvider.cs index 264436d780..ccc6de27af 100644 --- a/src/WebJobs.Script/StorageProvider/StorageClientProvider.cs +++ b/src/WebJobs.Script/StorageProvider/StorageClientProvider.cs @@ -59,7 +59,7 @@ public virtual TClient Create(string name, IConfiguration configuration, TClient } protected virtual TClient CreateClient(IConfiguration configuration, TokenCredential tokenCredential, TClientOptions options) - { + { return (TClient)_componentFactory.CreateClient(typeof(TClient), configuration, tokenCredential, options); } @@ -76,6 +76,9 @@ protected bool IsConnectionStringPresent(IConfiguration configuration) private TClientOptions CreateClientOptions(IConfiguration configuration) { var clientOptions = (TClientOptions)_componentFactory.CreateClientOptions(typeof(TClientOptions), null, configuration); + + // Disable distributed tracing by default to reduce the noise in the traces. + clientOptions.Diagnostics.IsDistributedTracingEnabled = false; return clientOptions; } } From c4b86d4861cf1d6d5db3772318e72e8b363d7d69 Mon Sep 17 00:00:00 2001 From: Rohit Ranjan <90008725+RohitRanjanMS@users.noreply.github.com> Date: Tue, 7 Jan 2025 00:16:07 -0800 Subject: [PATCH 2/3] Removing space. --- src/WebJobs.Script/StorageProvider/StorageClientProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Script/StorageProvider/StorageClientProvider.cs b/src/WebJobs.Script/StorageProvider/StorageClientProvider.cs index ccc6de27af..e2bbea5647 100644 --- a/src/WebJobs.Script/StorageProvider/StorageClientProvider.cs +++ b/src/WebJobs.Script/StorageProvider/StorageClientProvider.cs @@ -59,7 +59,7 @@ public virtual TClient Create(string name, IConfiguration configuration, TClient } protected virtual TClient CreateClient(IConfiguration configuration, TokenCredential tokenCredential, TClientOptions options) - { + { return (TClient)_componentFactory.CreateClient(typeof(TClient), configuration, tokenCredential, options); } From 1ce3818150c74d18d2726665e52cbd834b0b41d6 Mon Sep 17 00:00:00 2001 From: Rohit Ranjan <90008725+RohitRanjanMS@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:54:39 -0800 Subject: [PATCH 3/3] Adding unit test. --- .../OpenTelemetryConfigurationExtensions.cs | 36 ++++++++-- ...enTelemetryConfigurationExtensionsTests.cs | 70 +++++++++++++++++-- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs index 3ca4f15a0c..07202dcdb1 100644 --- a/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs +++ b/src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs @@ -28,16 +28,22 @@ internal static class OpenTelemetryConfigurationExtensions internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder, HostBuilderContext context) { + // Initializing OTel services during placeholder mode as well to avoid the cost of JITting these objects during specialization. bool isPlaceholderMode = SystemEnvironment.Instance.IsPlaceholderModeEnabled(); + bool enableOtlp = isPlaceholderMode || + !string.IsNullOrEmpty(GetConfigurationValue(EnvironmentSettingNames.OtlpEndpoint, context.Configuration)); - // Initializing AppInsights and OTel services during placeholder mode as well to avoid the cost of JITting these objects during specialization. - // Azure Monitor Exporter requires a connection string to be initialized. Use placeholder connection string. + // Azure Monitor Exporter requires a connection string to be initialized. Use placeholder connection string accordingly. string azMonConnectionString = isPlaceholderMode ? "InstrumentationKey=00000000-0000-0000-0000-000000000000;" : GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration); + bool enableAzureMonitor = !string.IsNullOrEmpty(azMonConnectionString); - bool enableOtlp = isPlaceholderMode || - !string.IsNullOrEmpty(GetConfigurationValue(EnvironmentSettingNames.OtlpEndpoint, context.Configuration)); + if (!isPlaceholderMode && !enableOtlp && !enableAzureMonitor) + { + // Skip OpenTelemetry configuration if OTLP and Azure Monitor are both disabled and not in placeholder mode. + return; + } loggingBuilder .AddOpenTelemetry(o => @@ -47,7 +53,7 @@ internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder, { o.AddOtlpExporter(); } - if (!string.IsNullOrEmpty(azMonConnectionString)) + if (enableAzureMonitor) { o.AddAzureMonitorLogExporter(options => options.ConnectionString = azMonConnectionString); } @@ -75,13 +81,29 @@ internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder, b.AddAspNetCoreInstrumentation(); b.AddHttpClientInstrumentation(o => { - o.FilterHttpRequestMessage = (httpRequestMessage) => httpRequestMessage.RequestUri?.AbsoluteUri is string uri && !ExcludedRequestSubstrings.Any(uri.Contains); + o.FilterHttpRequestMessage = static (httpRequestMessage) => + { + if (httpRequestMessage.RequestUri?.AbsoluteUri is not { Length: > 0 } uri) + { + return false; + } + + foreach (string substring in ExcludedRequestSubstrings) + { + if (uri.IndexOf(substring, StringComparison.Ordinal) >= 0) + { + return false; + } + } + + return true; + }; }); if (enableOtlp) { b.AddOtlpExporter(); } - if (!string.IsNullOrEmpty(azMonConnectionString)) + if (enableAzureMonitor) { b.AddAzureMonitorTraceExporter(options => options.ConnectionString = azMonConnectionString); b.AddLiveMetrics(options => options.ConnectionString = azMonConnectionString); diff --git a/test/WebJobs.Script.Tests/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensionsTests.cs b/test/WebJobs.Script.Tests/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensionsTests.cs index 2e26d76544..0d223d9985 100644 --- a/test/WebJobs.Script.Tests/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensionsTests.cs @@ -231,18 +231,76 @@ public void ResourceDetectorLocalDevelopment() Assert.Equal(4, resource.Attributes.Count()); } + [Fact] + public void OpenTelemetryBuilder_InPlaceholderMode() + { + IHost host; + using (new TestScopedEnvironmentVariable(new Dictionary { { EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1" } })) + { + host = new HostBuilder() + .ConfigureLogging((context, builder) => + { + builder.ConfigureOpenTelemetry(context); + }) + .ConfigureServices(s => + { + s.AddSingleton(SystemEnvironment.Instance); + }) + .Build(); + } + + var a = host.Services.GetServices(); + + var tracerProvider = host.Services.GetService(); + Assert.NotNull(tracerProvider); + + var loggerProvider = host.Services.GetService(); + Assert.NotNull(loggerProvider); + + var openTelemetryLoggerOptions = host.Services.GetService>(); + Assert.NotNull(openTelemetryLoggerOptions); + Assert.True(openTelemetryLoggerOptions.Value.IncludeFormattedMessage); + } + + [Fact] + public void OpenTelemetryBuilder_NotInPlaceholderMode() + { + IHost host; + using (new TestScopedEnvironmentVariable(new Dictionary { { EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0" } })) + { + host = new HostBuilder() + .ConfigureLogging((context, builder) => + { + builder.ConfigureOpenTelemetry(context); + }) + .ConfigureServices(s => + { + s.AddSingleton(SystemEnvironment.Instance); + }) + .Build(); + } + + var a = host.Services.GetServices(); + + var tracerProvider = host.Services.GetService(); + Assert.Null(tracerProvider); + + var loggerProvider = host.Services.GetService(); + Assert.Null(loggerProvider); + } + // The OpenTelemetryEventListener is fine because it's a no-op if there are no otel events to listen to private bool HasOtelServices(IServiceCollection sc) => sc.Any(sd => sd.ServiceType != typeof(OpenTelemetryEventListener) && sd.ServiceType.FullName.Contains("OpenTelemetry")); private static IDisposable SetupDefaultEnvironmentVariables() { return new TestScopedEnvironmentVariable(new Dictionary - { - { "WEBSITE_SITE_NAME", "appName" }, - { "WEBSITE_RESOURCE_GROUP", "rg" }, - { "WEBSITE_OWNER_NAME", "AAAAA-AAAAA-AAAAA-AAA+appName-EastUSwebspace" }, - { "REGION_NAME", "EastUS" } - }); + { + { "WEBSITE_SITE_NAME", "appName" }, + { "WEBSITE_RESOURCE_GROUP", "rg" }, + { "WEBSITE_OWNER_NAME", "AAAAA-AAAAA-AAAAA-AAA+appName-EastUSwebspace" }, + { "REGION_NAME", "EastUS" } + }); } } } \ No newline at end of file