JIT optimization and reducing noise in telemetry.

src/WebJobs.Script/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs

@@ -3,8 +3,8 @@
 
 using System;
 using System.ComponentModel;
-using System.Diagnostics;
 using System.Diagnostics.Tracing;
+using System.Linq;
 using Azure.Monitor.OpenTelemetry.Exporter;
 using Azure.Monitor.OpenTelemetry.LiveMetrics;
 using Microsoft.Extensions.Configuration;
@@ -20,13 +20,29 @@ namespace Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry
 {
     internal static class OpenTelemetryConfigurationExtensions
     {
+        private static readonly string[] ExcludedRequestSubstrings =
+        [
+            "azure-webjobs-hosts",
+            "azureFunctionsRpcMessages"
+        ];
+
         internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder, HostBuilderContext context)
         {
-            string azMonConnectionString = GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration);
-            bool enableOtlp = false;
-            if (!string.IsNullOrEmpty(GetConfigurationValue(EnvironmentSettingNames.OtlpEndpoint, context.Configuration)))
+            // Initializing OTel services during placeholder mode as well to avoid the cost of JITting these objects during specialization.
+            bool isPlaceholderMode = SystemEnvironment.Instance.IsPlaceholderModeEnabled();
+            bool enableOtlp = isPlaceholderMode ||
+                             !string.IsNullOrEmpty(GetConfigurationValue(EnvironmentSettingNames.OtlpEndpoint, context.Configuration));
+
+            // Azure Monitor Exporter requires a connection string to be initialized. Use placeholder connection string accordingly.
+            string azMonConnectionString = isPlaceholderMode
+                ? "InstrumentationKey=00000000-0000-0000-0000-000000000000;"
+                : GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration);
+            bool enableAzureMonitor = !string.IsNullOrEmpty(azMonConnectionString);
+
+            if (!isPlaceholderMode && !enableOtlp && !enableAzureMonitor)
             {
-                enableOtlp = true;
+                // Skip OpenTelemetry configuration if OTLP and Azure Monitor are both disabled and not in placeholder mode.
+                return;
             }
 
             loggingBuilder
@@ -37,7 +53,7 @@ internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder,
                     {
                         o.AddOtlpExporter();
                     }
-                    if (!string.IsNullOrEmpty(azMonConnectionString))
+                    if (enableAzureMonitor)
                     {
                         o.AddAzureMonitorLogExporter(options => options.ConnectionString = azMonConnectionString);
                     }
@@ -65,17 +81,29 @@ internal static void ConfigureOpenTelemetry(this ILoggingBuilder loggingBuilder,
                     b.AddAspNetCoreInstrumentation();
                     b.AddHttpClientInstrumentation(o =>
                     {
-                        o.FilterHttpRequestMessage = _ =>
+                        o.FilterHttpRequestMessage = static (httpRequestMessage) =>
                         {
-                            Activity activity = Activity.Current?.Parent;
-                            return (activity == null || !activity.Source.Name.Equals("Azure.Core.Http")) ? true : false;
+                            if (httpRequestMessage.RequestUri?.AbsoluteUri is not { Length: > 0 } uri)
+                            {
+                                return false;
+                            }
+
+                            foreach (string substring in ExcludedRequestSubstrings)
+                            {
+                                if (uri.IndexOf(substring, StringComparison.Ordinal) >= 0)
+                                {
+                                    return false;
+                                }
+                            }
+
+                            return true;
                         };
                     });
                     if (enableOtlp)
                     {
                         b.AddOtlpExporter();
                     }
-                    if (!string.IsNullOrEmpty(azMonConnectionString))
+                    if (enableAzureMonitor)
                     {
                         b.AddAzureMonitorTraceExporter(options => options.ConnectionString = azMonConnectionString);
                         b.AddLiveMetrics(options => options.ConnectionString = azMonConnectionString);

src/WebJobs.Script/Diagnostics/OpenTelemetry/TelemetryMode.cs

@@ -7,6 +7,7 @@ internal enum TelemetryMode
     {
         None = 0, // or Default
         ApplicationInsights = 1,
-        OpenTelemetry = 2
+        OpenTelemetry = 2,
+        Placeholder = 3
     }
 }

src/WebJobs.Script/ScriptHostBuilderExtensions.cs

@@ -419,20 +419,9 @@ public static IHostBuilder SetAzureFunctionsConfigurationRoot(this IHostBuilder
 
         internal static void ConfigureTelemetry(this ILoggingBuilder loggingBuilder, HostBuilderContext context)
         {
-            TelemetryMode mode;
-            var telemetryModeSection = context.Configuration.GetSection(ConfigurationPath.Combine(ConfigurationSectionNames.JobHost, ConfigurationSectionNames.TelemetryMode));
-            if (telemetryModeSection.Exists() && Enum.TryParse(telemetryModeSection.Value, true, out TelemetryMode telemetryMode))
-            {
-                mode = telemetryMode;
-            }
-            else
-            {
-                // Default to ApplicationInsights.
-                mode = TelemetryMode.ApplicationInsights;
-            }
+            var telemetryMode = GetTelemetryMode(context);
 
-            // Use switch statement so any change to the enum results in a build error if we don't handle it.
-            switch (mode)
+            switch (telemetryMode)
             {
                 case TelemetryMode.ApplicationInsights:
                 case TelemetryMode.None:
@@ -441,16 +430,23 @@ internal static void ConfigureTelemetry(this ILoggingBuilder loggingBuilder, Hos
                 case TelemetryMode.OpenTelemetry:
                     loggingBuilder.ConfigureOpenTelemetry(context);
                     break;
+                case TelemetryMode.Placeholder:
+                    loggingBuilder.ConfigureApplicationInsights(context);
+                    loggingBuilder.ConfigureOpenTelemetry(context);
+                    break;
             }
         }
 
         internal static void ConfigureApplicationInsights(this ILoggingBuilder builder, HostBuilderContext context)
         {
             string appInsightsInstrumentationKey = GetConfigurationValue(EnvironmentSettingNames.AppInsightsInstrumentationKey, context.Configuration);
-            string appInsightsConnectionString = GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration);
 
-            // Initializing AppInsights services during placeholder mode as well to avoid the cost of JITting these objects during specialization
-            if (!string.IsNullOrEmpty(appInsightsInstrumentationKey) || !string.IsNullOrEmpty(appInsightsConnectionString) || SystemEnvironment.Instance.IsPlaceholderModeEnabled())
+            // Initializing AppInsights services during placeholder mode as well to avoid the cost of JITting these objects during specialization.
+            // Use placeholder connection.
+            string appInsightsConnectionString = SystemEnvironment.Instance.IsPlaceholderModeEnabled() ? "InstrumentationKey=00000000-0000-0000-0000-000000000000;"
+                : GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration);
+
+            if (!string.IsNullOrEmpty(appInsightsInstrumentationKey) || !string.IsNullOrEmpty(appInsightsConnectionString))
             {
                 string eventLogLevel = GetConfigurationValue(EnvironmentSettingNames.AppInsightsEventListenerLogLevel, context.Configuration);
                 string authString = GetConfigurationValue(EnvironmentSettingNames.AppInsightsAuthenticationString, context.Configuration);
@@ -616,5 +612,24 @@ private static string GetConfigurationValue(string key, IConfiguration configura
                 return null;
             }
         }
+
+        private static TelemetryMode GetTelemetryMode(HostBuilderContext context)
+        {
+            var telemetryModeSection = context.Configuration.GetSection(ConfigurationPath.Combine(ConfigurationSectionNames.JobHost, ConfigurationSectionNames.TelemetryMode));
+
+            if (telemetryModeSection.Exists() && Enum.TryParse(telemetryModeSection.Value, true, out TelemetryMode telemetryMode))
+            {
+                return telemetryMode;
+            }
+
+            if (SystemEnvironment.Instance.IsPlaceholderModeEnabled())
+            {
+                // Initialize AppInsights SDK and OTel services during placeholder mode to avoid JIT cost during specialization.
+                return TelemetryMode.Placeholder;
+            }
+
+            // Default to ApplicationInsights.
+            return TelemetryMode.ApplicationInsights;
+        }
     }
 }

src/WebJobs.Script/StorageProvider/StorageClientProvider.cs

@@ -76,6 +76,9 @@ protected bool IsConnectionStringPresent(IConfiguration configuration)
         private TClientOptions CreateClientOptions(IConfiguration configuration)
         {
             var clientOptions = (TClientOptions)_componentFactory.CreateClientOptions(typeof(TClientOptions), null, configuration);
+
+            // Disable distributed tracing by default to reduce the noise in the traces.
+            clientOptions.Diagnostics.IsDistributedTracingEnabled = false;
             return clientOptions;
         }
     }

test/WebJobs.Script.Tests/Diagnostics/OpenTelemetry/OpenTelemetryConfigurationExtensionsTests.cs

@@ -231,18 +231,76 @@ public void ResourceDetectorLocalDevelopment()
             Assert.Equal(4, resource.Attributes.Count());
         }
 
+        [Fact]
+        public void OpenTelemetryBuilder_InPlaceholderMode()
+        {
+            IHost host;
+            using (new TestScopedEnvironmentVariable(new Dictionary<string, string> { { EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1" } }))
+            {
+                host = new HostBuilder()
+                    .ConfigureLogging((context, builder) =>
+                    {
+                        builder.ConfigureOpenTelemetry(context);
+                    })
+                    .ConfigureServices(s =>
+                    {
+                        s.AddSingleton<IEnvironment>(SystemEnvironment.Instance);
+                    })
+                    .Build();
+            }
+
+            var a = host.Services.GetServices<object>();
+
+            var tracerProvider = host.Services.GetService<TracerProvider>();
+            Assert.NotNull(tracerProvider);
+
+            var loggerProvider = host.Services.GetService<ILoggerProvider>();
+            Assert.NotNull(loggerProvider);
+
+            var openTelemetryLoggerOptions = host.Services.GetService<IOptions<OpenTelemetryLoggerOptions>>();
+            Assert.NotNull(openTelemetryLoggerOptions);
+            Assert.True(openTelemetryLoggerOptions.Value.IncludeFormattedMessage);
+        }
+
+        [Fact]
+        public void OpenTelemetryBuilder_NotInPlaceholderMode()
+        {
+            IHost host;
+            using (new TestScopedEnvironmentVariable(new Dictionary<string, string> { { EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0" } }))
+            {
+                host = new HostBuilder()
+                    .ConfigureLogging((context, builder) =>
+                    {
+                        builder.ConfigureOpenTelemetry(context);
+                    })
+                    .ConfigureServices(s =>
+                    {
+                        s.AddSingleton<IEnvironment>(SystemEnvironment.Instance);
+                    })
+                    .Build();
+            }
+
+            var a = host.Services.GetServices<object>();
+
+            var tracerProvider = host.Services.GetService<TracerProvider>();
+            Assert.Null(tracerProvider);
+
+            var loggerProvider = host.Services.GetService<ILoggerProvider>();
+            Assert.Null(loggerProvider);
+        }
+
         // The OpenTelemetryEventListener is fine because it's a no-op if there are no otel events to listen to
         private bool HasOtelServices(IServiceCollection sc) => sc.Any(sd => sd.ServiceType != typeof(OpenTelemetryEventListener) && sd.ServiceType.FullName.Contains("OpenTelemetry"));
 
         private static IDisposable SetupDefaultEnvironmentVariables()
         {
             return new TestScopedEnvironmentVariable(new Dictionary<string, string>
-        {
-            { "WEBSITE_SITE_NAME", "appName" },
-            { "WEBSITE_RESOURCE_GROUP", "rg" },
-            { "WEBSITE_OWNER_NAME", "AAAAA-AAAAA-AAAAA-AAA+appName-EastUSwebspace" },
-            { "REGION_NAME", "EastUS" }
-        });
+            {
+                { "WEBSITE_SITE_NAME", "appName" },
+                { "WEBSITE_RESOURCE_GROUP", "rg" },
+                { "WEBSITE_OWNER_NAME", "AAAAA-AAAAA-AAAAA-AAA+appName-EastUSwebspace" },
+                { "REGION_NAME", "EastUS" }
+            });
         }
     }
 }