diagnostics: honor stopper quiescence when reporting diagnostics

Previously, we would block on the HTTP diagnostics request until the
preset timeout regardless of stopper quiescence. This change ensures
that the `ctx` passed to the diagnostics reporter and the HTTP client
is cancelled if the stopper is quiescing.

Resolves: #136739

Release note: None

pkg/ccl/serverccl/diagnosticsccl/reporter_test.go

@@ -41,6 +41,14 @@ import (
 
 const elemName = "somestring"
 
+var setTelemetryHttpTimeout = func(newVal time.Duration) func() {
+	prior := diagnostics.TelemetryHttpTimeout
+	diagnostics.TelemetryHttpTimeout = newVal
+	return func() {
+		diagnostics.TelemetryHttpTimeout = prior
+	}
+}
+
 func TestTenantReport(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
@@ -297,7 +305,7 @@ func TestTelemetry_SuccessfulTelemetryPing(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
-	diagnostics.TelemetryHttpTimeout = 3 * time.Second
+	defer setTelemetryHttpTimeout(3 * time.Second)()
 	rt := startReporterTest(t, base.TestIsSpecificToStorageLayerAndNeedsASystemTenant)
 	defer rt.Close()
 
@@ -366,6 +374,29 @@ func TestTelemetry_SuccessfulTelemetryPing(t *testing.T) {
 
 }
 
+// This test will block on `stopper.Stop` if the diagnostics reporter
+// doesn't honor stopper quiescence when making its HTTP request.
+func TestTelemetryQuiesce(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	defer setTelemetryHttpTimeout(100 * time.Second)()
+	rt := startReporterTest(t, base.TestIsSpecificToStorageLayerAndNeedsASystemTenant)
+	defer rt.Close()
+
+	ctx := context.Background()
+	setupCluster(t, rt.serverDB)
+
+	defer rt.diagServer.SetWaitSeconds(200)()
+	dr := rt.server.DiagnosticsReporter().(*diagnostics.Reporter)
+	stopper := rt.server.Stopper()
+
+	dr.PeriodicallyReportDiagnostics(ctx, stopper)
+	stopper.Stop(ctx)
+	<-stopper.IsStopped()
+}
+
 func TestUsageQuantization(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)

pkg/server/diagnostics/reporter.go

@@ -192,7 +192,11 @@ func (r *Reporter) PeriodicallyReportDiagnostics(ctx context.Context, stopper *s
 			// Consider something like rand.Float() > resetFreq/reportFreq here to sample
 			// stat reset periods for reporting.
 			if shouldReportDiagnostics(ctx, r.Settings) {
-				r.ReportDiagnostics(ctx)
+				func() {
+					ctx, cancel := stopper.WithCancelOnQuiesce(ctx)
+					defer cancel()
+					r.ReportDiagnostics(ctx)
+				}()
 			}
 
 			nextReport = nextReport.Add(reportFrequency.Get(&r.Settings.SV))