metrics, pprof: support reloading services with SIGHUP

Reload prometheus and pprof services, if the config is updated.

Closes #1868.

Signed-off-by: Andrey Butusov <[email protected]>

CHANGELOG.md

@@ -22,6 +22,7 @@ attribute, which is used for container domain name in NNS contracts (#2954)
 - New `peapod-to-fstree` tool providing peapod-to-fstree data migration (#3013)
 - Reloading node attributes with SIGHUP (#3005)
 - Reloading pool sizes (#3018)
+- Reloading pprof/metrics services with SIGHUP (#3016)
 
 ### Fixed
 - Do not search for tombstones when handling their expiration, use local indexes instead (#2929)

cmd/neofs-node/config.go

@@ -295,7 +295,8 @@ type internals struct {
 	closers []func()
 	// services that are useful for debug (e.g. when a regular closer does not
 	// close), must be close at the very end of application life cycle
-	veryLastClosers map[string]func()
+	veryLastClosersLock sync.RWMutex
+	veryLastClosers     map[string]func()
 
 	apiVersion   version.Version
 	healthStatus atomic.Int32
@@ -871,6 +872,9 @@ func (c *cfg) configWatcher(ctx context.Context) {
 		case <-ch:
 			c.log.Info("SIGHUP has been received, rereading configuration...")
 
+			oldMetrics := writeMetricConfig(c.cfgReader)
+			oldProfiler := writeProfilerConfig(c.cfgReader)
+
 			err := c.readConfig(c.cfgReader)
 			if err != nil {
 				c.log.Error("configuration reading", zap.Error(err))
@@ -881,6 +885,11 @@ func (c *cfg) configWatcher(ctx context.Context) {
 
 			c.reloadObjectPoolSizes()
 
+			// Prometheus and pprof
+
+			// nolint:contextcheck
+			c.reloadMetricsAndPprof(oldMetrics, oldProfiler)
+
 			// Logger
 
 			err = c.internals.logLevel.UnmarshalText([]byte(c.logger.level))
@@ -965,3 +974,30 @@ func writeSystemAttributes(c *cfg) error {
 
 	return nil
 }
+
+func (c *cfg) reloadMetricsAndPprof(oldMetrics metricConfig, oldProfiler profilerConfig) {
+	c.veryLastClosersLock.Lock()
+	defer c.veryLastClosersLock.Unlock()
+
+	// Metrics
+
+	if oldMetrics.isUpdated(c.cfgReader) {
+		if closer, ok := c.veryLastClosers[metricName]; ok {
+			closer()
+		}
+		delete(c.veryLastClosers, metricName)
+
+		preRunAndLog(c, metricName, initMetrics(c))
+	}
+
+	//Profiler
+
+	if oldProfiler.isUpdated(c.cfgReader) {
+		if closer, ok := c.veryLastClosers[profilerName]; ok {
+			closer()
+		}
+		delete(c.veryLastClosers, profilerName)
+
+		preRunAndLog(c, profilerName, initProfiler(c))
+	}
+}

cmd/neofs-node/main.go

@@ -184,9 +184,12 @@ func shutdown(c *cfg) {
 	for _, closer := range c.closers {
 		closer()
 	}
+
+	c.veryLastClosersLock.RLock()
 	for _, lastCloser := range c.veryLastClosers {
 		lastCloser()
 	}
+	c.veryLastClosersLock.RUnlock()
 
 	c.log.Debug("waiting for all processes to stop")
 

cmd/neofs-node/metrics.go

@@ -1,6 +1,9 @@
 package main
 
 import (
+	"time"
+
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
 	metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics"
 	httputil "github.com/nspcc-dev/neofs-node/pkg/util/http"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -25,3 +28,23 @@ func initMetrics(c *cfg) *httputil.Server {
 
 	return srv
 }
+
+type metricConfig struct {
+	enabled         bool
+	shutdownTimeout time.Duration
+	address         string
+}
+
+func writeMetricConfig(c *config.Config) metricConfig {
+	return metricConfig{
+		enabled:         metricsconfig.Enabled(c),
+		shutdownTimeout: metricsconfig.ShutdownTimeout(c),
+		address:         metricsconfig.Address(c),
+	}
+}
+
+func (m1 metricConfig) isUpdated(c *config.Config) bool {
+	return m1.enabled != metricsconfig.Enabled(c) ||
+		m1.shutdownTimeout != metricsconfig.ShutdownTimeout(c) ||
+		m1.address != metricsconfig.Address(c)
+}

cmd/neofs-node/pprof.go

@@ -1,6 +1,9 @@
 package main
 
 import (
+	"time"
+
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
 	profilerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/profiler"
 	httputil "github.com/nspcc-dev/neofs-node/pkg/util/http"
 )
@@ -24,3 +27,23 @@ func initProfiler(c *cfg) *httputil.Server {
 
 	return srv
 }
+
+type profilerConfig struct {
+	enabled         bool
+	shutdownTimeout time.Duration
+	address         string
+}
+
+func writeProfilerConfig(c *config.Config) profilerConfig {
+	return profilerConfig{
+		enabled:         profilerconfig.Enabled(c),
+		shutdownTimeout: profilerconfig.ShutdownTimeout(c),
+		address:         profilerconfig.Address(c),
+	}
+}
+
+func (m1 profilerConfig) isUpdated(c *config.Config) bool {
+	return m1.enabled != profilerconfig.Enabled(c) ||
+		m1.shutdownTimeout != profilerconfig.ShutdownTimeout(c) ||
+		m1.address != profilerconfig.Address(c)
+}