Alert when PG shared_buffers is smaller than open chunks

docs/mixin/alerts/alerts.yaml

@@ -342,3 +342,18 @@ groups:
       summary: High uncompressed data.
       description: "High uncompressed data in Promscale, on average, {{ $value }} uncompressed chunks per metric."
       runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscaleCompressionLow.md
+  - alert: PromscalePostgreSQLSharedBuffersLow
+    expr: |
+      (
+        ((promscale_sql_database_open_chunks_total_table_size + promscale_sql_database_open_chunks_total_index_size) 
+        / 
+        promscale_sql_database_shared_buffers_size)
+        > 1 )
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      summary: Promscale database performance will be affected.
+      description: "Currently open chunks are {{ $value | humanizePercentage }} of PostgreSQL shared_buffers. This will impact database performance."
+      runbook_url: https://github.com/timescale/promscale/blob/master/docs/runbooks/PromscalePostgreSQLSharedBuffersLow.md
+

docs/runbooks/PromscalePostgreSQLSharedBuffersLow.md

@@ -0,0 +1,21 @@
+# PromscalePostgreSQLSharedBuffersLow
+
+## Meaning
+
+Open chunks (the chunks where data is current written into) can't fit into
+PostgreSQL shared buffers. Total size is calculated by summing up all chunk
+relations and indexes sizes.
+
+## Impact
+
+Database performance will be affected, especially the ingest speed. The effect
+will be less if you are running PostgreSQL on fast local disk.
+
+## Mitigation
+
+Increase percentage of PostgreSQL memory allocated to shared buffers.
+If you have already allocated huge percentage of memory to shared buffers (eg 75%)
+you should consider increasing database memory.
+`shared_buffers` can be set through `postgresql.conf`.
+To make sure that your new setting is applied you can run: 
+`SELECT * FROM pg_settings WHERE name = 'shared_buffers';` 

pkg/pgmodel/metrics/database/metrics.go

@@ -578,6 +578,48 @@ var metrics = []metricQueryWrap{
 		),
 		query: `select count(*)::bigint from _prom_catalog.metric`,
 	},
+	{
+		metrics: gauges(
+			prometheus.GaugeOpts{
+				Namespace: util.PromNamespace,
+				Subsystem: "sql_database",
+				Name:      "shared_buffers_size",
+				Help:      "Size of shared_buffers in bytes",
+			},
+		),
+		query: `SELECT (setting::BIGINT*pg_size_bytes(unit))::BIGINT FROM pg_settings WHERE name = 'shared_buffers'`,
+	},
+	{
+		metrics: gauges(
+			prometheus.GaugeOpts{
+				Namespace: util.PromNamespace,
+				Subsystem: "sql_database",
+				Name:      "open_chunks_total_table_size",
+				Help:      "Total table size of currently open chunks in bytes",
+			},
+			prometheus.GaugeOpts{
+				Namespace: util.PromNamespace,
+				Subsystem: "sql_database",
+				Name:      "open_chunks_total_index_size",
+				Help:      "Total indexes size of currently open chunks in bytes",
+			},
+		),
+		query: `SELECT 
+					coalesce(sum(chunk_total_size)::BIGINT,0) as total_table_size, 
+					coalesce(sum(chunk_index_size)::BIGINT, 0) as total_index_size 
+				FROM (
+					SELECT DISTINCT ON (hypertable_id)
+					  pg_indexes_size(format('%I.%I', c.schema_name, c.table_name)) chunk_index_size,
+					  pg_total_relation_size(format('%I.%I', c.schema_name, c.table_name)) chunk_total_size
+					FROM _timescaledb_catalog.dimension_slice ds 
+					INNER JOIN _timescaledb_catalog.chunk_constraint cc on (cc.dimension_slice_id = ds.id)
+					INNER JOIN _timescaledb_catalog.chunk c on (c.id = cc.chunk_id) 
+					WHERE range_end > _timescaledb_internal.time_to_internal(now()- interval '30 minutes') 
+					and range_start < _timescaledb_internal.time_to_internal(now())
+					ORDER BY hypertable_id, range_end DESC
+					) AS info;`,
+		customPollConfig: updateAtMostEvery(6 * time.Minute),
+	},
 }
 
 // GetMetric returns the metric whose Description best matches the supplied name.