From 96b6bbc6282a5f8b0face4e83657f5f7f624c311 Mon Sep 17 00:00:00 2001 From: Michael Todorovic Date: Fri, 4 Oct 2024 15:55:13 +0200 Subject: [PATCH 1/4] feat: add support for pg_stat_user_indexes Signed-off-by: Michael Todorovic --- README.md | 3 + collector/pg_stat_user_indexes.go | 192 +++++++++++++++++++++++++ collector/pg_stat_user_indexes_test.go | 81 +++++++++++ 3 files changed, 276 insertions(+) create mode 100644 collector/pg_stat_user_indexes.go create mode 100644 collector/pg_stat_user_indexes_test.go diff --git a/README.md b/README.md index 4c464e210..53c85c303 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,9 @@ This will build the docker image as `prometheuscommunity/postgres_exporter:${bra * `[no-]collector.stat_statements` Enable the `stat_statements` collector (default: disabled). +* `[no-]collector.stat_user_indexes` + Enable the `stat_user_indexes` collector (default: disabled). + * `[no-]collector.stat_user_tables` Enable the `stat_user_tables` collector (default: enabled). diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go new file mode 100644 index 000000000..29a315db8 --- /dev/null +++ b/collector/pg_stat_user_indexes.go @@ -0,0 +1,192 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/blang/semver/v4" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + registerCollector(statUserIndexesSubsystem, defaultDisabled, NewPGStatUserIndexesCollector) +} + +type PGStatUserIndexesCollector struct { + log log.Logger +} + +const statUserIndexesSubsystem = "stat_user_indexes" + +func NewPGStatUserIndexesCollector(config collectorConfig) (Collector, error) { + return &PGStatUserIndexesCollector{log: config.logger}, nil +} + +var ( + statUserIndexesIdxScan = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_scan_total"), + "Number of scans for this index", + []string{"datname", "schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + + statUserIndexesLastIdxScan = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "last_idx_scan_time"), + "Last timestamp of scan for this index", + []string{"datname", "schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + + statUserIndexesIdxTupRead = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_tup_read"), + "Number of tuples read for this index", + []string{"datname", "schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + + statUserIndexesIdxTupFetch = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_tup_fetch"), + "Number of tuples fetch for this index", + []string{"datname", "schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) +) + +func statUserIndexesQuery(columns []string) string { + return fmt.Sprintf("SELECT %s FROM pg_stat_user_indexes;", strings.Join(columns, ",")) +} + +func (c *PGStatUserIndexesCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + + columns := []string{ + "current_database() datname", + "schemaname", + "relname", + "indexrelname", + "idx_scan", + "idx_tup_read", + "idx_tup_fetch", + } + + lastIdxScanAvail := instance.version.GTE(semver.MustParse("16.0.0")) + if lastIdxScanAvail { + columns = append(columns, "date_part('epoch', last_idx_scan) as last_idx_scan") + } + + rows, err := db.QueryContext(ctx, + statUserIndexesQuery(columns), + ) + + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var datname, schemaname, relname, indexrelname sql.NullString + var idxScan, lastIdxScan, idxTupRead, idxTupFetch sql.NullFloat64 + + r := []any{ + &datname, + &schemaname, + &relname, + &indexrelname, + &idxScan, + &idxTupRead, + &idxTupFetch, + } + + if lastIdxScanAvail { + r = append(r, &lastIdxScan) + } + + if err := rows.Scan(r...); err != nil { + return err + } + datnameLabel := "unknown" + if datname.Valid { + datnameLabel = datname.String + } + schemanameLabel := "unknown" + if schemaname.Valid { + schemanameLabel = schemaname.String + } + relnameLabel := "unknown" + if relname.Valid { + relnameLabel = relname.String + } + indexrelnameLabel := "unknown" + if indexrelname.Valid { + indexrelnameLabel = indexrelname.String + } + + if lastIdxScanAvail && !lastIdxScan.Valid { + level.Debug(c.log).Log("msg", "Skipping collecting metric because it has no active_time") + continue + } + + labels := []string{datnameLabel, schemanameLabel, relnameLabel, indexrelnameLabel} + + idxScanMetric := 0.0 + if idxScan.Valid { + idxScanMetric = idxScan.Float64 + } + ch <- prometheus.MustNewConstMetric( + statUserIndexesIdxScan, + prometheus.CounterValue, + idxScanMetric, + labels..., + ) + + idxTupReadMetric := 0.0 + if idxTupRead.Valid { + idxTupReadMetric = idxTupRead.Float64 + } + ch <- prometheus.MustNewConstMetric( + statUserIndexesIdxTupRead, + prometheus.CounterValue, + idxTupReadMetric, + labels..., + ) + + idxTupFetchMetric := 0.0 + if idxTupFetch.Valid { + idxTupFetchMetric = idxTupFetch.Float64 + } + ch <- prometheus.MustNewConstMetric( + statUserIndexesIdxTupFetch, + prometheus.CounterValue, + idxTupFetchMetric, + labels..., + ) + + if lastIdxScanAvail { + ch <- prometheus.MustNewConstMetric( + statUserIndexesLastIdxScan, + prometheus.CounterValue, + lastIdxScan.Float64, + labels..., + ) + } + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_stat_user_indexes_test.go b/collector/pg_stat_user_indexes_test.go new file mode 100644 index 000000000..c30914e44 --- /dev/null +++ b/collector/pg_stat_user_indexes_test.go @@ -0,0 +1,81 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgStatUserIndexesCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + + columns := []string{ + "datname", + "schemaname", + "relname", + "indexrelname", + "idx_scan", + "idx_tup_read", + "idx_tup_fetch", + } + rows := sqlmock.NewRows(columns). + AddRow("postgres", "public", "pgtest_accounts", "pgtest_accounts_pkey", "8", "9", "5") + + cols := []string{ + "current_database() datname", + "schemaname", + "relname", + "indexrelname", + "idx_scan", + "idx_tup_read", + "idx_tup_fetch", + } + + mock.ExpectQuery(sanitizeQuery(statUserIndexesQuery(cols))).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatUserIndexesCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatUserIndexesCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"datname": "postgres", "indexrelname": "pgtest_accounts_pkey", "schemaname": "public", "relname": "pgtest_accounts"}, value: 8, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"datname": "postgres", "indexrelname": "pgtest_accounts_pkey", "schemaname": "public", "relname": "pgtest_accounts"}, value: 9, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"datname": "postgres", "indexrelname": "pgtest_accounts_pkey", "schemaname": "public", "relname": "pgtest_accounts"}, value: 5, metricType: dto.MetricType_COUNTER}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 69223b331e3be10b2ea1172cc30da6dfbb0f10bc Mon Sep 17 00:00:00 2001 From: Michael Todorovic Date: Fri, 11 Oct 2024 11:23:57 +0200 Subject: [PATCH 2/4] feat: add pg_index_properties and pg_index_size_bytes Signed-off-by: Michael Todorovic --- collector/pg_index.go | 170 +++++++++++++++++++++++++++++++++++++ collector/pg_index_test.go | 84 ++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 collector/pg_index.go create mode 100644 collector/pg_index_test.go diff --git a/collector/pg_index.go b/collector/pg_index.go new file mode 100644 index 000000000..b233ddeb6 --- /dev/null +++ b/collector/pg_index.go @@ -0,0 +1,170 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + registerCollector(pgIndexSubsystem, defaultDisabled, NewPgIndexCollector) +} + +type PGIndexCollector struct { + log log.Logger +} + +const pgIndexSubsystem = "index" + +func NewPgIndexCollector(config collectorConfig) (Collector, error) { + return &PGIndexCollector{log: config.logger}, nil +} + +var ( + pgIndexProperties = prometheus.NewDesc( + prometheus.BuildFQName(namespace, pgIndexSubsystem, "properties"), + "Postgresql index properties", + []string{"datname", "schemaname", "relname", "indexrelname", "is_unique", "is_primary", "is_valid", "is_ready"}, + prometheus.Labels{}, + ) + pgIndexSize = prometheus.NewDesc( + prometheus.BuildFQName(namespace, pgIndexSubsystem, "size_bytes"), + "Postgresql index size in bytes", + []string{"datname", "schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) +) + +func pgIndexQuery(columns []string) string { + return fmt.Sprintf("SELECT %s FROM pg_catalog.pg_stat_user_indexes s JOIN pg_catalog.pg_index i ON s.indexrelid = i.indexrelid WHERE i.indislive='1';", strings.Join(columns, ",")) +} + +func boolToString(b bool) string { + if b { + return "1" + } + return "0" +} + +func (c *PGIndexCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + + columns := []string{ + "current_database() datname", + "s.schemaname", + "s.relname", + "s.indexrelname", + "i.indisunique", + "i.indisprimary", + "i.indisvalid", + "i.indisready", + "pg_relation_size(i.indexrelid) AS indexsize", + } + + rows, err := db.QueryContext(ctx, + pgIndexQuery(columns), + ) + + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var datname, schemaname, relname, indexrelname sql.NullString + var idxIsUnique, idxIsPrimary, idxIsValid, idxIsReady sql.NullBool + var idxSize sql.NullFloat64 + + r := []any{ + &datname, + &schemaname, + &relname, + &indexrelname, + &idxIsUnique, + &idxIsPrimary, + &idxIsValid, + &idxIsReady, + &idxSize, + } + + if err := rows.Scan(r...); err != nil { + return err + } + datnameLabel := "unknown" + if datname.Valid { + datnameLabel = datname.String + } + schemanameLabel := "unknown" + if schemaname.Valid { + schemanameLabel = schemaname.String + } + relnameLabel := "unknown" + if relname.Valid { + relnameLabel = relname.String + } + indexrelnameLabel := "unknown" + if indexrelname.Valid { + indexrelnameLabel = indexrelname.String + } + + indexIsUniqueLabel := "unknown" + if idxIsUnique.Valid { + indexIsUniqueLabel = boolToString(idxIsUnique.Bool) + } + + indexIsPrimaryLabel := "unknown" + if idxIsPrimary.Valid { + indexIsPrimaryLabel = boolToString(idxIsPrimary.Bool) + } + + indexIsValidLabel := "unknown" + if idxIsValid.Valid { + indexIsValidLabel = boolToString(idxIsValid.Bool) + } + + indexIsReadyLabel := "unknown" + if idxIsReady.Valid { + indexIsReadyLabel = boolToString(idxIsReady.Bool) + } + + indexSizeMetric := -1.0 + if idxSize.Valid { + indexSizeMetric = idxSize.Float64 + } + + propertiesLabels := []string{datnameLabel, schemanameLabel, relnameLabel, indexrelnameLabel, indexIsUniqueLabel, indexIsPrimaryLabel, indexIsValidLabel, indexIsReadyLabel} + ch <- prometheus.MustNewConstMetric( + pgIndexProperties, + prometheus.CounterValue, + 1, + propertiesLabels..., + ) + + sizeLabels := []string{datnameLabel, schemanameLabel, relnameLabel, indexrelnameLabel} + ch <- prometheus.MustNewConstMetric( + pgIndexSize, + prometheus.GaugeValue, + indexSizeMetric, + sizeLabels..., + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_index_test.go b/collector/pg_index_test.go new file mode 100644 index 000000000..6797c568e --- /dev/null +++ b/collector/pg_index_test.go @@ -0,0 +1,84 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgIndexesCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + + columns := []string{ + "datname", + "schemaname", + "relname", + "indexrelname", + "indisunique", + "indisprimary", + "indisvalid", + "indisready", + "indexsize", + } + rows := sqlmock.NewRows(columns). + AddRow("postgres", "public", "pgtest_accounts", "pgtest_accounts_pkey", "0", "1", "1", "1", "123456789") + + cols := []string{ + "current_database() datname", + "s.schemaname", + "s.relname", + "s.indexrelname", + "i.indisunique", + "i.indisprimary", + "i.indisvalid", + "i.indisready", + "pg_relation_size(i.indexrelid) AS indexsize", + } + + mock.ExpectQuery(sanitizeQuery(pgIndexQuery(cols))).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGIndexCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGIndexCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"datname": "postgres", "indexrelname": "pgtest_accounts_pkey", "schemaname": "public", "relname": "pgtest_accounts", "is_unique": "0", "is_primary": "1", "is_valid": "1", "is_ready": "1"}, value: 1, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"datname": "postgres", "indexrelname": "pgtest_accounts_pkey", "schemaname": "public", "relname": "pgtest_accounts"}, value: 123456789, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 9342607ee2075fc7786efe087808c8aa07c7ae0f Mon Sep 17 00:00:00 2001 From: Michael Todorovic Date: Fri, 11 Oct 2024 11:31:58 +0200 Subject: [PATCH 3/4] doc: missing index Signed-off-by: Michael Todorovic --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 53c85c303..10f9760ef 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,9 @@ This will build the docker image as `prometheuscommunity/postgres_exporter:${bra * `[no-]collector.database_wraparound` Enable the `database_wraparound` collector (default: disabled). +* `[no-]collector.index` + Enable the `index` collector (default: disabled). + * `[no-]collector.locks` Enable the `locks` collector (default: enabled). From f682fed50ccb7c940d0757c529f7b6f8ae98923f Mon Sep 17 00:00:00 2001 From: Michael Todorovic Date: Mon, 18 Nov 2024 10:54:21 +0100 Subject: [PATCH 4/4] fix: move to slog Signed-off-by: Michael Todorovic --- collector/pg_index.go | 5 +++-- collector/pg_stat_user_indexes.go | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/collector/pg_index.go b/collector/pg_index.go index b233ddeb6..0db4f39ae 100644 --- a/collector/pg_index.go +++ b/collector/pg_index.go @@ -18,7 +18,8 @@ import ( "fmt" "strings" - "github.com/go-kit/log" + "log/slog" + "github.com/prometheus/client_golang/prometheus" ) @@ -27,7 +28,7 @@ func init() { } type PGIndexCollector struct { - log log.Logger + log *slog.Logger } const pgIndexSubsystem = "index" diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go index 29a315db8..d0071603e 100644 --- a/collector/pg_stat_user_indexes.go +++ b/collector/pg_stat_user_indexes.go @@ -18,9 +18,9 @@ import ( "fmt" "strings" + "log/slog" + "github.com/blang/semver/v4" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" ) @@ -29,7 +29,7 @@ func init() { } type PGStatUserIndexesCollector struct { - log log.Logger + log *slog.Logger } const statUserIndexesSubsystem = "stat_user_indexes" @@ -137,7 +137,7 @@ func (c *PGStatUserIndexesCollector) Update(ctx context.Context, instance *insta } if lastIdxScanAvail && !lastIdxScan.Valid { - level.Debug(c.log).Log("msg", "Skipping collecting metric because it has no active_time") + c.log.Debug("Skipping collecting metric because it has no active_time") continue }