From f235f568816e9e1c15e8a030e76d25c13666310d Mon Sep 17 00:00:00 2001 From: Rahul Aggarwal Date: Thu, 25 May 2023 14:42:23 -0400 Subject: [PATCH] Related to: #103744 This change exposes a metric Ingest.Count on Pebble, which keeps track of the total number of ingestion operations performed. --- ingest.go | 4 ++++ metrics.go | 8 ++++++++ metrics_test.go | 3 +++ testdata/event_listener | 2 ++ testdata/ingest | 2 ++ testdata/metrics | 8 ++++++++ tool/testdata/db_lsm | 1 + 7 files changed, 28 insertions(+) diff --git a/ingest.go b/ingest.go index 617c8b268f..aaf9ddbbcb 100644 --- a/ingest.go +++ b/ingest.go @@ -816,6 +816,7 @@ func (d *DB) ingest( if err != nil { return IngestOperationStats{}, err } + if len(meta) == 0 { // All of the sstables to be ingested were empty. Nothing to do. return IngestOperationStats{}, nil @@ -1106,6 +1107,9 @@ func (d *DB) ingestApply( }); err != nil { return nil, err } + + d.mu.versions.metrics.Ingest.Count++ + d.updateReadStateLocked(d.opts.DebugCheck) d.updateTableStatsLocked(ve.NewFiles) // The ingestion may have pushed a level over the threshold for compaction, diff --git a/metrics.go b/metrics.go index cd90d03515..b030de9120 100644 --- a/metrics.go +++ b/metrics.go @@ -184,6 +184,11 @@ type Metrics struct { Duration time.Duration } + Ingest struct { + // The total number of ingestions + Count uint64 + } + Flush struct { // The total number of flushes. Count int64 @@ -504,6 +509,9 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) { notApplicable, notApplicable, redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses))) + w.Printf(" ingest %9d\n", + redact.Safe(m.Ingest.Count), + ) } func hitRate(hits, misses int64) float64 { diff --git a/metrics_test.go b/metrics_test.go index 9c5ea8c496..9a84656f26 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -59,6 +59,7 @@ func TestMetricsFormat(t *testing.T) { m.WAL.Size = 24 m.WAL.BytesIn = 25 m.WAL.BytesWritten = 26 + m.Ingest.Count = 27 for i := range m.Levels { l := &m.Levels[i] @@ -101,6 +102,7 @@ zmemtbl 14 13 B snaps 4 - 1024 (score == earliest seq num) titers 21 filter - - 47.4% (score == utility) + ingest 27 ` if s := "\n" + m.String(); expected != s { t.Fatalf("expected%s\nbut found%s", expected, s) @@ -296,6 +298,7 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 0 ` got := redact.Sprintf("%s", &Metrics{}).Redact() diff --git a/testdata/event_listener b/testdata/event_listener index 75619a0861..9d5fb953f6 100644 --- a/testdata/event_listener +++ b/testdata/event_listener @@ -282,6 +282,7 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 1 # Set up a scenario where the table to be ingested overlaps with the memtable. # The table is ingested as a flushable. The flush metrics refect the flushed @@ -377,6 +378,7 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 1 sstables ---- diff --git a/testdata/ingest b/testdata/ingest index bc457b22ca..3334e58254 100644 --- a/testdata/ingest +++ b/testdata/ingest @@ -52,6 +52,8 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 1 + iter seek-ge a diff --git a/testdata/metrics b/testdata/metrics index 539ad9d87d..22c015068a 100644 --- a/testdata/metrics +++ b/testdata/metrics @@ -38,6 +38,7 @@ zmemtbl 1 256 K snaps 0 - 0 (score == earliest seq num) titers 1 filter - - 0.0% (score == utility) + ingest 0 disk-usage ---- @@ -86,6 +87,7 @@ zmemtbl 2 512 K snaps 0 - 0 (score == earliest seq num) titers 2 filter - - 0.0% (score == utility) + ingest 0 disk-usage ---- @@ -119,6 +121,7 @@ zmemtbl 1 256 K snaps 0 - 0 (score == earliest seq num) titers 2 filter - - 0.0% (score == utility) + ingest 0 # Closing iter c will release one of the zombie sstables. The other # zombie sstable is still referenced by iter b. @@ -149,6 +152,7 @@ zmemtbl 1 256 K snaps 0 - 0 (score == earliest seq num) titers 1 filter - - 0.0% (score == utility) + ingest 0 disk-usage ---- @@ -182,6 +186,7 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 0 disk-usage ---- @@ -241,6 +246,7 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 0 additional-metrics ---- @@ -284,6 +290,7 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 0 additional-metrics ---- @@ -373,3 +380,4 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 0 diff --git a/tool/testdata/db_lsm b/tool/testdata/db_lsm index 5ccf3737f6..cca58ab774 100644 --- a/tool/testdata/db_lsm +++ b/tool/testdata/db_lsm @@ -31,3 +31,4 @@ zmemtbl 0 0 B snaps 0 - 0 (score == earliest seq num) titers 0 filter - - 0.0% (score == utility) + ingest 0