metrics.go

// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
// of this source code is governed by a BSD-style license that can be found in
// the LICENSE file.

package pebble

import (
	"fmt"
	"math"
	"time"

	"github.com/cockroachdb/pebble/internal/base"
	"github.com/cockroachdb/pebble/internal/cache"
	"github.com/cockroachdb/pebble/internal/humanize"
	"github.com/cockroachdb/pebble/internal/manual"
	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
	"github.com/cockroachdb/pebble/record"
	"github.com/cockroachdb/pebble/sstable"
	"github.com/cockroachdb/pebble/wal"
	"github.com/cockroachdb/redact"
	"github.com/prometheus/client_golang/prometheus"
)

// CacheMetrics holds metrics for the block and table cache.
type CacheMetrics = cache.Metrics

// FilterMetrics holds metrics for the filter policy
type FilterMetrics = sstable.FilterMetrics

// ThroughputMetric is a cumulative throughput metric. See the detailed
// comment in base.
type ThroughputMetric = base.ThroughputMetric

// SecondaryCacheMetrics holds metrics for the persistent secondary cache
// that caches commonly accessed blocks from blob storage on a local
// file system.
type SecondaryCacheMetrics = sharedcache.Metrics

// LevelMetrics holds per-level metrics such as the number of files and total
// size of the files, and compaction related metrics.
type LevelMetrics struct {
	// The number of sublevels within the level. The sublevel count corresponds
	// to the read amplification for the level. An empty level will have a
	// sublevel count of 0, implying no read amplification. Only L0 will have
	// a sublevel count other than 0 or 1.
	Sublevels int32
	// The total number of files in the level.
	NumFiles int64
	// The total number of virtual sstables in the level.
	NumVirtualFiles uint64
	// The total size in bytes of the files in the level.
	Size int64
	// The total size of the virtual sstables in the level.
	VirtualSize uint64
	// The level's compaction score. This is the compensatedScoreRatio in the
	// candidateLevelInfo.
	Score float64
	// The number of incoming bytes from other levels read during
	// compactions. This excludes bytes moved and bytes ingested. For L0 this is
	// the bytes written to the WAL.
	BytesIn uint64
	// The number of bytes ingested. The sibling metric for tables is
	// TablesIngested.
	BytesIngested uint64
	// The number of bytes moved into the level by a "move" compaction. The
	// sibling metric for tables is TablesMoved.
	BytesMoved uint64
	// The number of bytes read for compactions at the level. This includes bytes
	// read from other levels (BytesIn), as well as bytes read for the level.
	BytesRead uint64
	// The number of bytes written during compactions. The sibling
	// metric for tables is TablesCompacted. This metric may be summed
	// with BytesFlushed to compute the total bytes written for the level.
	BytesCompacted uint64
	// The number of bytes written during flushes. The sibling
	// metrics for tables is TablesFlushed. This metric is always
	// zero for all levels other than L0.
	BytesFlushed uint64
	// The number of sstables compacted to this level.
	TablesCompacted uint64
	// The number of sstables flushed to this level.
	TablesFlushed uint64
	// The number of sstables ingested into the level.
	TablesIngested uint64
	// The number of sstables moved to this level by a "move" compaction.
	TablesMoved uint64
	// The number of sstables deleted in a level by a delete-only compaction.
	TablesDeleted uint64
	// The number of sstables excised in a level by a delete-only compaction.
	TablesExcised uint64

	MultiLevel struct {
		// BytesInTop are the total bytes in a multilevel compaction coming from the top level.
		BytesInTop uint64

		// BytesIn, exclusively for multiLevel compactions.
		BytesIn uint64

		// BytesRead, exclusively for multilevel compactions.
		BytesRead uint64
	}

	// Additional contains misc additional metrics that are not always printed.
	Additional struct {
		// The sum of Properties.ValueBlocksSize for all the sstables in this
		// level. Printed by LevelMetrics.format iff there is at least one level
		// with a non-zero value.
		ValueBlocksSize uint64
		// Cumulative metrics about bytes written to data blocks and value blocks,
		// via compactions (except move compactions) or flushes. Not printed by
		// LevelMetrics.format, but are available to sophisticated clients.
		BytesWrittenDataBlocks  uint64
		BytesWrittenValueBlocks uint64
	}
}

// Add updates the counter metrics for the level.
func (m *LevelMetrics) Add(u *LevelMetrics) {
	m.NumFiles += u.NumFiles
	m.NumVirtualFiles += u.NumVirtualFiles
	m.VirtualSize += u.VirtualSize
	m.Size += u.Size
	m.BytesIn += u.BytesIn
	m.BytesIngested += u.BytesIngested
	m.BytesMoved += u.BytesMoved
	m.BytesRead += u.BytesRead
	m.BytesCompacted += u.BytesCompacted
	m.BytesFlushed += u.BytesFlushed
	m.TablesCompacted += u.TablesCompacted
	m.TablesFlushed += u.TablesFlushed
	m.TablesIngested += u.TablesIngested
	m.TablesMoved += u.TablesMoved
	m.MultiLevel.BytesInTop += u.MultiLevel.BytesInTop
	m.MultiLevel.BytesRead += u.MultiLevel.BytesRead
	m.MultiLevel.BytesIn += u.MultiLevel.BytesIn
	m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks
	m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks
	m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize
}

// WriteAmp computes the write amplification for compactions at this
// level. Computed as (BytesFlushed + BytesCompacted) / BytesIn.
func (m *LevelMetrics) WriteAmp() float64 {
	if m.BytesIn == 0 {
		return 0
	}
	return float64(m.BytesFlushed+m.BytesCompacted) / float64(m.BytesIn)
}

var categoryCompaction = sstable.RegisterCategory("pebble-compaction", sstable.NonLatencySensitiveQoSLevel)
var categoryIngest = sstable.RegisterCategory("pebble-ingest", sstable.LatencySensitiveQoSLevel)
var categoryGet = sstable.RegisterCategory("pebble-get", sstable.LatencySensitiveQoSLevel)

// Metrics holds metrics for various subsystems of the DB such as the Cache,
// Compactions, WAL, and per-Level metrics.
//
// TODO(peter): The testing of these metrics is relatively weak. There should
// be testing that performs various operations on a DB and verifies that the
// metrics reflect those operations.
type Metrics struct {
	BlockCache CacheMetrics

	Compact struct {
		// The total number of compactions, and per-compaction type counts.
		Count                 int64
		DefaultCount          int64
		DeleteOnlyCount       int64
		ElisionOnlyCount      int64
		CopyCount             int64
		MoveCount             int64
		ReadCount             int64
		TombstoneDensityCount int64
		RewriteCount          int64
		MultiLevelCount       int64
		CounterLevelCount     int64
		// An estimate of the number of bytes that need to be compacted for the LSM
		// to reach a stable state.
		EstimatedDebt uint64
		// Number of bytes present in sstables being written by in-progress
		// compactions. This value will be zero if there are no in-progress
		// compactions.
		InProgressBytes int64
		// Number of compactions that are in-progress.
		NumInProgress int64
		// Number of compactions that were cancelled.
		CancelledCount int64
		// CancelledBytes the number of bytes written by compactions that were
		// cancelled.
		CancelledBytes int64
		// MarkedFiles is a count of files that are marked for
		// compaction. Such files are compacted in a rewrite compaction
		// when no other compactions are picked.
		MarkedFiles int
		// Duration records the cumulative duration of all compactions since the
		// database was opened.
		Duration time.Duration
	}

	Ingest struct {
		// The total number of ingestions
		Count uint64
	}

	Flush struct {
		// The total number of flushes.
		Count           int64
		WriteThroughput ThroughputMetric
		// Number of flushes that are in-progress. In the current implementation
		// this will always be zero or one.
		NumInProgress int64
		// AsIngestCount is a monotonically increasing counter of flush operations
		// handling ingested tables.
		AsIngestCount uint64
		// AsIngestCount is a monotonically increasing counter of tables ingested as
		// flushables.
		AsIngestTableCount uint64
		// AsIngestBytes is a monotonically increasing counter of the bytes flushed
		// for flushables that originated as ingestion operations.
		AsIngestBytes uint64
	}

	Filter FilterMetrics

	Levels [numLevels]LevelMetrics

	MemTable struct {
		// The number of bytes allocated by memtables and large (flushable)
		// batches.
		Size uint64
		// The count of memtables.
		Count int64
		// The number of bytes present in zombie memtables which are no longer
		// referenced by the current DB state. An unbounded number of memtables
		// may be zombie if they're still in use by an iterator. One additional
		// memtable may be zombie if it's no longer in use and waiting to be
		// recycled.
		ZombieSize uint64
		// The count of zombie memtables.
		ZombieCount int64
	}

	Keys struct {
		// The approximate count of internal range key set keys in the database.
		RangeKeySetsCount uint64
		// The approximate count of internal tombstones (DEL, SINGLEDEL and
		// RANGEDEL key kinds) within the database.
		TombstoneCount uint64
		// A cumulative total number of missized DELSIZED keys encountered by
		// compactions since the database was opened.
		MissizedTombstonesCount uint64
	}

	Snapshots struct {
		// The number of currently open snapshots.
		Count int
		// The sequence number of the earliest, currently open snapshot.
		EarliestSeqNum base.SeqNum
		// A running tally of keys written to sstables during flushes or
		// compactions that would've been elided if it weren't for open
		// snapshots.
		PinnedKeys uint64
		// A running cumulative sum of the size of keys and values written to
		// sstables during flushes or compactions that would've been elided if
		// it weren't for open snapshots.
		PinnedSize uint64
	}

	Table struct {
		// The number of bytes present in obsolete tables which are no longer
		// referenced by the current DB state or any open iterators.
		ObsoleteSize uint64
		// The count of obsolete tables.
		ObsoleteCount int64
		// The number of bytes present in zombie tables which are no longer
		// referenced by the current DB state but are still in use by an iterator.
		ZombieSize uint64
		// The count of zombie tables.
		ZombieCount int64
		// The count of sstables backing virtual tables.
		BackingTableCount uint64
		// The sum of the sizes of the BackingTableCount sstables that are backing virtual tables.
		BackingTableSize uint64
		// The number of sstables that are compressed with an unknown compression
		// algorithm.
		CompressedCountUnknown int64
		// The number of sstables that are compressed with the default compression
		// algorithm, snappy.
		CompressedCountSnappy int64
		// The number of sstables that are compressed with zstd.
		CompressedCountZstd int64
		// The number of sstables that are uncompressed.
		CompressedCountNone int64

		// Local file sizes.
		Local struct {
			// LiveSize is the number of bytes in live tables.
			LiveSize uint64
			// ObsoleteSize is the number of bytes in obsolete tables.
			ObsoleteSize uint64
			// ZombieSize is the number of bytes in zombie tables.
			ZombieSize uint64
		}
	}

	TableCache CacheMetrics

	// Count of the number of open sstable iterators.
	TableIters int64
	// Uptime is the total time since this DB was opened.
	Uptime time.Duration

	WAL struct {
		// Number of live WAL files.
		Files int64
		// Number of obsolete WAL files.
		ObsoleteFiles int64
		// Physical size of the obsolete WAL files.
		ObsoletePhysicalSize uint64
		// Size of the live data in the WAL files. Note that with WAL file
		// recycling this is less than the actual on-disk size of the WAL files.
		Size uint64
		// Physical size of the WAL files on-disk. With WAL file recycling,
		// this is greater than the live data in WAL files.
		//
		// TODO(sumeer): it seems this does not include ObsoletePhysicalSize.
		// Should the comment be updated?
		PhysicalSize uint64
		// Number of logical bytes written to the WAL.
		BytesIn uint64
		// Number of bytes written to the WAL.
		BytesWritten uint64
		// Failover contains failover stats. Empty if failover is not enabled.
		Failover wal.FailoverStats
	}

	LogWriter struct {
		FsyncLatency prometheus.Histogram
		record.LogWriterMetrics
	}

	CategoryStats []sstable.CategoryStatsAggregate

	SecondaryCacheMetrics SecondaryCacheMetrics

	private struct {
		optionsFileSize  uint64
		manifestFileSize uint64
	}

	manualMemory manual.Metrics
}

var (
	// FsyncLatencyBuckets are prometheus histogram buckets suitable for a histogram
	// that records latencies for fsyncs.
	FsyncLatencyBuckets = append(
		prometheus.LinearBuckets(0.0, float64(time.Microsecond*100), 50),
		prometheus.ExponentialBucketsRange(float64(time.Millisecond*5), float64(10*time.Second), 50)...,
	)

	// SecondaryCacheIOBuckets exported to enable exporting from package pebble to
	// enable exporting metrics with below buckets in CRDB.
	SecondaryCacheIOBuckets = sharedcache.IOBuckets
	// SecondaryCacheChannelWriteBuckets exported to enable exporting from package
	// pebble to enable exporting metrics with below buckets in CRDB.
	SecondaryCacheChannelWriteBuckets = sharedcache.ChannelWriteBuckets
)

// DiskSpaceUsage returns the total disk space used by the database in bytes,
// including live and obsolete files. This only includes local files, i.e.,
// remote files (as known to objstorage.Provider) are not included.
func (m *Metrics) DiskSpaceUsage() uint64 {
	var usageBytes uint64
	usageBytes += m.WAL.PhysicalSize
	usageBytes += m.WAL.ObsoletePhysicalSize
	usageBytes += m.Table.Local.LiveSize
	usageBytes += m.Table.Local.ObsoleteSize
	usageBytes += m.Table.Local.ZombieSize
	usageBytes += m.private.optionsFileSize
	usageBytes += m.private.manifestFileSize
	// TODO(sumeer): InProgressBytes does not distinguish between local and
	// remote files. This causes a small error. Fix.
	usageBytes += uint64(m.Compact.InProgressBytes)
	return usageBytes
}

// NumVirtual is the number of virtual sstables in the latest version
// summed over every level in the lsm.
func (m *Metrics) NumVirtual() uint64 {
	var n uint64
	for _, level := range m.Levels {
		n += level.NumVirtualFiles
	}
	return n
}

// VirtualSize is the sum of the sizes of the virtual sstables in the
// latest version. BackingTableSize - VirtualSize gives an estimate for
// the space amplification caused by not compacting virtual sstables.
func (m *Metrics) VirtualSize() uint64 {
	var size uint64
	for _, level := range m.Levels {
		size += level.VirtualSize
	}
	return size
}

// ReadAmp returns the current read amplification of the database.
// It's computed as the number of sublevels in L0 + the number of non-empty
// levels below L0.
func (m *Metrics) ReadAmp() int {
	var ramp int32
	for _, l := range m.Levels {
		ramp += l.Sublevels
	}
	return int(ramp)
}

// Total returns the sum of the per-level metrics and WAL metrics.
func (m *Metrics) Total() LevelMetrics {
	var total LevelMetrics
	for level := 0; level < numLevels; level++ {
		l := &m.Levels[level]
		total.Add(l)
		total.Sublevels += l.Sublevels
	}
	// Compute total bytes-in as the bytes written to the WAL + bytes ingested.
	total.BytesIn = m.WAL.BytesWritten + total.BytesIngested
	// Add the total bytes-in to the total bytes-flushed. This is to account for
	// the bytes written to the log and bytes written externally and then
	// ingested.
	total.BytesFlushed += total.BytesIn
	return total
}

// String pretty-prints the metrics as below:
//
//	      |                             |       |       |   ingested   |     moved    |    written   |       |    amp
//	level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w
//	------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------
//	    0 |   101   102B     0B       0 | 103.0 |  104B |   112   104B |   113   106B |   221   217B |  107B |   1  2.1
//	    1 |   201   202B     0B       0 | 203.0 |  204B |   212   204B |   213   206B |   421   417B |  207B |   2  2.0
//	    2 |   301   302B     0B       0 | 303.0 |  304B |   312   304B |   313   306B |   621   617B |  307B |   3  2.0
//	    3 |   401   402B     0B       0 | 403.0 |  404B |   412   404B |   413   406B |   821   817B |  407B |   4  2.0
//	    4 |   501   502B     0B       0 | 503.0 |  504B |   512   504B |   513   506B |  1.0K  1017B |  507B |   5  2.0
//	    5 |   601   602B     0B       0 | 603.0 |  604B |   612   604B |   613   606B |  1.2K  1.2KB |  607B |   6  2.0
//	    6 |   701   702B     0B       0 |     - |  704B |   712   704B |   713   706B |  1.4K  1.4KB |  707B |   7  2.0
//	total |  2.8K  2.7KB     0B       0 |     - | 2.8KB |  2.9K  2.8KB |  2.9K  2.8KB |  5.7K  8.4KB | 2.8KB |  28  3.0
//	-------------------------------------------------------------------------------------------------------------------
//	WAL: 22 files (24B)  in: 25B  written: 26B (4% overhead)
//	Flushes: 8
//	Compactions: 5  estimated debt: 6B  in progress: 2 (7B)
//	default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  multi-level: 33
//	MemTables: 12 (11B)  zombie: 14 (13B)
//	Zombie tables: 16 (15B)
//	Backing tables: 0 (0B)
//	Block cache: 2 entries (1B)  hit rate: 42.9%
//	Table cache: 18 entries (17B)  hit rate: 48.7%
//	Secondary cache: 40 entries (40B)  hit rate: 49.9%
//	Snapshots: 4  earliest seq num: 1024
//	Table iters: 21
//	Filter utility: 47.4%
//	Ingestions: 27  as flushable: 36 (34B in 35 tables)
func (m *Metrics) String() string {
	return redact.StringWithoutMarkers(m)
}

var _ redact.SafeFormatter = &Metrics{}

// SafeFormat implements redact.SafeFormatter.
func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
	// NB: Pebble does not make any assumptions as to which Go primitive types
	// have been registered as safe with redact.RegisterSafeType and does not
	// register any types itself. Some of the calls to `redact.Safe`, etc are
	// superfluous in the context of CockroachDB, which registers all the Go
	// numeric types as safe.

	// TODO(jackson): There are a few places where we use redact.SafeValue
	// instead of redact.RedactableString. This is necessary because of a bug
	// whereby formatting a redact.RedactableString argument does not respect
	// width specifiers. When the issue is fixed, we can convert these to
	// RedactableStrings. https://github.com/cockroachdb/redact/issues/17

	multiExists := m.Compact.MultiLevelCount > 0
	appendIfMulti := func(line redact.SafeString) {
		if multiExists {
			w.SafeString(line)
		}
	}
	newline := func() {
		w.SafeString("\n")
	}

	w.SafeString("      |                             |       |       |   ingested   |     moved    |    written   |       |    amp")
	appendIfMulti("   |     multilevel")
	newline()
	w.SafeString("level | tables  size val-bl vtables | score |   in  | tables  size | tables  size | tables  size |  read |   r   w")
	appendIfMulti("  |    top   in  read")
	newline()
	w.SafeString("------+-----------------------------+-------+-------+--------------+--------------+--------------+-------+---------")
	appendIfMulti("-+------------------")
	newline()

	// formatRow prints out a row of the table.
	formatRow := func(m *LevelMetrics, score float64) {
		scoreStr := "-"
		if !math.IsNaN(score) {
			// Try to keep the string no longer than 5 characters.
			switch {
			case score < 99.995:
				scoreStr = fmt.Sprintf("%.2f", score)
			case score < 999.95:
				scoreStr = fmt.Sprintf("%.1f", score)
			default:
				scoreStr = fmt.Sprintf("%.0f", score)
			}
		}
		var wampStr string
		if wamp := m.WriteAmp(); wamp > 99.5 {
			wampStr = fmt.Sprintf("%.0f", wamp)
		} else {
			wampStr = fmt.Sprintf("%.1f", wamp)
		}

		w.Printf("| %5s %6s %6s %7s | %5s | %5s | %5s %6s | %5s %6s | %5s %6s | %5s | %3d %4s",
			humanize.Count.Int64(m.NumFiles),
			humanize.Bytes.Int64(m.Size),
			humanize.Bytes.Uint64(m.Additional.ValueBlocksSize),
			humanize.Count.Uint64(m.NumVirtualFiles),
			redact.Safe(scoreStr),
			humanize.Bytes.Uint64(m.BytesIn),
			humanize.Count.Uint64(m.TablesIngested),
			humanize.Bytes.Uint64(m.BytesIngested),
			humanize.Count.Uint64(m.TablesMoved),
			humanize.Bytes.Uint64(m.BytesMoved),
			humanize.Count.Uint64(m.TablesFlushed+m.TablesCompacted),
			humanize.Bytes.Uint64(m.BytesFlushed+m.BytesCompacted),
			humanize.Bytes.Uint64(m.BytesRead),
			redact.Safe(m.Sublevels),
			redact.Safe(wampStr))

		if multiExists {
			w.Printf(" | %5s %5s %5s",
				humanize.Bytes.Uint64(m.MultiLevel.BytesInTop),
				humanize.Bytes.Uint64(m.MultiLevel.BytesIn),
				humanize.Bytes.Uint64(m.MultiLevel.BytesRead))
		}
		newline()
	}

	var total LevelMetrics
	for level := 0; level < numLevels; level++ {
		l := &m.Levels[level]
		w.Printf("%5d ", redact.Safe(level))

		// Format the score.
		score := math.NaN()
		if level < numLevels-1 {
			score = l.Score
		}
		formatRow(l, score)
		total.Add(l)
		total.Sublevels += l.Sublevels
	}
	// Compute total bytes-in as the bytes written to the WAL + bytes ingested.
	total.BytesIn = m.WAL.BytesWritten + total.BytesIngested
	// Add the total bytes-in to the total bytes-flushed. This is to account for
	// the bytes written to the log and bytes written externally and then
	// ingested.
	total.BytesFlushed += total.BytesIn
	w.SafeString("total ")
	formatRow(&total, math.NaN())

	w.SafeString("-------------------------------------------------------------------------------------------------------------------")
	appendIfMulti("--------------------")
	newline()
	w.Printf("WAL: %d files (%s)  in: %s  written: %s (%.0f%% overhead)",
		redact.Safe(m.WAL.Files),
		humanize.Bytes.Uint64(m.WAL.Size),
		humanize.Bytes.Uint64(m.WAL.BytesIn),
		humanize.Bytes.Uint64(m.WAL.BytesWritten),
		redact.Safe(percent(int64(m.WAL.BytesWritten)-int64(m.WAL.BytesIn), int64(m.WAL.BytesIn))))
	failoverStats := m.WAL.Failover
	failoverStats.FailoverWriteAndSyncLatency = nil
	if failoverStats == (wal.FailoverStats{}) {
		w.Printf("\n")
	} else {
		w.Printf(" failover: (switches: %d, primary: %s, secondary: %s)\n", m.WAL.Failover.DirSwitchCount,
			m.WAL.Failover.PrimaryWriteDuration.String(), m.WAL.Failover.SecondaryWriteDuration.String())
	}

	w.Printf("Flushes: %d\n", redact.Safe(m.Flush.Count))

	w.Printf("Compactions: %d  estimated debt: %s  in progress: %d (%s)\n",
		redact.Safe(m.Compact.Count),
		humanize.Bytes.Uint64(m.Compact.EstimatedDebt),
		redact.Safe(m.Compact.NumInProgress),
		humanize.Bytes.Int64(m.Compact.InProgressBytes))

	w.Printf("             default: %d  delete: %d  elision: %d  move: %d  read: %d  tombstone-density: %d  rewrite: %d  copy: %d  multi-level: %d\n",
		redact.Safe(m.Compact.DefaultCount),
		redact.Safe(m.Compact.DeleteOnlyCount),
		redact.Safe(m.Compact.ElisionOnlyCount),
		redact.Safe(m.Compact.MoveCount),
		redact.Safe(m.Compact.ReadCount),
		redact.Safe(m.Compact.TombstoneDensityCount),
		redact.Safe(m.Compact.RewriteCount),
		redact.Safe(m.Compact.CopyCount),
		redact.Safe(m.Compact.MultiLevelCount))

	w.Printf("MemTables: %d (%s)  zombie: %d (%s)\n",
		redact.Safe(m.MemTable.Count),
		humanize.Bytes.Uint64(m.MemTable.Size),
		redact.Safe(m.MemTable.ZombieCount),
		humanize.Bytes.Uint64(m.MemTable.ZombieSize))

	w.Printf("Zombie tables: %d (%s, local: %s)\n",
		redact.Safe(m.Table.ZombieCount),
		humanize.Bytes.Uint64(m.Table.ZombieSize),
		humanize.Bytes.Uint64(m.Table.Local.ZombieSize))

	w.Printf("Backing tables: %d (%s)\n",
		redact.Safe(m.Table.BackingTableCount),
		humanize.Bytes.Uint64(m.Table.BackingTableSize))
	w.Printf("Virtual tables: %d (%s)\n",
		redact.Safe(m.NumVirtual()),
		humanize.Bytes.Uint64(m.VirtualSize()))
	w.Printf("Local tables size: %s\n", humanize.Bytes.Uint64(m.Table.Local.LiveSize))
	w.SafeString("Compression types:")
	if count := m.Table.CompressedCountSnappy; count > 0 {
		w.Printf(" snappy: %d", redact.Safe(count))
	}
	if count := m.Table.CompressedCountZstd; count > 0 {
		w.Printf(" zstd: %d", redact.Safe(count))
	}
	if count := m.Table.CompressedCountNone; count > 0 {
		w.Printf(" none: %d", redact.Safe(count))
	}
	if count := m.Table.CompressedCountUnknown; count > 0 {
		w.Printf(" unknown: %d", redact.Safe(count))
	}
	w.Print("\n")

	formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
			name,
			humanize.Count.Int64(m.Count),
			humanize.Bytes.Int64(m.Size),
			redact.Safe(hitRate(m.Hits, m.Misses)))
	}
	formatCacheMetrics(&m.BlockCache, "Block cache")
	formatCacheMetrics(&m.TableCache, "Table cache")

	formatSharedCacheMetrics := func(w redact.SafePrinter, m *SecondaryCacheMetrics, name redact.SafeString) {
		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
			name,
			humanize.Count.Int64(m.Count),
			humanize.Bytes.Int64(m.Size),
			redact.Safe(hitRate(m.ReadsWithFullHit, m.ReadsWithPartialHit+m.ReadsWithNoHit)))
	}
	formatSharedCacheMetrics(w, &m.SecondaryCacheMetrics, "Secondary cache")

	w.Printf("Snapshots: %d  earliest seq num: %d\n",
		redact.Safe(m.Snapshots.Count),
		redact.Safe(m.Snapshots.EarliestSeqNum))

	w.Printf("Table iters: %d\n", redact.Safe(m.TableIters))
	w.Printf("Filter utility: %.1f%%\n", redact.Safe(hitRate(m.Filter.Hits, m.Filter.Misses)))
	w.Printf("Ingestions: %d  as flushable: %d (%s in %d tables)\n",
		redact.Safe(m.Ingest.Count),
		redact.Safe(m.Flush.AsIngestCount),
		humanize.Bytes.Uint64(m.Flush.AsIngestBytes),
		redact.Safe(m.Flush.AsIngestTableCount))

	var inUseTotal uint64
	for i := range m.manualMemory {
		inUseTotal += m.manualMemory[i].InUseBytes
	}
	inUse := func(purpose manual.Purpose) uint64 {
		return m.manualMemory[purpose].InUseBytes
	}
	w.Printf("Cgo memory usage: %s  block cache: %s (data: %s, maps: %s, entries: %s)  memtables: %s\n",
		humanize.Bytes.Uint64(inUseTotal),
		humanize.Bytes.Uint64(inUse(manual.BlockCacheData)+inUse(manual.BlockCacheMap)+inUse(manual.BlockCacheEntry)),
		humanize.Bytes.Uint64(inUse(manual.BlockCacheData)),
		humanize.Bytes.Uint64(inUse(manual.BlockCacheMap)),
		humanize.Bytes.Uint64(inUse(manual.BlockCacheEntry)),
		humanize.Bytes.Uint64(inUse(manual.MemTable)),
	)
}

func hitRate(hits, misses int64) float64 {
	return percent(hits, hits+misses)
}

func percent(numerator, denominator int64) float64 {
	if denominator == 0 {
		return 0
	}
	return 100 * float64(numerator) / float64(denominator)
}

// StringForTests is identical to m.String() on 64-bit platforms. It is used to
// provide a platform-independent result for tests.
func (m *Metrics) StringForTests() string {
	mCopy := *m
	if math.MaxInt == math.MaxInt32 {
		// This is the difference in Sizeof(sstable.Reader{})) between 64 and 32 bit
		// platforms.
		const tableCacheSizeAdjustment = 212
		mCopy.TableCache.Size += mCopy.TableCache.Count * tableCacheSizeAdjustment
	}
	// Don't show cgo memory statistics as they can vary based on architecture,
	// invariants tag, etc.
	mCopy.manualMemory = manual.Metrics{}
	return redact.StringWithoutMarkers(&mCopy)
}