Merge cockroachdb#127290

127290: system: Add new table_metadata system table r=kyle-a-wong a=kyle-a-wong Adds a new system table, `table_metadata` for holding table level statatistics and metadata. This table is intended to be used as a cache and will be periodically updated via a new job. Epic: [CRDB-37558](https://cockroachlabs.atlassian.net/browse/CRDB-37558) Release note: None Co-authored-by: Kyle Wong <[email protected]>
msbutler · Aug 22, 2024 · 4344440 · 4344440
2 parents db79d36 + 9f0e04a
commit 4344440
Show file tree

Hide file tree

Showing 108 changed files with 1,082 additions and 408 deletions.
diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt
@@ -392,4 +392,4 @@ trace.snapshot.rate	duration	0s	if non-zero, interval at which background trace
 trace.span_registry.enabled	boolean	true	if set, ongoing traces can be seen at https://<ui>/#/debug/tracez	application
 trace.zipkin.collector	string		the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.	application
 ui.display_timezone	enumeration	etc/utc	the timezone used to format timestamps in the ui [etc/utc = 0, america/new_york = 1]	application
-version	version	1000024.2-upgrading-to-1000024.3-step-006	set the active cluster version in the format '<major>.<minor>'	application
+version	version	1000024.2-upgrading-to-1000024.3-step-008	set the active cluster version in the format '<major>.<minor>'	application
diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html
@@ -348,6 +348,6 @@
 <tr><td><div id="setting-trace-span-registry-enabled" class="anchored"><code>trace.span_registry.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if set, ongoing traces can be seen at https://&lt;ui&gt;/#/debug/tracez</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-trace-zipkin-collector" class="anchored"><code>trace.zipkin.collector</code></div></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as &lt;host&gt;:&lt;port&gt;. If no port is specified, 9411 will be used.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-display-timezone" class="anchored"><code>ui.display_timezone</code></div></td><td>enumeration</td><td><code>etc/utc</code></td><td>the timezone used to format timestamps in the ui [etc/utc = 0, america/new_york = 1]</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
-<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000024.2-upgrading-to-1000024.3-step-006</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000024.2-upgrading-to-1000024.3-step-008</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 </tbody>
 </table>
diff --git a/pkg/ccl/backupccl/system_schema.go b/pkg/ccl/backupccl/system_schema.go
@@ -847,6 +847,9 @@ var systemTableBackupConfiguration = map[string]systemBackupConfiguration{
 	systemschema.TransactionExecInsightsTable.GetName(): {
 		shouldIncludeInClusterBackup: optOutOfClusterBackup,
 	},
+	systemschema.TableMetadata.GetName(): {
+		shouldIncludeInClusterBackup: optOutOfClusterBackup,
+	},
 }
 
 func rekeySystemTable(

diff --git a/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant b/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant
@@ -357,12 +357,12 @@ txn_id  txn_fingerprint_id  query  implicit_txn  session_id  start_time  end_tim
 query ITTI
 SELECT range_id, start_pretty, end_pretty, lease_holder FROM crdb_internal.ranges
 ----
-69  /Tenant/10  /Tenant/11  1
+70  /Tenant/10  /Tenant/11  1
 
 query ITT
 SELECT range_id, start_pretty, end_pretty FROM crdb_internal.ranges_no_leases
 ----
-69  /Tenant/10  /Tenant/11
+70  /Tenant/10  /Tenant/11
 
 query IT
 SELECT zone_id, target FROM crdb_internal.zones ORDER BY 1

diff --git a/pkg/ccl/logictestccl/testdata/logic_test/multi_region b/pkg/ccl/logictestccl/testdata/logic_test/multi_region
@@ -1614,4 +1614,3 @@ statement ok
 ALTER DATABASE drop_region_126549 DROP REGION "us-east-1";
 
 subtest end
-
diff --git a/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error b/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error
@@ -463,7 +463,7 @@ skipif config multiregion-9node-3region-3azs-vec-off
 query I retry
 SELECT DISTINCT range_id FROM [SHOW RANGES FROM TABLE messages_rbr]
 ----
-74
+75
 
 # Update does not fail when accessing all rows in messages_rbr because lookup
 # join does not error out the lookup table in phase 1.

diff --git a/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_insert_fast_path b/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_insert_fast_path
@@ -200,7 +200,7 @@ query T rowsort
 SELECT message FROM [SHOW KV TRACE FOR SESSION]
 WHERE message LIKE '%batch%' AND message LIKE '%Scan%'
 ----
-r68: sending batch 4 Scan to (n1,s1):1
+r69: sending batch 4 Scan to (n1,s1):1
 
 # Regression test for #115377.
 statement ok

diff --git a/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior b/pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_query_behavior
@@ -264,7 +264,7 @@ ap-southeast-2  23
 query TT
 SELECT start_key, end_key FROM [SHOW RANGE FROM TABLE regional_by_row_table FOR ROW ('ap-southeast-2', 1)]
 ----
-<before:/Table/66>  …
+<before:/Table/67>  …
 
 query TIIII
 SELECT crdb_region, pk, pk2, a, b FROM regional_by_row_table
@@ -402,7 +402,7 @@ SELECT start_key, end_key, replicas, lease_holder FROM [SHOW RANGES FROM INDEX r
 ORDER BY 1
 ----
 start_key           end_key               replicas  lease_holder
-<before:/Table/66>  …/"\x80"/0            {1}       1
+<before:/Table/67>  …/"\x80"/0            {1}       1
 …/"\x80"/0          …/"\xc0"/0            {4}       4
 …/"\xc0"/0          <after:/Table/110/5>  {7}       7
 

diff --git a/pkg/ccl/multiregionccl/multiregion_system_table_test.go b/pkg/ccl/multiregionccl/multiregion_system_table_test.go
@@ -300,6 +300,7 @@ func TestMrSystemDatabase(t *testing.T) {
 				{"TABLE system.public.statement_diagnostics_requests"},
 				{"TABLE system.public.statement_execution_insights"},
 				{"TABLE system.public.statement_statistics"},
+				{"TABLE system.public.table_metadata"},
 				{"TABLE system.public.task_payloads"},
 				{"TABLE system.public.tenant_settings"},
 				{"TABLE system.public.tenant_tasks"},
@@ -341,6 +342,7 @@ func TestMrSystemDatabase(t *testing.T) {
 				{"TABLE system.public.statement_diagnostics_requests"},
 				{"TABLE system.public.statement_execution_insights"},
 				{"TABLE system.public.statement_statistics"},
+				{"TABLE system.public.table_metadata"},
 				{"TABLE system.public.task_payloads"},
 				{"TABLE system.public.tenant_settings"},
 				{"TABLE system.public.tenant_tasks"},

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/basic b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/basic
@@ -74,6 +74,7 @@ upsert /Table/6{3-4}                       database system (host)
 upsert /Table/6{4-5}                       database system (host)
 upsert /Table/6{5-6}                       database system (host)
 upsert /Table/6{6-7}                       database system (host)
+upsert /Table/6{7-8}                       database system (host)
 
 exec-sql
 CREATE DATABASE db;
@@ -130,6 +131,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Table/10{6-7}                             num_replicas=7 num_voters=5
 /Table/10{7-8}                             num_replicas=7
 /Table/11{2-3}                             num_replicas=7
@@ -247,6 +249,8 @@ delete /Table/6{5-6}
 upsert /Table/6{5-6}                       ttl_seconds=100 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
 delete /Table/6{6-7}
 upsert /Table/6{6-7}                       ttl_seconds=100 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
+delete /Table/6{7-8}
+upsert /Table/6{7-8}                       ttl_seconds=100 ignore_strict_gc=true num_replicas=5 rangefeed_enabled=true
 
 state offset=5 limit=42
 ----

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/indexes b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/indexes
@@ -41,6 +41,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Table/10{6-7}                             range default
 
 exec-sql
@@ -82,6 +83,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Table/106{-/2}                            num_replicas=7
 /Table/106/{2-3}                           num_replicas=7 num_voters=5
 /Table/10{6/3-7}                           num_replicas=7
@@ -125,6 +127,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Table/106{-/2}                            ttl_seconds=3600 num_replicas=7
 /Table/106/{2-3}                           ttl_seconds=25 num_replicas=7 num_voters=5
 /Table/10{6/3-7}                           ttl_seconds=3600 num_replicas=7
@@ -158,6 +161,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Table/106{-/2}                            ttl_seconds=3600 num_replicas=9
 /Table/106/{2-3}                           ttl_seconds=25 num_replicas=9 num_voters=5
 /Table/10{6/3-7}                           ttl_seconds=3600 num_replicas=9
@@ -205,3 +209,4 @@ state offset=46
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/basic b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/basic
@@ -42,6 +42,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Tenant/10{-\x00}                          database system (tenant)
 /Tenant/11{-\x00}                          database system (tenant)
 /Tenant/12{-\x00}                          database system (tenant)
@@ -109,6 +110,7 @@ upsert /Tenant/10/Table/6{3-4}             database system (tenant)
 upsert /Tenant/10/Table/6{4-5}             database system (tenant)
 upsert /Tenant/10/Table/6{5-6}             database system (tenant)
 upsert /Tenant/10/Table/6{6-7}             database system (tenant)
+upsert /Tenant/10/Table/6{7-8}             database system (tenant)
 
 state offset=47
 ----
@@ -133,6 +135,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Tenant/10{-/Table/4}                      database system (tenant)
 /Tenant/10/Table/{4-5}                     database system (tenant)
 /Tenant/10/Table/{5-6}                     database system (tenant)
@@ -189,6 +192,7 @@ state offset=47
 /Tenant/10/Table/6{4-5}                    database system (tenant)
 /Tenant/10/Table/6{5-6}                    database system (tenant)
 /Tenant/10/Table/6{6-7}                    database system (tenant)
+/Tenant/10/Table/6{7-8}                    database system (tenant)
 /Tenant/11{-\x00}                          database system (tenant)
 /Tenant/12{-\x00}                          database system (tenant)
 
@@ -213,6 +217,7 @@ upsert /Tenant/10/Table/11{3-4}            rangefeed_enabled=true
 state offset=81
 ----
 ...
+/Tenant/10/Table/2{0-1}                    database system (tenant)
 /Tenant/10/Table/2{1-2}                    database system (tenant)
 /Tenant/10/Table/2{3-4}                    database system (tenant)
 /Tenant/10/Table/2{4-5}                    database system (tenant)
@@ -255,6 +260,7 @@ state offset=81
 /Tenant/10/Table/6{4-5}                    database system (tenant)
 /Tenant/10/Table/6{5-6}                    database system (tenant)
 /Tenant/10/Table/6{6-7}                    database system (tenant)
+/Tenant/10/Table/6{7-8}                    database system (tenant)
 /Tenant/10/Table/10{6-7}                   rangefeed_enabled=true
 /Tenant/10/Table/10{7-8}                   rangefeed_enabled=true
 /Tenant/10/Table/11{2-3}                   rangefeed_enabled=true

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/protectedts b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/protectedts
@@ -39,6 +39,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Tenant/10{-\x00}                          database system (tenant)
 /Tenant/11{-\x00}                          database system (tenant)
 
@@ -125,6 +126,7 @@ upsert /Tenant/10/Table/6{3-4}             database system (tenant)
 upsert /Tenant/10/Table/6{4-5}             database system (tenant)
 upsert /Tenant/10/Table/6{5-6}             database system (tenant)
 upsert /Tenant/10/Table/6{6-7}             database system (tenant)
+upsert /Tenant/10/Table/6{7-8}             database system (tenant)
 
 exec-sql tenant=10
 CREATE DATABASE db;

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/range_tenants b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/range_tenants
@@ -58,6 +58,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Tenant/10{-\x00}                          database system (tenant)
 /Tenant/11{-\x00}                          ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
 /Tenant/12{-\x00}                          ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
@@ -94,6 +95,7 @@ state offset=47
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Tenant/10{-/Table/4}                      database system (tenant)
 /Tenant/10/Table/{4-5}                     database system (tenant)
 /Tenant/10/Table/{5-6}                     database system (tenant)
@@ -150,6 +152,7 @@ state offset=47
 /Tenant/10/Table/6{4-5}                    database system (tenant)
 /Tenant/10/Table/6{5-6}                    database system (tenant)
 /Tenant/10/Table/6{6-7}                    database system (tenant)
+/Tenant/10/Table/6{7-8}                    database system (tenant)
 /Tenant/11{-\x00}                          ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
 /Tenant/12{-\x00}                          ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
 
@@ -187,6 +190,7 @@ mutations discard tenant=11
 state offset=81
 ----
 ...
+/Tenant/10/Table/2{0-1}                    database system (tenant)
 /Tenant/10/Table/2{1-2}                    database system (tenant)
 /Tenant/10/Table/2{3-4}                    database system (tenant)
 /Tenant/10/Table/2{4-5}                    database system (tenant)
@@ -229,6 +233,7 @@ state offset=81
 /Tenant/10/Table/6{4-5}                    database system (tenant)
 /Tenant/10/Table/6{5-6}                    database system (tenant)
 /Tenant/10/Table/6{6-7}                    database system (tenant)
+/Tenant/10/Table/6{7-8}                    database system (tenant)
 /Tenant/10/Table/10{6-7}                   rangefeed_enabled=true
 /Tenant/10/Table/10{7-8}                   rangefeed_enabled=true
 /Tenant/11{-/Table/4}                      ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
@@ -287,6 +292,7 @@ state offset=81
 /Tenant/11/Table/6{4-5}                    ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
 /Tenant/11/Table/6{5-6}                    ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
 /Tenant/11/Table/6{6-7}                    ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
+/Tenant/11/Table/6{7-8}                    ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
 /Tenant/12{-\x00}                          ttl_seconds=18000 ignore_strict_gc=true rangefeed_enabled=true
 
 query-sql tenant=11

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/tenant_end_key_split b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/multitenant/tenant_end_key_split
@@ -25,6 +25,7 @@ state offset=59
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Tenant/11{-\x00}                          database system (tenant)
 /Tenant/12{-\x00}                          database system (tenant)
 
@@ -53,6 +54,7 @@ state offset=60 limit=3
 state offset=103
 ----
 ...
+/Tenant/11/Table/4{5-6}                    ignore_strict_gc=true rangefeed_enabled=true exclude_data_from_backup=true
 /Tenant/11/Table/4{6-7}                    database system (tenant)
 /Tenant/11/Table/4{7-8}                    ignore_strict_gc=true rangefeed_enabled=true exclude_data_from_backup=true
 /Tenant/11/Table/4{8-9}                    database system (tenant)
@@ -73,6 +75,7 @@ state offset=103
 /Tenant/11/Table/6{4-5}                    database system (tenant)
 /Tenant/11/Table/6{5-6}                    database system (tenant)
 /Tenant/11/Table/6{6-7}                    database system (tenant)
+/Tenant/11/Table/6{7-8}                    database system (tenant)
 /Tenant/12{-\x00}                          database system (tenant)
 
 # Just another view of what the tenant's reconciler actually did. It got rid of
@@ -138,6 +141,7 @@ upsert /Tenant/11/Table/6{3-4}             database system (tenant)
 upsert /Tenant/11/Table/6{4-5}             database system (tenant)
 upsert /Tenant/11/Table/6{5-6}             database system (tenant)
 upsert /Tenant/11/Table/6{6-7}             database system (tenant)
+upsert /Tenant/11/Table/6{7-8}             database system (tenant)
 
 # Initialize a new tenant, tenant=10, that DOES have a pre-existing tenant,
 # tenant=11, next to it.

diff --git a/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/named_zones b/pkg/ccl/spanconfigccl/spanconfigreconcilerccl/testdata/named_zones
@@ -144,6 +144,7 @@ state offset=46
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Table/10{6-7}                             ttl_seconds=50
 
 # Make sure future descendants observe the same.
@@ -179,6 +180,7 @@ state offset=46
 /Table/6{4-5}                              database system (host)
 /Table/6{5-6}                              database system (host)
 /Table/6{6-7}                              database system (host)
+/Table/6{7-8}                              database system (host)
 /Table/10{6-7}                             ttl_seconds=50
 /Table/10{7-8}                             ttl_seconds=50
Original file line number	Diff line number	Diff line change
Expand Up		@@ -1614,4 +1614,3 @@ statement ok
		ALTER DATABASE drop_region_126549 DROP REGION "us-east-1";

		subtest end