upgrades: backfill new jobs tables/cols

Release note: none.
Epic: none.

docs/generated/settings/settings-for-tenants.txt

@@ -402,4 +402,4 @@ trace.span_registry.enabled	boolean	false	if set, ongoing traces can be seen at
 trace.zipkin.collector	string		the address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.	application
 ui.database_locality_metadata.enabled	boolean	true	if enabled shows extended locality data about databases and tables in DB Console which can be expensive to compute	application
 ui.display_timezone	enumeration	etc/utc	the timezone used to format timestamps in the ui [etc/utc = 0, america/new_york = 1]	application
-version	version	1000024.3-upgrading-to-1000025.1-step-014	set the active cluster version in the format '<major>.<minor>'	application
+version	version	1000024.3-upgrading-to-1000025.1-step-018	set the active cluster version in the format '<major>.<minor>'	application

docs/generated/settings/settings.html

@@ -361,6 +361,6 @@
 <tr><td><div id="setting-trace-zipkin-collector" class="anchored"><code>trace.zipkin.collector</code></div></td><td>string</td><td><code></code></td><td>the address of a Zipkin instance to receive traces, as &lt;host&gt;:&lt;port&gt;. If no port is specified, 9411 will be used.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-database-locality-metadata-enabled" class="anchored"><code>ui.database_locality_metadata.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if enabled shows extended locality data about databases and tables in DB Console which can be expensive to compute</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-ui-display-timezone" class="anchored"><code>ui.display_timezone</code></div></td><td>enumeration</td><td><code>etc/utc</code></td><td>the timezone used to format timestamps in the ui [etc/utc = 0, america/new_york = 1]</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
-<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000024.3-upgrading-to-1000025.1-step-014</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-version" class="anchored"><code>version</code></div></td><td>version</td><td><code>1000024.3-upgrading-to-1000025.1-step-018</code></td><td>set the active cluster version in the format &#39;&lt;major&gt;.&lt;minor&gt;&#39;</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 </tbody>
 </table>

pkg/clusterversion/cockroach_versions.go

@@ -210,6 +210,16 @@ const (
 	// V25_1_AddJobsColumns added new columns to system.jobs.
 	V25_1_AddJobsColumns
 
+	// V25_1_JobsWritesFence is an empty version that is used to add a "fence"
+	// between the column addition version and the backfill version. This allows
+	// the backfill version's upgrade step to make the assumption that all nodes
+	// will be writing to the new columns, since moving from fence to backfill can
+	// only start once no nodes are still on add-columnns.
+	V25_1_JobsWritesFence
+
+	// V25_1_JobsBackfill backfills the new jobs tables and columns.
+	V25_1_JobsBackfill
+
 	// *************************************************
 	// Step (1) Add new versions above this comment.
 	// Do not add new versions to a patch release.
@@ -255,6 +265,8 @@ var versionTable = [numKeys]roachpb.Version{
 	V25_1_BatchStreamRPC:            {Major: 24, Minor: 3, Internal: 10},
 	V25_1_PreparedTransactionsTable: {Major: 24, Minor: 3, Internal: 12},
 	V25_1_AddJobsColumns:            {Major: 24, Minor: 3, Internal: 14},
+	V25_1_JobsWritesFence:           {Major: 24, Minor: 3, Internal: 16},
+	V25_1_JobsBackfill:              {Major: 24, Minor: 3, Internal: 18},
 
 	// *************************************************
 	// Step (2): Add new versions above this comment.

pkg/sql/catalog/bootstrap/testdata/testdata

@@ -1,7 +1,7 @@
-system hash=6651b026bcec07b0bc33acbed6a792a94efbd58c69849c05c87c2e54295b42cb
+system hash=8e0584706f674381cb24315b2254e0e8ba091a6246c670d4243fe9603ad4cb5e
 ----
 [{"key":"8b"}
-,{"key":"8b89898a89","value":"0312470a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518032200280140004a006a0a08d8843d10031800200e7000"}
+,{"key":"8b89898a89","value":"0312470a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518032200280140004a006a0a08d8843d1003180020127000"}
 ,{"key":"8b898b8a89","value":"030a94030a0a64657363726970746f721803200128013a0042270a02696410011a0c08011040180030005014600020003000680070007800800100880100980100422f0a0a64657363726970746f7210021a0c08081000180030005011600020013000680070007800800100880100980100480352710a077072696d61727910011801220269642a0a64657363726970746f72300140004a10080010001a00200028003000380040005a0070027a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060026a210a0b0a0561646d696e102018200a0a0a04726f6f741020182012046e6f64651803800101880103980100b201130a077072696d61727910001a02696420012800b201240a1066616d5f325f64657363726970746f7210021a0a64657363726970746f7220022802b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300d00300d80300e00300f80300880400"}
 ,{"key":"8b898c8a89","value":"030acd050a0575736572731804200128013a00422d0a08757365726e616d6510011a0c0807100018003000501960002000300068007000780080010088010098010042330a0e68617368656450617373776f726410021a0c0808100018003000501160002001300068007000780080010088010098010042320a066973526f6c6510031a0c08001000180030005010600020002a0566616c73653000680070007800800100880100980100422c0a07757365725f696410041a0c080c100018003000501a60002000300068007000780080010088010098010048055290010a077072696d617279100118012208757365726e616d652a0e68617368656450617373776f72642a066973526f6c652a07757365725f6964300140004a10080010001a00200028003000380040005a007002700370047a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00102e00100e90100000000000000005a740a1175736572735f757365725f69645f696478100218012207757365725f69643004380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060036a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651803800101880103980100b201240a077072696d61727910001a08757365726e616d651a07757365725f6964200120042804b2012c0a1466616d5f325f68617368656450617373776f726410021a0e68617368656450617373776f726420022802b2011c0a0c66616d5f335f6973526f6c6510031a066973526f6c6520032803b80104c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880303a80300b00300d00300d80300e00300f80300880400"}
 ,{"key":"8b898d8a89","value":"030a83030a057a6f6e65731805200128013a0042270a02696410011a0c08011040180030005014600020003000680070007800800100880100980100422b0a06636f6e66696710021a0c080810001800300050116000200130006800700078008001008801009801004803526d0a077072696d61727910011801220269642a06636f6e666967300140004a10080010001a00200028003000380040005a0070027a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651803800101880103980100b201130a077072696d61727910001a02696420012800b2011c0a0c66616d5f325f636f6e66696710021a06636f6e66696720022802b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300d00300d80300e00300f80300880400"}
@@ -213,10 +213,10 @@ system hash=6651b026bcec07b0bc33acbed6a792a94efbd58c69849c05c87c2e54295b42cb
 ,{"key":"d0"}
 ]
 
-tenant hash=d7334e8ea1d51838ab3251d375cb1d4e3aeeee8c98e34c70f0ea4b6f60a8a39c
+tenant hash=1fdb41445d6744a299fa74a47ca0246d66eecb1bfda15329afc1354c83404c1e
 ----
 [{"key":""}
-,{"key":"8b89898a89","value":"0312470a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518032200280140004a006a0a08d8843d10031800200e7000"}
+,{"key":"8b89898a89","value":"0312470a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518032200280140004a006a0a08d8843d1003180020127000"}
 ,{"key":"8b898b8a89","value":"030a94030a0a64657363726970746f721803200128013a0042270a02696410011a0c08011040180030005014600020003000680070007800800100880100980100422f0a0a64657363726970746f7210021a0c08081000180030005011600020013000680070007800800100880100980100480352710a077072696d61727910011801220269642a0a64657363726970746f72300140004a10080010001a00200028003000380040005a0070027a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060026a210a0b0a0561646d696e102018200a0a0a04726f6f741020182012046e6f64651803800101880103980100b201130a077072696d61727910001a02696420012800b201240a1066616d5f325f64657363726970746f7210021a0a64657363726970746f7220022802b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300d00300d80300e00300f80300880400"}
 ,{"key":"8b898c8a89","value":"030acd050a0575736572731804200128013a00422d0a08757365726e616d6510011a0c0807100018003000501960002000300068007000780080010088010098010042330a0e68617368656450617373776f726410021a0c0808100018003000501160002001300068007000780080010088010098010042320a066973526f6c6510031a0c08001000180030005010600020002a0566616c73653000680070007800800100880100980100422c0a07757365725f696410041a0c080c100018003000501a60002000300068007000780080010088010098010048055290010a077072696d617279100118012208757365726e616d652a0e68617368656450617373776f72642a066973526f6c652a07757365725f6964300140004a10080010001a00200028003000380040005a007002700370047a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00102e00100e90100000000000000005a740a1175736572735f757365725f69645f696478100218012207757365725f69643004380140004a10080010001a00200028003000380040005a007a0408002000800100880100900103980100a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060036a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651803800101880103980100b201240a077072696d61727910001a08757365726e616d651a07757365725f6964200120042804b2012c0a1466616d5f325f68617368656450617373776f726410021a0e68617368656450617373776f726420022802b2011c0a0c66616d5f335f6973526f6c6510031a066973526f6c6520032803b80104c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880303a80300b00300d00300d80300e00300f80300880400"}
 ,{"key":"8b898d8a89","value":"030a83030a057a6f6e65731805200128013a0042270a02696410011a0c08011040180030005014600020003000680070007800800100880100980100422b0a06636f6e66696710021a0c080810001800300050116000200130006800700078008001008801009801004803526d0a077072696d61727910011801220269642a06636f6e666967300140004a10080010001a00200028003000380040005a0070027a0408002000800100880100900104980101a20106080012001800a80100b20100ba0100c00100c80100d00101e00100e901000000000000000060026a250a0d0a0561646d696e10e00318e0030a0c0a04726f6f7410e00318e00312046e6f64651803800101880103980100b201130a077072696d61727910001a02696420012800b2011c0a0c66616d5f325f636f6e66696710021a06636f6e66696720022802b80103c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200800300880302a80300b00300d00300d80300e00300f80300880400"}

pkg/sql/catalog/systemschema/system.go

@@ -1386,7 +1386,7 @@ const SystemDatabaseName = catconstants.SystemDatabaseName
 // release version).
 //
 // NB: Don't set this to clusterversion.Latest; use a specific version instead.
-var SystemDatabaseSchemaBootstrapVersion = clusterversion.V25_1_AddJobsColumns.Version()
+var SystemDatabaseSchemaBootstrapVersion = clusterversion.V25_1_JobsBackfill.Version()
 
 // MakeSystemDatabaseDesc constructs a copy of the system database
 // descriptor.

pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog

@@ -110,7 +110,7 @@ skipif config local-mixed-24.3
 query IT
 SELECT id, strip_volatile(descriptor) FROM crdb_internal.kv_catalog_descriptor WHERE id IN (1, 2, 3, 29, 4294966963) OR (id > 100 and id < 200) ORDER BY id
 ----
-1           {"database": {"id": 1, "name": "system", "privileges": {"ownerProto": "node", "users": [{"privileges": "2048", "userProto": "admin", "withGrantOption": "2048"}, {"privileges": "2048", "userProto": "root", "withGrantOption": "2048"}], "version": 3}, "systemDatabaseSchemaVersion": {"internal": 14, "majorVal": 1000024, "minorVal": 3}, "version": "1"}}
+1           {"database": {"id": 1, "name": "system", "privileges": {"ownerProto": "node", "users": [{"privileges": "2048", "userProto": "admin", "withGrantOption": "2048"}, {"privileges": "2048", "userProto": "root", "withGrantOption": "2048"}], "version": 3}, "systemDatabaseSchemaVersion": {"internal": 18, "majorVal": 1000024, "minorVal": 3}, "version": "1"}}
 3           {"table": {"columns": [{"id": 1, "name": "id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "descriptor", "nullable": true, "type": {"family": "BytesFamily", "oid": 17}}], "formatVersion": 3, "id": 3, "name": "descriptor", "nextColumnId": 3, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "parentId": 1, "primaryIndex": {"constraintId": 1, "encodingType": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [1], "keyColumnNames": ["id"], "name": "primary", "partitioning": {}, "sharded": {}, "storeColumnIds": [2], "storeColumnNames": ["descriptor"], "unique": true, "version": 4}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "admin", "withGrantOption": "32"}, {"privileges": "32", "userProto": "root", "withGrantOption": "32"}], "version": 3}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 29, "version": "1"}}
 29          {"schema": {"defaultPrivileges": {"type": "SCHEMA"}, "id": 29, "name": "public", "privileges": {"ownerProto": "node", "users": [{"privileges": "2", "userProto": "admin", "withGrantOption": "2"}, {"privileges": "516", "userProto": "public"}, {"privileges": "2", "userProto": "root", "withGrantOption": "2"}], "version": 3}, "version": "1"}}
 101         {"schema": {"id": 101, "name": "public", "parentId": 100, "privileges": {"ownerProto": "root", "users": [{"privileges": "2", "userProto": "admin", "withGrantOption": "2"}, {"privileges": "516", "userProto": "public"}, {"privileges": "2", "userProto": "root", "withGrantOption": "2"}], "version": 3}, "version": "1"}}

pkg/upgrade/upgrades/BUILD.bazel

@@ -54,6 +54,7 @@ go_library(
         "//pkg/upgrade",
         "//pkg/upgrade/upgradebase",
         "//pkg/util/envutil",
+        "//pkg/util/hlc",
         "//pkg/util/log",
         "//pkg/util/protoutil",
         "//pkg/util/timeutil",
@@ -80,6 +81,7 @@ go_test(
         "schema_changes_external_test.go",
         "schema_changes_helpers_test.go",
         "upgrades_test.go",
+        "v25_1_add_jobs_tables_test.go",
         "v25_1_prepared_transactions_table_test.go",
         "version_starvation_test.go",
     ],
@@ -124,6 +126,7 @@ go_test(
         "//pkg/sql/sem/tree",
         "//pkg/sql/types",
         "//pkg/testutils",
+        "//pkg/testutils/jobutils",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/skip",
         "//pkg/testutils/sqlutils",

pkg/upgrade/upgrades/upgrades.go

@@ -83,7 +83,13 @@ var upgrades = []upgradebase.Upgrade{
 		addJobsColumns,
 		upgrade.RestoreActionNotRequired("cluster restore does not restore the new field"),
 	),
-
+	upgrade.NewTenantUpgrade(
+		"backfill new jobs tables",
+		clusterversion.V25_1_JobsBackfill.Version(),
+		upgrade.NoPrecondition,
+		backfillJobsTablesAndColumns,
+		upgrade.RestoreActionNotRequired("cluster restore does not restore jobs tables"),
+	),
 	// Note: when starting a new release version, the first upgrade (for
 	// Vxy_zStart) must be a newFirstUpgrade. Keep this comment at the bottom.
 }

pkg/upgrade/upgrades/v25_1_add_jobs_tables.go

@@ -9,11 +9,16 @@ import (
 	"context"
 
 	"github.com/cockroachdb/cockroach/pkg/clusterversion"
+	"github.com/cockroachdb/cockroach/pkg/jobs"
 	"github.com/cockroachdb/cockroach/pkg/keys"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descs"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
+	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
 	"github.com/cockroachdb/cockroach/pkg/upgrade"
+	"github.com/cockroachdb/cockroach/pkg/util/hlc"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/errors"
 )
 
 // addJobsTables adds the job_progress, job_progress_history, job_status and
@@ -76,3 +81,111 @@ func addJobsColumns(
 		keys.JobsTableID,
 		systemschema.JobsTable)
 }
+
+func TestingSetJobsBackfillPageSize(size int) {
+	jobsBackfillPageSize = size
+}
+
+var jobsBackfillPageSize = 32
+
+// backfills the new jobs tables and columns
+func backfillJobsTablesAndColumns(
+	ctx context.Context, cv clusterversion.ClusterVersion, d upgrade.TenantDeps,
+) error {
+	log.Infof(ctx, "backfilling new jobs tables and columns")
+	jobsBackfilled := 0
+	for {
+		var done int
+		if err := d.DB.DescsTxn(ctx, func(ctx context.Context, tx descs.Txn) (retErr error) {
+			// Find rows that have not been backfilled, which we will detect via a
+			// NULL owner. The coalesce() here isn't necessary since the vtable will
+			// produce empty strings rather than nulls if a job somehow is missing an
+			// owner, but since the owner not being null is what breaks the loop the
+			// extra coalesce here just ensures wouldn't loop forever if that were to
+			// change out from under us.
+			q, err := tx.QueryIteratorEx(ctx, "jobs-backfill-read", tx.KV(), sessiondata.NodeUserSessionDataOverride,
+				`SELECT
+				v.job_id,
+				v.description,
+				coalesce(v.user_name, ''),
+				v.finished,
+				v.error,
+				v.running_status,
+				v.fraction_completed,
+				v.high_water_timestamp
+				FROM crdb_internal.jobs v
+				LEFT JOIN system.jobs j ON j.id = v.job_id
+				WHERE j.owner IS NULL
+				LIMIT $1`, jobsBackfillPageSize,
+			)
+			if err != nil {
+				return err
+			}
+			defer func() {
+				retErr = errors.CombineErrors(retErr, q.Close())
+			}()
+
+			done = 0
+			for {
+				ok, err := q.Next(ctx)
+				if err != nil {
+					return err
+				}
+				if !ok {
+					break
+				}
+
+				row := q.Cur()
+
+				// Update the job row.
+				if _, err := tx.ExecEx(ctx, "jobs-backfill-jobs", tx.KV(),
+					sessiondata.NodeUserSessionDataOverride,
+					`UPDATE system.jobs
+					SET description = $1,
+					owner = $2,
+					finished = $3,
+					error_msg = NULLIF($4, '')
+					WHERE id = $5`, row[1], row[2], row[3], row[4], row[0],
+				); err != nil {
+					return err
+				}
+				// If we see a running_status, we need to update the status.
+				if row[5] != tree.DNull {
+					if err := jobs.StatusStorage(tree.MustBeDInt(row[0])).Set(ctx, tx, string(tree.MustBeDString(row[5]))); err != nil {
+						return err
+					}
+				}
+				// If we see a fraction_completed or high_water_timestamp, we need to
+				// update the progress.
+				if row[6] != tree.DNull || row[7] != tree.DNull {
+					var frac float64
+					var ts hlc.Timestamp
+					if row[6] != tree.DNull {
+						frac = float64(tree.MustBeDFloat(row[6]))
+					}
+					if row[7] != tree.DNull {
+						d := tree.MustBeDDecimal(row[7]).Decimal
+						ts, err = hlc.DecimalToHLC(&d)
+						if err != nil {
+							return err
+						}
+					}
+					if err := jobs.ProgressStorage(tree.MustBeDInt(row[0])).Set(ctx, tx, frac, ts); err != nil {
+						return err
+					}
+				}
+				done++
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		jobsBackfilled += done
+		if done == 0 {
+			break
+		}
+	}
+	log.Infof(ctx, "finished backfilling new jobs tables and columns for %d jobs", jobsBackfilled)
+	return nil
+}