test

build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh

@@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"  # For $root
 source "$dir/teamcity-bazel-support.sh"  # For run_bazel
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS -e MVT_UPGRADE_PATH -e MVT_DEPLOYMENT_MODE" \
 			       run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

pkg/cmd/roachtest/roachtestutil/clusterupgrade/clusterupgrade.go

@@ -101,19 +101,46 @@ func CurrentVersion() *Version {
 // MustParseVersion parses the version string given (with or without
 // leading 'v') and returns the corresponding `Version` object.
 func MustParseVersion(v string) *Version {
+	parsedVersion, err := ParseVersion(v)
+	if err != nil {
+		panic(err)
+	}
+	return parsedVersion
+}
+
+// ParseVersion parses the version string given (with or without
+// leading 'v') and returns the corresponding `Version` object. Returns
+// an error if the version string is not valid.
+func ParseVersion(v string) (*Version, error) {
 	// The current version is rendered differently (see String()
 	// implementation). If the user passed that string representation,
 	// return the current version object.
 	if currentVersion := CurrentVersion(); v == currentVersion.String() {
-		return currentVersion
+		return currentVersion, nil
 	}
 
 	versionStr := v
 	if !strings.HasPrefix(v, "v") {
 		versionStr = "v" + v
 	}
 
-	return &Version{*version.MustParse(versionStr)}
+	parsedVersion, err := version.Parse(versionStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Version{*parsedVersion}, nil
+}
+
+// LatestPatchRelease returns the latest patch release version for a given
+// release series.
+func LatestPatchRelease(series string) (*Version, error) {
+	seriesStr := strings.TrimPrefix(series, "v")
+	versionStr, err := release.LatestPatch(seriesStr)
+	if err != nil {
+		return nil, err
+	}
+	return MustParseVersion(versionStr), nil
 }
 
 // BinaryVersion returns the binary version running on the node

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go

@@ -70,6 +70,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"os"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -134,6 +135,13 @@ const (
 	SystemOnlyDeployment      = DeploymentMode("system-only")
 	SharedProcessDeployment   = DeploymentMode("shared-process")
 	SeparateProcessDeployment = DeploymentMode("separate-process")
+
+	// These env vars are used by the planner to generate plans with
+	// certain specs regardless of the seed. This can be useful for
+	// forcing certain plans to be generated for debugging without needing
+	// trial and error.
+	deploymentModeOverrideEnv = "MVT_DEPLOYMENT_MODE"
+	upgradePathOverrideEnv    = "MVT_UPGRADE_PATH"
 )
 
 var (
@@ -780,19 +788,27 @@ func (t *Test) plan() (plan *TestPlan, retErr error) {
 
 	// Pick a random deployment mode to use in this test run among the
 	// list of enabled deployment modes enabled for this test.
-	deploymentMode := t.options.enabledDeploymentModes[t.prng.Intn(len(t.options.enabledDeploymentModes))]
+	deploymentMode := t.deploymentMode()
 	t.updateOptionsForDeploymentMode(deploymentMode)
 
-	previousReleases, err := t.choosePreviousReleases()
+	upgradePath, err := t.choosePreviousReleases()
 	if err != nil {
 		return nil, err
 	}
+	upgradePath = append(upgradePath, clusterupgrade.CurrentVersion())
+
+	if override := os.Getenv(upgradePathOverrideEnv); override != "" {
+		upgradePath, err = parseUpgradePathOverride(override)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	tenantDescriptor := t.tenantDescriptor(deploymentMode)
-	initialRelease := previousReleases[0]
+	initialRelease := upgradePath[0]
 
 	planner := testPlanner{
-		versions:       append(previousReleases, clusterupgrade.CurrentVersion()),
+		versions:       upgradePath,
 		deploymentMode: deploymentMode,
 		seed:           t.seed,
 		currentContext: newInitialContext(initialRelease, t.crdbNodes, tenantDescriptor),
@@ -839,6 +855,7 @@ func (t *Test) runCommandFunc(nodes option.NodeListOption, cmd string) stepFunc
 // only available on v22.2.0+.
 func (t *Test) choosePreviousReleases() ([]*clusterupgrade.Version, error) {
 	skipVersions := t.prng.Float64() < t.options.skipVersionProbability
+
 	isAvailable := func(v *clusterupgrade.Version) bool {
 		if t.clusterArch() != vm.ArchARM64 {
 			return true
@@ -930,6 +947,15 @@ func (t *Test) numUpgrades() int {
 	) + t.options.minUpgrades
 }
 
+func (t *Test) deploymentMode() DeploymentMode {
+	deploymentMode := t.options.enabledDeploymentModes[t.prng.Intn(len(t.options.enabledDeploymentModes))]
+	if deploymentModeOverride := os.Getenv(deploymentModeOverrideEnv); deploymentModeOverride != "" {
+		deploymentMode = DeploymentMode(deploymentModeOverride)
+		t.logger.Printf("%s override set: %s", deploymentModeOverrideEnv, deploymentModeOverride)
+	}
+	return deploymentMode
+}
+
 // latestPredecessor is an implementation of `predecessorFunc` that
 // always picks the latest predecessor for the given release version,
 // ignoring the minimum supported version declared by the test.
@@ -1324,3 +1350,38 @@ func assertValidTest(test *Test, fatalFunc func(...interface{})) {
 		))
 	}
 }
+
+// parseUpgradePathOverride parses the upgrade path override and returns it as a list
+// of versions for the framework to use instead of generating a path based on
+// the seed. It assumes the user knows what it's doing and forgoes validation
+// of legal upgrade paths.
+func parseUpgradePathOverride(override string) ([]*clusterupgrade.Version, error) {
+	versions := strings.Split(override, ",")
+	var upgradePath []*clusterupgrade.Version
+	for _, v := range versions {
+		// Special case for the current version, as the current version on
+		// master is usually a long prerelease.
+		if v == "current" || v == "<current>" {
+			upgradePath = append(upgradePath, clusterupgrade.CurrentVersion())
+			continue
+		}
+
+		parsedVersion, err := clusterupgrade.ParseVersion(v)
+		if err == nil {
+			upgradePath = append(upgradePath, parsedVersion)
+			continue
+		}
+
+		// If the supplied version is invalid, it might be a release series.
+		// Support parsing release series as well since the user might not
+		// care about the exact patch version.
+		parsedVersion, err = clusterupgrade.LatestPatchRelease(v)
+		if err != nil {
+			return nil, errors.Newf("unable to parse version: %s", v)
+		}
+
+		upgradePath = append(upgradePath, parsedVersion)
+	}
+
+	return upgradePath, nil
+}