diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh index b5228c329c4b..cbdd326761f9 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS -e MVT_UPGRADE_PATH -e MVT_DEPLOYMENT_MODE" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh index 00b3844cd031..6fff6ba986e6 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS -e MVT_UPGRADE_PATH -e MVT_DEPLOYMENT_MODE" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh index 001531b16062..ef298733e5dd 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS -e MVT_UPGRADE_PATH -e MVT_DEPLOYMENT_MODE" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/pkg/cmd/roachtest/roachtestutil/clusterupgrade/clusterupgrade.go b/pkg/cmd/roachtest/roachtestutil/clusterupgrade/clusterupgrade.go index 2a0cba408f34..135107eefef9 100644 --- a/pkg/cmd/roachtest/roachtestutil/clusterupgrade/clusterupgrade.go +++ b/pkg/cmd/roachtest/roachtestutil/clusterupgrade/clusterupgrade.go @@ -101,11 +101,22 @@ func CurrentVersion() *Version { // MustParseVersion parses the version string given (with or without // leading 'v') and returns the corresponding `Version` object. func MustParseVersion(v string) *Version { + parsedVersion, err := ParseVersion(v) + if err != nil { + panic(err) + } + return parsedVersion +} + +// ParseVersion parses the version string given (with or without +// leading 'v') and returns the corresponding `Version` object. Returns +// an error if the version string is not valid. +func ParseVersion(v string) (*Version, error) { // The current version is rendered differently (see String() // implementation). If the user passed that string representation, // return the current version object. if currentVersion := CurrentVersion(); v == currentVersion.String() { - return currentVersion + return currentVersion, nil } versionStr := v @@ -113,7 +124,26 @@ func MustParseVersion(v string) *Version { versionStr = "v" + v } - return &Version{*version.MustParse(versionStr)} + parsedVersion, err := version.Parse(versionStr) + if err != nil { + return nil, err + } + + return &Version{*parsedVersion}, nil +} + +// LatestPatchRelease returns the latest patch release version for a given +// release series. +func LatestPatchRelease(series string) (*Version, error) { + seriesStr := strings.TrimPrefix(series, "v") + versionStr, err := release.LatestPatch(seriesStr) + if err != nil { + return nil, err + } + + // release.LatestPatch uses mustParseVersion internally, so the returned + // version is guaranteed to be valid. + return MustParseVersion(versionStr), nil } // BinaryVersion returns the binary version running on the node diff --git a/pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go b/pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go index b8f144a027cd..d32318cbce1b 100644 --- a/pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go +++ b/pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go @@ -70,6 +70,7 @@ import ( "context" "fmt" "math/rand" + "os" "path/filepath" "slices" "strings" @@ -136,6 +137,22 @@ const ( SeparateProcessDeployment = DeploymentMode("separate-process") ) +// These env vars are used by the planner to generate plans with +// certain specs regardless of the seed. This can be useful for +// forcing certain plans to be generated for debugging without needing +// trial and error. +const ( + // deploymentModeOverrideEnv overrides the deployment mode used. + // - MVT_DEPLOYMENT_MODE=system + deploymentModeOverrideEnv = "MVT_DEPLOYMENT_MODE" + + // upgradePathOverrideEnv is parsed to override the upgrade path used. + // Specifying a release series uses the latest patch release. + // - MVT_UPGRADE_PATH=24.1.5,24.2.0,current + // - MVT_UPGRADE_PATH=24.1,24.2,current + upgradePathOverrideEnv = "MVT_UPGRADE_PATH" +) + var ( // possibleDelays lists the possible delays to be added to // concurrent steps. @@ -780,19 +797,27 @@ func (t *Test) plan() (plan *TestPlan, retErr error) { // Pick a random deployment mode to use in this test run among the // list of enabled deployment modes enabled for this test. - deploymentMode := t.options.enabledDeploymentModes[t.prng.Intn(len(t.options.enabledDeploymentModes))] + deploymentMode := t.deploymentMode() t.updateOptionsForDeploymentMode(deploymentMode) - previousReleases, err := t.choosePreviousReleases() + upgradePath, err := t.choosePreviousReleases() if err != nil { return nil, err } + upgradePath = append(upgradePath, clusterupgrade.CurrentVersion()) + + if override := os.Getenv(upgradePathOverrideEnv); override != "" { + upgradePath, err = parseUpgradePathOverride(override) + if err != nil { + return nil, err + } + } tenantDescriptor := t.tenantDescriptor(deploymentMode) - initialRelease := previousReleases[0] + initialRelease := upgradePath[0] planner := testPlanner{ - versions: append(previousReleases, clusterupgrade.CurrentVersion()), + versions: upgradePath, deploymentMode: deploymentMode, seed: t.seed, currentContext: newInitialContext(initialRelease, t.crdbNodes, tenantDescriptor), @@ -839,6 +864,7 @@ func (t *Test) runCommandFunc(nodes option.NodeListOption, cmd string) stepFunc // only available on v22.2.0+. func (t *Test) choosePreviousReleases() ([]*clusterupgrade.Version, error) { skipVersions := t.prng.Float64() < t.options.skipVersionProbability + isAvailable := func(v *clusterupgrade.Version) bool { if t.clusterArch() != vm.ArchARM64 { return true @@ -930,6 +956,15 @@ func (t *Test) numUpgrades() int { ) + t.options.minUpgrades } +func (t *Test) deploymentMode() DeploymentMode { + deploymentMode := t.options.enabledDeploymentModes[t.prng.Intn(len(t.options.enabledDeploymentModes))] + if deploymentModeOverride := os.Getenv(deploymentModeOverrideEnv); deploymentModeOverride != "" { + deploymentMode = DeploymentMode(deploymentModeOverride) + t.logger.Printf("%s override set: %s", deploymentModeOverrideEnv, deploymentModeOverride) + } + return deploymentMode +} + // latestPredecessor is an implementation of `predecessorFunc` that // always picks the latest predecessor for the given release version, // ignoring the minimum supported version declared by the test. @@ -1324,3 +1359,38 @@ func assertValidTest(test *Test, fatalFunc func(...interface{})) { )) } } + +// parseUpgradePathOverride parses the upgrade path override and returns it as a list +// of versions for the framework to use instead of generating a path based on +// the seed. It assumes the user knows what it's doing and forgoes validation +// of legal upgrade paths. +func parseUpgradePathOverride(override string) ([]*clusterupgrade.Version, error) { + versions := strings.Split(override, ",") + var upgradePath []*clusterupgrade.Version + for _, v := range versions { + // Special case for the current version, as the current version on + // master is usually a long prerelease. + if v == "current" || v == "" { + upgradePath = append(upgradePath, clusterupgrade.CurrentVersion()) + continue + } + + parsedVersion, err := clusterupgrade.ParseVersion(v) + if err == nil { + upgradePath = append(upgradePath, parsedVersion) + continue + } + + // If the supplied version is invalid, it might be a release series. + // Support parsing release series as well since the user might not + // care about the exact patch version. + parsedVersion, err = clusterupgrade.LatestPatchRelease(v) + if err != nil { + return nil, errors.Newf("unable to parse version: %s", v) + } + + upgradePath = append(upgradePath, parsedVersion) + } + + return upgradePath, nil +}