Skip to content

Commit

Permalink
roachtest: allow for overriding mvt planner params through env vars
Browse files Browse the repository at this point in the history 
When debugging mixed version tests, one often wishes to generate a specific
upgrade path or use a specific deployment mode. The latter is not possible
without changing the source code while the former is not supported at all.

This change exposes two env vars, MVT_UPGRADE_PATH and MVT_DEPLOYMENT_MODE
to allow for overriding mixed version plan generation.
  • Loading branch information
@DarrylWong
DarrylWong committed Dec 19, 2024
1 parent 46e8e0f commit 0e22e10
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 9 deletions.
2 changes: 1 addition & 1 deletion build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
source "$dir/teamcity-support.sh" # For $root
source "$dir/teamcity-bazel-support.sh" # For run_bazel

BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS -e MVT_UPGRADE_PATH -e MVT_DEPLOYMENT_MODE" \
run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh
2 changes: 1 addition & 1 deletion build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
source "$dir/teamcity-support.sh" # For $root
source "$dir/teamcity-bazel-support.sh" # For run_bazel

BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS -e MVT_UPGRADE_PATH -e MVT_DEPLOYMENT_MODE" \
run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh
2 changes: 1 addition & 1 deletion build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
source "$dir/teamcity-support.sh" # For $root
source "$dir/teamcity-bazel-support.sh" # For run_bazel

BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS -e MVT_UPGRADE_PATH -e MVT_DEPLOYMENT_MODE" \
run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh
34 changes: 32 additions & 2 deletions pkg/cmd/roachtest/roachtestutil/clusterupgrade/clusterupgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,19 +101,49 @@ func CurrentVersion() *Version {
// MustParseVersion parses the version string given (with or without
// leading 'v') and returns the corresponding `Version` object.
func MustParseVersion(v string) *Version {
parsedVersion, err := ParseVersion(v)
if err != nil {
panic(err)
}
return parsedVersion
}

// ParseVersion parses the version string given (with or without
// leading 'v') and returns the corresponding `Version` object. Returns
// an error if the version string is not valid.
func ParseVersion(v string) (*Version, error) {
// The current version is rendered differently (see String()
// implementation). If the user passed that string representation,
// return the current version object.
if currentVersion := CurrentVersion(); v == currentVersion.String() {
return currentVersion
return currentVersion, nil
}

versionStr := v
if !strings.HasPrefix(v, "v") {
versionStr = "v" + v
}

return &Version{*version.MustParse(versionStr)}
parsedVersion, err := version.Parse(versionStr)
if err != nil {
return nil, err
}

return &Version{*parsedVersion}, nil
}

// LatestPatchRelease returns the latest patch release version for a given
// release series.
func LatestPatchRelease(series string) (*Version, error) {
seriesStr := strings.TrimPrefix(series, "v")
versionStr, err := release.LatestPatch(seriesStr)
if err != nil {
return nil, err
}

// release.LatestPatch uses mustParseVersion internally, so the returned
// version is guaranteed to be valid.
return MustParseVersion(versionStr), nil
}

// BinaryVersion returns the binary version running on the node
Expand Down
78 changes: 74 additions & 4 deletions pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ import (
"context"
"fmt"
"math/rand"
"os"
"path/filepath"
"slices"
"strings"
Expand Down Expand Up @@ -136,6 +137,22 @@ const (
SeparateProcessDeployment = DeploymentMode("separate-process")
)

// These env vars are used by the planner to generate plans with
// certain specs regardless of the seed. This can be useful for
// forcing certain plans to be generated for debugging without needing
// trial and error.
const (
// deploymentModeOverrideEnv overrides the deployment mode used.
// - MVT_DEPLOYMENT_MODE=system
deploymentModeOverrideEnv = "MVT_DEPLOYMENT_MODE"

// upgradePathOverrideEnv is parsed to override the upgrade path used.
// Specifying a release series uses the latest patch release.
// - MVT_UPGRADE_PATH=24.1.5,24.2.0,current
// - MVT_UPGRADE_PATH=24.1,24.2,current
upgradePathOverrideEnv = "MVT_UPGRADE_PATH"
)

var (
// possibleDelays lists the possible delays to be added to
// concurrent steps.
Expand Down Expand Up @@ -780,19 +797,27 @@ func (t *Test) plan() (plan *TestPlan, retErr error) {

// Pick a random deployment mode to use in this test run among the
// list of enabled deployment modes enabled for this test.
deploymentMode := t.options.enabledDeploymentModes[t.prng.Intn(len(t.options.enabledDeploymentModes))]
deploymentMode := t.deploymentMode()
t.updateOptionsForDeploymentMode(deploymentMode)

previousReleases, err := t.choosePreviousReleases()
upgradePath, err := t.choosePreviousReleases()
if err != nil {
return nil, err
}
upgradePath = append(upgradePath, clusterupgrade.CurrentVersion())

if override := os.Getenv(upgradePathOverrideEnv); override != "" {
upgradePath, err = parseUpgradePathOverride(override)
if err != nil {
return nil, err
}
}

tenantDescriptor := t.tenantDescriptor(deploymentMode)
initialRelease := previousReleases[0]
initialRelease := upgradePath[0]

planner := testPlanner{
versions: append(previousReleases, clusterupgrade.CurrentVersion()),
versions: upgradePath,
deploymentMode: deploymentMode,
seed: t.seed,
currentContext: newInitialContext(initialRelease, t.crdbNodes, tenantDescriptor),
Expand Down Expand Up @@ -839,6 +864,7 @@ func (t *Test) runCommandFunc(nodes option.NodeListOption, cmd string) stepFunc
// only available on v22.2.0+.
func (t *Test) choosePreviousReleases() ([]*clusterupgrade.Version, error) {
skipVersions := t.prng.Float64() < t.options.skipVersionProbability

isAvailable := func(v *clusterupgrade.Version) bool {
if t.clusterArch() != vm.ArchARM64 {
return true
Expand Down Expand Up @@ -930,6 +956,15 @@ func (t *Test) numUpgrades() int {
) + t.options.minUpgrades
}

func (t *Test) deploymentMode() DeploymentMode {
deploymentMode := t.options.enabledDeploymentModes[t.prng.Intn(len(t.options.enabledDeploymentModes))]
if deploymentModeOverride := os.Getenv(deploymentModeOverrideEnv); deploymentModeOverride != "" {
deploymentMode = DeploymentMode(deploymentModeOverride)
t.logger.Printf("%s override set: %s", deploymentModeOverrideEnv, deploymentModeOverride)
}
return deploymentMode
}

// latestPredecessor is an implementation of `predecessorFunc` that
// always picks the latest predecessor for the given release version,
// ignoring the minimum supported version declared by the test.
Expand Down Expand Up @@ -1324,3 +1359,38 @@ func assertValidTest(test *Test, fatalFunc func(...interface{})) {
))
}
}

// parseUpgradePathOverride parses the upgrade path override and returns it as a list
// of versions for the framework to use instead of generating a path based on
// the seed. It assumes the user knows what it's doing and forgoes validation
// of legal upgrade paths.
func parseUpgradePathOverride(override string) ([]*clusterupgrade.Version, error) {
versions := strings.Split(override, ",")
var upgradePath []*clusterupgrade.Version
for _, v := range versions {
// Special case for the current version, as the current version on
// master is usually a long prerelease.
if v == "current" || v == "<current>" {
upgradePath = append(upgradePath, clusterupgrade.CurrentVersion())
continue
}

parsedVersion, err := clusterupgrade.ParseVersion(v)
if err == nil {
upgradePath = append(upgradePath, parsedVersion)
continue
}

// If the supplied version is invalid, it might be a release series.
// Support parsing release series as well since the user might not
// care about the exact patch version.
parsedVersion, err = clusterupgrade.LatestPatchRelease(v)
if err != nil {
return nil, errors.Newf("unable to parse version: %s", v)
}

upgradePath = append(upgradePath, parsedVersion)
}

return upgradePath, nil
}

0 comments on commit 0e22e10

Please sign in to comment.