From 089a644a7910635129539c61625701ccd94c3ab3 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Mon, 18 Nov 2024 09:29:47 +0530 Subject: [PATCH] roachtest: fix bench roachtests failing with validation when using json The roachtests starting failing with validation errors `Error: error creating metrics exporter: file path must end with .json` after this PR was merged https://github.com/cockroachdb/cockroach/pull/133035.The path defined in `GetWorkloadHistogramArgs` was jumbled. Also there was an error in https://github.com/cockroachdb/cockroach/pull/135239. This change intends to fix all the errors. Epic: none Release note: None --- .../nightlies/roachtest_nightly_aws.sh | 2 +- .../nightlies/roachtest_nightly_aws_fips.sh | 2 +- .../nightlies/roachtest_nightly_azure.sh | 2 +- .../nightlies/roachtest_nightly_gce.sh | 2 +- .../nightlies/roachtest_nightly_gce_fips.sh | 2 +- .../nightlies/roachtest_nightly_impl.sh | 1 - build/teamcity/util/roachtest_util.sh | 29 +++++-------------- pkg/cmd/roachtest/roachtestutil/utils.go | 2 +- 8 files changed, 13 insertions(+), 29 deletions(-) diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh index b5228c329c4b..6d8235585a1d 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_aws_fips.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_aws_fips.sh index 43765b6a263d..b1f57f1c1ad8 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_aws_fips.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_aws_fips.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e FIPS_ENABLED=1 -e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e FIPS_ENABLED=1 -e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh index 00b3844cd031..86b93565390b 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh index 001531b16062..6f7b9e97d8ea 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce_fips.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce_fips.sh index ba4ec38854ed..941492753dc8 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_gce_fips.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_gce_fips.sh @@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" # For $root source "$dir/teamcity-bazel-support.sh" # For run_bazel -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e FIPS_ENABLED=1 -e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e FIPS_ENABLED=1 -e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD" \ run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh diff --git a/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh b/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh index ad9bcd5d44c4..ed58b84a0bf7 100755 --- a/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh +++ b/build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh @@ -84,5 +84,4 @@ build/teamcity-roachtest-invoke.sh \ --suite nightly \ --selective-tests="${SELECTIVE_TESTS:-true}" \ --side-eye-token="${SIDE_EYE_API_TOKEN}" \ - --export-openmetrics="${EXPORT_OPENMETRICS}" \ "${TESTS}" diff --git a/build/teamcity/util/roachtest_util.sh b/build/teamcity/util/roachtest_util.sh index 42385d38d930..3f9468ca991e 100644 --- a/build/teamcity/util/roachtest_util.sh +++ b/build/teamcity/util/roachtest_util.sh @@ -23,21 +23,12 @@ source $root/build/teamcity/util/roachtest_arch_util.sh # date at the time of the start of the run (which identifies the version of the # code run best). stats_dir="$(date +"%Y%m%d")-${TC_BUILD_ID}" -stats_file_name="stats.json" - -if [[ "${EXPORT_OPENMETRICS}" == "true"]]; then - stats_file_name="stats.om" -fi # Set up a function we'll invoke at the end. function upload_stats { if tc_release_branch; then bucket="${ROACHTEST_BUCKET:-cockroach-nightly-${CLOUD}}" - if [[ "${EXPORT_OPENMETRICS}" == "true"]]; then - bucket="${ROACHTEST_BUCKET:-cockroach-roachperf-nightly/metrics-loader/incoming/${CLOUD}}" - fi - - if [[ "${CLOUD}" == "gce" && "${EXPORT_OPENMETRICS}" == "false" ]]; then + if [[ "${CLOUD}" == "gce" ]]; then # GCE, having been there first, gets an exemption. bucket="cockroach-nightly" fi @@ -55,12 +46,12 @@ function upload_stats { remote_artifacts_dir="${remote_artifacts_dir}-fips" fi - # The ${stats_file_name} files need some path translation: - # ${artifacts}/path/to/test/${stats_file_name} + # The stats.json files need some path translation: + # ${artifacts}/path/to/test/stats.json # to - # gs://${bucket}/artifacts/${stats_dir}/path/to/test/${stats_file_name} + # gs://${bucket}/artifacts/${stats_dir}/path/to/test/stats.json # - # `find` below will expand "{}" as ./path/to/test/${stats_file_name}. We need + # `find` below will expand "{}" as ./path/to/test/stats.json. We need # to bend over backwards to remove the `./` prefix or gsutil will have # a `.` folder in ${stats_dir}, which we don't want. (cd "${artifacts}" && \ @@ -75,7 +66,7 @@ function upload_stats { fi gsutil cp "${f}" "gs://${bucket}/${artifacts_dir}/${stats_dir}/${f}" fi - done <<< "$(find . -name ${stats_file_name} | sed 's/^\.\///')") + done <<< "$(find . -name stats.json | sed 's/^\.\///')") fi } @@ -97,17 +88,11 @@ function upload_binaries { } function upload_all { - - if [[ "${EXPORT_OPENMETRICS}" == "true"]]; then - echo "$ROACHPERF_OPENMETRICS_CREDENTIALS" > roachperf.json - gcloud auth activate-service-account --key-file=roachperf.json - fi - upload_stats upload_binaries } -# Upload any ${stats_file_name} we can find, and some binaries, no matter what happens. +# Upload any stats.json we can find, and some binaries, no matter what happens. trap upload_all EXIT # Set up the parameters for the roachtest invocation. diff --git a/pkg/cmd/roachtest/roachtestutil/utils.go b/pkg/cmd/roachtest/roachtestutil/utils.go index f2fd49fa3719..7b6aa410d0b0 100644 --- a/pkg/cmd/roachtest/roachtestutil/utils.go +++ b/pkg/cmd/roachtest/roachtestutil/utils.go @@ -79,7 +79,7 @@ func GetWorkloadHistogramArgs(t test.Test, c cluster.Cluster, labels map[string] t.PerfArtifactsDir(), GetBenchmarkMetricsFileName(t), clusterstats.GetOpenmetricsLabelString(t, c, labels)) } else { // Since default is json, no need to add --histogram-export-format flag in this case and also the labels - histogramArgs = fmt.Sprintf(" --histograms=%s/%s", GetBenchmarkMetricsFileName(t), t.PerfArtifactsDir()) + histogramArgs = fmt.Sprintf(" --histograms=%s/%s", t.PerfArtifactsDir(), GetBenchmarkMetricsFileName(t)) } return histogramArgs