diff --git a/docs/generated/metrics/metrics.html b/docs/generated/metrics/metrics.html
index c3bb5502776c..3ef9b32c5da8 100644
--- a/docs/generated/metrics/metrics.html
+++ b/docs/generated/metrics/metrics.html
@@ -19,6 +19,7 @@
STORAGE | admission.admitted.kv-stores.high-pri | Number of requests admitted | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.admitted.kv-stores.locking-normal-pri | Number of requests admitted | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.admitted.kv-stores.normal-pri | Number of requests admitted | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
+STORAGE | admission.admitted.kv-stores.user-high-pri | Number of requests admitted | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.admitted.kv.high-pri | Number of requests admitted | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.admitted.kv.locking-normal-pri | Number of requests admitted | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.admitted.kv.normal-pri | Number of requests admitted | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
@@ -54,6 +55,7 @@
STORAGE | admission.errored.kv-stores.high-pri | Number of requests not admitted due to error | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.errored.kv-stores.locking-normal-pri | Number of requests not admitted due to error | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.errored.kv-stores.normal-pri | Number of requests not admitted due to error | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
+STORAGE | admission.errored.kv-stores.user-high-pri | Number of requests not admitted due to error | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.errored.kv.high-pri | Number of requests not admitted due to error | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.errored.kv.locking-normal-pri | Number of requests not admitted due to error | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.errored.kv.normal-pri | Number of requests not admitted due to error | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
@@ -101,6 +103,7 @@
STORAGE | admission.requested.kv-stores.high-pri | Number of requests | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.requested.kv-stores.locking-normal-pri | Number of requests | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.requested.kv-stores.normal-pri | Number of requests | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
+STORAGE | admission.requested.kv-stores.user-high-pri | Number of requests | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.requested.kv.high-pri | Number of requests | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.requested.kv.locking-normal-pri | Number of requests | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
STORAGE | admission.requested.kv.normal-pri | Number of requests | Requests | COUNTER | COUNT | AVG | NON_NEGATIVE_DERIVATIVE |
@@ -128,6 +131,7 @@
STORAGE | admission.wait_durations.kv-stores.high-pri | Wait time durations for requests that waited | Wait time Duration | HISTOGRAM | NANOSECONDS | AVG | NONE |
STORAGE | admission.wait_durations.kv-stores.locking-normal-pri | Wait time durations for requests that waited | Wait time Duration | HISTOGRAM | NANOSECONDS | AVG | NONE |
STORAGE | admission.wait_durations.kv-stores.normal-pri | Wait time durations for requests that waited | Wait time Duration | HISTOGRAM | NANOSECONDS | AVG | NONE |
+STORAGE | admission.wait_durations.kv-stores.user-high-pri | Wait time durations for requests that waited | Wait time Duration | HISTOGRAM | NANOSECONDS | AVG | NONE |
STORAGE | admission.wait_durations.kv.high-pri | Wait time durations for requests that waited | Wait time Duration | HISTOGRAM | NANOSECONDS | AVG | NONE |
STORAGE | admission.wait_durations.kv.locking-normal-pri | Wait time durations for requests that waited | Wait time Duration | HISTOGRAM | NANOSECONDS | AVG | NONE |
STORAGE | admission.wait_durations.kv.normal-pri | Wait time durations for requests that waited | Wait time Duration | HISTOGRAM | NANOSECONDS | AVG | NONE |
@@ -155,6 +159,7 @@
STORAGE | admission.wait_queue_length.kv-stores.high-pri | Length of wait queue | Requests | GAUGE | COUNT | AVG | NONE |
STORAGE | admission.wait_queue_length.kv-stores.locking-normal-pri | Length of wait queue | Requests | GAUGE | COUNT | AVG | NONE |
STORAGE | admission.wait_queue_length.kv-stores.normal-pri | Length of wait queue | Requests | GAUGE | COUNT | AVG | NONE |
+STORAGE | admission.wait_queue_length.kv-stores.user-high-pri | Length of wait queue | Requests | GAUGE | COUNT | AVG | NONE |
STORAGE | admission.wait_queue_length.kv.high-pri | Length of wait queue | Requests | GAUGE | COUNT | AVG | NONE |
STORAGE | admission.wait_queue_length.kv.locking-normal-pri | Length of wait queue | Requests | GAUGE | COUNT | AVG | NONE |
STORAGE | admission.wait_queue_length.kv.normal-pri | Length of wait queue | Requests | GAUGE | COUNT | AVG | NONE |
diff --git a/pkg/kv/kvserver/client_raft_log_queue_test.go b/pkg/kv/kvserver/client_raft_log_queue_test.go
index abaadb9d5bb0..10a70e2d2888 100644
--- a/pkg/kv/kvserver/client_raft_log_queue_test.go
+++ b/pkg/kv/kvserver/client_raft_log_queue_test.go
@@ -201,8 +201,10 @@ func TestRaftTracing(t *testing.T) {
expectedMessages := []string{
`replica_proposal_buf.* flushing proposal to Raft`,
`replica_proposal_buf.* registering local trace`,
+ // Note that we don't assert that the 1->3 MsgApp goes through, as
+ // the ordering may change between 1->2 and 1->3. It should be
+ // sufficient to just check one of them for tracing.
`replica_raft.* 1->2 MsgApp`,
- `replica_raft.* 1->3 MsgApp`,
`replica_raft.* AppendThread->1 MsgStorageAppendResp`,
`ack-ing replication success to the client`,
}
diff --git a/pkg/kv/kvserver/kvflowcontrol/BUILD.bazel b/pkg/kv/kvserver/kvflowcontrol/BUILD.bazel
index 077a49e8dd87..2caac173dc03 100644
--- a/pkg/kv/kvserver/kvflowcontrol/BUILD.bazel
+++ b/pkg/kv/kvserver/kvflowcontrol/BUILD.bazel
@@ -19,6 +19,7 @@ go_library(
"//pkg/util/admission/admissionpb",
"//pkg/util/ctxutil",
"//pkg/util/humanizeutil",
+ "//pkg/util/metamorphic",
"@com_github_cockroachdb_redact//:redact",
],
)
diff --git a/pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go b/pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go
index 09bb9a44cf7e..35365094b9c2 100644
--- a/pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go
+++ b/pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go
@@ -20,6 +20,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/util/admission/admissionpb"
"github.com/cockroachdb/cockroach/pkg/util/ctxutil"
"github.com/cockroachdb/cockroach/pkg/util/humanizeutil"
+ "github.com/cockroachdb/cockroach/pkg/util/metamorphic"
"github.com/cockroachdb/redact"
)
@@ -41,7 +42,11 @@ var Mode = settings.RegisterEnumSetting(
settings.SystemOnly,
"kvadmission.flow_control.mode",
"determines the 'mode' of flow control we use for replication traffic in KV, if enabled",
- modeDict[ApplyToElastic], /* default value */
+ metamorphic.ConstantWithTestChoice(
+ "kvadmission.flow_control.mode",
+ modeDict[ApplyToAll], /* default value */
+ modeDict[ApplyToElastic], /* other value */
+ ),
modeDict,
)
diff --git a/pkg/testutils/testcluster/testcluster.go b/pkg/testutils/testcluster/testcluster.go
index 93adda29e143..0c2f833dba45 100644
--- a/pkg/testutils/testcluster/testcluster.go
+++ b/pkg/testutils/testcluster/testcluster.go
@@ -1479,7 +1479,9 @@ func (tc *TestCluster) WaitForFullReplication() error {
// Force upreplication. Otherwise, if we rely on the scanner to do it,
// it'll take a while.
if err := s.ForceReplicationScanAndProcess(); err != nil {
- return err
+ log.Infof(context.TODO(), "%v", err)
+ notReplicated = true
+ return nil
}
if err := s.ComputeMetrics(context.TODO()); err != nil {
// This can sometimes fail since ComputeMetrics calls