kvflowcontrol: re-enable kvadmission.flow_control.mode metamorphism

The cluster setting `kvadmission.flow_control.mode` is used to control
whether replication admission control applies to only elastic work, or
applies to all work which sets an admission priority.

The cluster setting was prior to #132125, metamorphic in tests,
selecting either value with equal probability. #132125 disabled the
metamorphism in order to prevent the newly introduced send queue / pull
mode from being run unexpectedly, while still under testing.

With testing now in place, re-enable the metamorphism, leaving the
default value unchanged.

Resolves: #132364
Release note: None

pkg/kv/kvserver/client_replica_raft_overload_test.go

@@ -59,6 +59,11 @@ func TestReplicaRaftOverload(t *testing.T) {
 	{
 		_, err := tc.ServerConn(0).Exec(`SET CLUSTER SETTING admission.kv.pause_replication_io_threshold = 1.0`)
 		require.NoError(t, err)
+    // NB: Follower pausing is only enabled when the flow control mode is set to
+    // 'apply_to_elastic'. This is because the follower pausing mechanism is subsumed
+    // by RACv2 when the flow control mode is set to 'apply_to_all'.
+		_, err = tc.ServerConn(0).Exec(`SET CLUSTER SETTING kvadmission.flow_control.mode = 'apply_to_elastic'`)
+		require.NoError(t, err)
 	}
 	k := tc.ScratchRange(t)
 	tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...)

pkg/kv/kvserver/kvflowcontrol/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "//pkg/settings",
         "//pkg/settings/cluster",
         "//pkg/util/admission/admissionpb",
+        "//pkg/util/metamorphic",
         "@com_github_cockroachdb_redact//:redact",
         "@com_github_dustin_go_humanize//:go-humanize",
     ],

pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go

@@ -19,6 +19,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/util/admission/admissionpb"
+	"github.com/cockroachdb/cockroach/pkg/util/metamorphic"
 	"github.com/cockroachdb/redact"
 	"github.com/dustin/go-humanize"
 )
@@ -41,7 +42,11 @@ var Mode = settings.RegisterEnumSetting(
 	settings.SystemOnly,
 	"kvadmission.flow_control.mode",
 	"determines the 'mode' of flow control we use for replication traffic in KV, if enabled",
-	modeDict[ApplyToElastic], /* default value */
+	metamorphic.ConstantWithTestChoice(
+		"kvadmission.flow_control.mode",
+		modeDict[ApplyToElastic], /* default value */
+		modeDict[ApplyToAll],     /* other value */
+	),
 	modeDict,
 )