Merge cockroachdb#119372 cockroachdb#119968

119372: build: generate JS protos library r=koorosh a=koorosh

This change defines a new NPM package (in `ui/workspaces/crdb-js-proto`) with generated JS protobuf client code as a substitute to existing ``@cockroachlabs/crdb-protobuf-client`` which is not tree-shakable.
JS protobuf client code with help of `protoc` compiler and `protoc-gen-es` plugin.
It is supposed that this package will be consumed by CC Console.

Command to run builds:
```bash
 bazel build //pkg/ui/workspaces/crdb-js-proto
```

Resolves: https://cockroachlabs.atlassian.net/browse/CC-26921

Questions to answer:
- [ ] should it be a new NPM package (or replace existing ``@cockroachlabs/crdb-protobuf-client`` which we used to publish from `pkg/ui/workspaces/db-console/src/js`)?
- [ ] name of a package (`crdb-js-proto`) is not discussed yet, open for other suggestions.

Release note: None

119968: storage/fs: move StickyVFSRegistry r=RaduBerinde a=jbowens

Move the StickyVFSRegistry from the server package into the storage/fs package. This is in preparation for a refactor that moves the initialization of the virtual filesystems of all stores into the storage/fs package and performs the initialization before opening any of the store storage engines.

Release note: none
Epic: none

Co-authored-by: Andrii Vorobiov <[email protected]>
Co-authored-by: Jackson Owens <[email protected]>

.bazelignore

@@ -10,5 +10,6 @@ pkg/ui/workspaces/db-console/src/js/node_modules
 pkg/ui/workspaces/db-console/ccl/src/js/node_modules
 pkg/ui/workspaces/e2e-tests/node_modules
 pkg/ui/workspaces/eslint-plugin-crdb/node_modules
+pkg/ui/workspaces/crdb-api-client/node_modules
 tmp
 vendor

BUILD.bazel

@@ -102,6 +102,7 @@ exports_files([
 # gazelle:exclude pkg/ui/workspaces/db-console/src/js/node_modules
 # gazelle:exclude pkg/ui/workspaces/e2e-tests/node_modules
 # gazelle:exclude pkg/ui/workspaces/eslint-plugin-crdb/node_modules
+# gazelle:exclude pkg/ui/workspaces/crdb-api-client/node_modules
 # gazelle:exclude vendor
 # gazelle:exclude .vendor.tmp.*
 # gazelle:exclude **/zcgo_flags.go

WORKSPACE

@@ -275,6 +275,7 @@ npm_translate_lock(
         "//pkg/ui:pnpm-workspace.yaml",
         "//pkg/ui/patches:[email protected]",
         "//pkg/ui/workspaces/cluster-ui:package.json",
+        "//pkg/ui/workspaces/crdb-api-client:package.json",
         "//pkg/ui/workspaces/db-console:package.json",
         "//pkg/ui/workspaces/db-console/src/js:package.json",
         "//pkg/ui/workspaces/e2e-tests:package.json",

docs/generated/http/BUILD.bazel

@@ -1,64 +1,8 @@
+load(":defs.bzl", "PROTOBUF_TARGETS")
+
 genrule(
     name = "http",
-    srcs = [
-        "//pkg/build:build_proto",
-        "//pkg/clusterversion:clusterversion_proto",
-        "//pkg/config/zonepb:zonepb_proto",
-        "//pkg/geo/geopb:geopb_proto",
-        "//pkg/gossip:gossip_proto",
-        "//pkg/jobs/jobspb:jobspb_proto",
-        "//pkg/kv/kvpb:kvpb_proto",
-        "//pkg/kv/kvserver/concurrency/isolation:isolation_proto",
-        "//pkg/kv/kvserver/concurrency/lock:lock_proto",
-        "//pkg/kv/kvserver/kvserverpb:kvserverpb_proto",
-        "//pkg/kv/kvserver/liveness/livenesspb:livenesspb_proto",
-        "//pkg/kv/kvserver/loqrecovery/loqrecoverypb:loqrecoverypb_proto",
-        "//pkg/kv/kvserver/readsummary/rspb:rspb_proto",
-        "//pkg/multitenant/mtinfopb:mtinfopb_proto",
-        "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb:tenantcapabilitiespb_proto",
-        "//pkg/roachpb:roachpb_proto",
-        "//pkg/rpc/rpcpb:rpcpb_proto",
-        "//pkg/server/diagnostics/diagnosticspb:diagnosticspb_proto",
-        "//pkg/server/serverpb:serverpb_proto",
-        "//pkg/server/status/statuspb:statuspb_proto",
-        "//pkg/settings:settings_proto",
-        "//pkg/sql/appstatspb:appstatspb_proto",
-        "//pkg/sql/catalog/catenumpb:catenumpb_proto",
-        "//pkg/sql/catalog/catpb:catpb_proto",
-        "//pkg/sql/catalog/descpb:descpb_proto",
-        "//pkg/sql/catalog/fetchpb:fetchpb_proto",
-        "//pkg/sql/contentionpb:contentionpb_proto",
-        "//pkg/sql/execinfrapb:execinfrapb_proto",
-        "//pkg/sql/sem/semenumpb:semenumpb_proto",
-        "//pkg/sql/inverted:inverted_proto",
-        "//pkg/sql/lex:lex_proto",
-        "//pkg/sql/pgwire/pgerror:pgerror_proto",
-        "//pkg/sql/schemachanger/scpb:scpb_proto",
-        "//pkg/sql/sessiondatapb:sessiondatapb_proto",
-        "//pkg/sql/sqlstats/insights:insights_proto",
-        "//pkg/sql/types:types_proto",
-        "//pkg/storage/enginepb:enginepb_proto",
-        "//pkg/ts/catalog:catalog_proto",
-        "//pkg/ts/tspb:tspb_proto",
-        "//pkg/util/duration:duration_proto",
-        "//pkg/util/hlc:hlc_proto",
-        "//pkg/util/admission/admissionpb:admissionpb_proto",
-        "//pkg/util/log/logpb:logpb_proto",
-        "//pkg/util/metric:metric_proto",
-        "//pkg/util/timeutil/pgdate:pgdate_proto",
-        "//pkg/util/tracing/tracingpb:tracingpb_proto",
-        "//pkg/util:util_proto",
-        "@com_github_prometheus_client_model//io/prometheus/client:io_prometheus_client_proto",
-        "@com_github_cockroachdb_errors//errorspb:errorspb_proto",
-        "@com_github_gogo_protobuf//gogoproto:gogo_proto",
-        "@com_google_protobuf//:any_proto",
-        "@com_google_protobuf//:descriptor_proto",
-        "@com_google_protobuf//:duration_proto",
-        "@com_google_protobuf//:timestamp_proto",
-        "@go_googleapis//google/api:annotations_proto",
-        "@go_googleapis//google/api:http_proto",
-        "@io_etcd_go_raft_v3//raftpb:raftpb_proto",
-    ],
+    srcs = PROTOBUF_TARGETS,
     outs = [
         "full.md",
         "health-other.md",

docs/generated/http/defs.bzl

@@ -0,0 +1,62 @@
+# PROTOBUF_TARGETS lists the indirect dependencies needed to compile proto dependencies
+# of //pkg/server/serverpb:serverpb_proto target
+PROTOBUF_TARGETS = [
+    "//pkg/build:build_proto",
+    "//pkg/clusterversion:clusterversion_proto",
+    "//pkg/config/zonepb:zonepb_proto",
+    "//pkg/geo/geopb:geopb_proto",
+    "//pkg/gossip:gossip_proto",
+    "//pkg/jobs/jobspb:jobspb_proto",
+    "//pkg/kv/kvpb:kvpb_proto",
+    "//pkg/kv/kvserver/concurrency/isolation:isolation_proto",
+    "//pkg/kv/kvserver/concurrency/lock:lock_proto",
+    "//pkg/kv/kvserver/kvserverpb:kvserverpb_proto",
+    "//pkg/kv/kvserver/liveness/livenesspb:livenesspb_proto",
+    "//pkg/kv/kvserver/loqrecovery/loqrecoverypb:loqrecoverypb_proto",
+    "//pkg/kv/kvserver/readsummary/rspb:rspb_proto",
+    "//pkg/kv/kvserver/kvflowcontrol/kvflowcontrolpb:kvflowcontrolpb_proto",
+    "//pkg/multitenant/mtinfopb:mtinfopb_proto",
+    "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb:tenantcapabilitiespb_proto",
+    "//pkg/roachpb:roachpb_proto",
+    "//pkg/rpc/rpcpb:rpcpb_proto",
+    "//pkg/server/diagnostics/diagnosticspb:diagnosticspb_proto",
+    "//pkg/server/serverpb:serverpb_proto",
+    "//pkg/server/status/statuspb:statuspb_proto",
+    "//pkg/settings:settings_proto",
+    "//pkg/sql/appstatspb:appstatspb_proto",
+    "//pkg/sql/catalog/catenumpb:catenumpb_proto",
+    "//pkg/sql/catalog/catpb:catpb_proto",
+    "//pkg/sql/catalog/descpb:descpb_proto",
+    "//pkg/sql/catalog/fetchpb:fetchpb_proto",
+    "//pkg/sql/contentionpb:contentionpb_proto",
+    "//pkg/sql/execinfrapb:execinfrapb_proto",
+    "//pkg/sql/sem/semenumpb:semenumpb_proto",
+    "//pkg/sql/inverted:inverted_proto",
+    "//pkg/sql/lex:lex_proto",
+    "//pkg/sql/pgwire/pgerror:pgerror_proto",
+    "//pkg/sql/schemachanger/scpb:scpb_proto",
+    "//pkg/sql/sessiondatapb:sessiondatapb_proto",
+    "//pkg/sql/sqlstats/insights:insights_proto",
+    "//pkg/sql/types:types_proto",
+    "//pkg/storage/enginepb:enginepb_proto",
+    "//pkg/ts/catalog:catalog_proto",
+    "//pkg/ts/tspb:tspb_proto",
+    "//pkg/util/duration:duration_proto",
+    "//pkg/util/hlc:hlc_proto",
+    "//pkg/util/admission/admissionpb:admissionpb_proto",
+    "//pkg/util/log/logpb:logpb_proto",
+    "//pkg/util/metric:metric_proto",
+    "//pkg/util/timeutil/pgdate:pgdate_proto",
+    "//pkg/util/tracing/tracingpb:tracingpb_proto",
+    "//pkg/util:util_proto",
+    "@com_github_prometheus_client_model//io/prometheus/client:io_prometheus_client_proto",
+    "@com_github_cockroachdb_errors//errorspb:errorspb_proto",
+    "@com_github_gogo_protobuf//gogoproto:gogo_proto",
+    "@com_google_protobuf//:any_proto",
+    "@com_google_protobuf//:descriptor_proto",
+    "@com_google_protobuf//:duration_proto",
+    "@com_google_protobuf//:timestamp_proto",
+    "@go_googleapis//google/api:annotations_proto",
+    "@go_googleapis//google/api:http_proto",
+    "@io_etcd_go_raft_v3//raftpb:raftpb_proto",
+]

pkg/cli/BUILD.bazel

@@ -401,6 +401,7 @@ go_test(
         "//pkg/sql/protoreflect",
         "//pkg/sql/sem/catconstants",
         "//pkg/storage",
+        "//pkg/storage/fs",
         "//pkg/testutils",
         "//pkg/testutils/datapathutils",
         "//pkg/testutils/jobutils",

pkg/cli/debug_recover_loss_of_quorum_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
+	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/listenerutil"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
@@ -345,7 +346,7 @@ func TestStageVersionCheck(t *testing.T) {
 	listenerReg := listenerutil.NewListenerRegistry()
 	defer listenerReg.Close()
 
-	storeReg := server.NewStickyVFSRegistry()
+	storeReg := fs.NewStickyRegistry()
 	tc := testcluster.NewTestCluster(t, 4, base.TestClusterArgs{
 		ServerArgs: base.TestServerArgs{
 			// This logic is specific to the storage layer.
@@ -468,7 +469,7 @@ func TestHalfOnlineLossOfQuorumRecovery(t *testing.T) {
 		sa[i] = base.TestServerArgs{
 			Knobs: base.TestingKnobs{
 				Server: &server.TestingKnobs{
-					StickyVFSRegistry: server.NewStickyVFSRegistry(),
+					StickyVFSRegistry: fs.NewStickyRegistry(),
 				},
 			},
 			StoreSpecs: []base.StoreSpec{

pkg/cli/democluster/BUILD.bazel

@@ -38,6 +38,7 @@ go_library(
         "//pkg/sql/isql",
         "//pkg/sql/sem/catconstants",
         "//pkg/sql/sqlclustersettings",
+        "//pkg/storage/fs",
         "//pkg/testutils/serverutils",
         "//pkg/testutils/serverutils/regionlatency",
         "//pkg/util/envutil",
@@ -79,6 +80,7 @@ go_test(
         "//pkg/security/securitytest",
         "//pkg/server",
         "//pkg/sql/sqlclustersettings",
+        "//pkg/storage/fs",
         "//pkg/testutils",
         "//pkg/testutils/serverutils/regionlatency",
         "//pkg/testutils/skip",

pkg/cli/democluster/demo_cluster.go

@@ -44,6 +44,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/isql"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/catconstants"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlclustersettings"
+	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils/regionlatency"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
@@ -89,7 +90,7 @@ type transientCluster struct {
 	adminPassword string
 	adminUser     username.SQLUsername
 
-	stickyVFSRegistry server.StickyVFSRegistry
+	stickyVFSRegistry fs.StickyRegistry
 
 	drainAndShutdown func(ctx context.Context, adminClient serverpb.AdminClient) error
 
@@ -212,7 +213,7 @@ func NewDemoCluster(
 		}
 	}
 
-	c.stickyVFSRegistry = server.NewStickyVFSRegistry()
+	c.stickyVFSRegistry = fs.NewStickyRegistry()
 	return c, nil
 }
 
@@ -908,7 +909,7 @@ func (demoCtx *Context) testServerArgsForTransientCluster(
 	serverIdx int,
 	joinAddr string,
 	demoDir string,
-	stickyVFSRegistry server.StickyVFSRegistry,
+	stickyVFSRegistry fs.StickyRegistry,
 ) base.TestServerArgs {
 	// Assign a path to the store spec, to be saved.
 	storeSpec := base.DefaultTestStoreSpec

pkg/cli/democluster/demo_cluster_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/security/securitytest"
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlclustersettings"
+	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils/regionlatency"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
@@ -54,7 +55,7 @@ func TestTestServerArgsForTransientCluster(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
-	stickyVFSRegistry := server.NewStickyVFSRegistry()
+	stickyVFSRegistry := fs.NewStickyRegistry()
 
 	testCases := []struct {
 		serverIdx         int
@@ -179,7 +180,7 @@ func TestTransientClusterSimulateLatencies(t *testing.T) {
 		demoCtx:           demoCtx,
 		stopper:           stop.NewStopper(),
 		demoDir:           certsDir,
-		stickyVFSRegistry: server.NewStickyVFSRegistry(),
+		stickyVFSRegistry: fs.NewStickyRegistry(),
 		infoLog:           log.Infof,
 		warnLog:           log.Warningf,
 		shoutLog:          log.Ops.Shoutf,
@@ -291,7 +292,7 @@ func TestTransientClusterMultitenant(t *testing.T) {
 		demoCtx:           demoCtx,
 		stopper:           stop.NewStopper(),
 		demoDir:           certsDir,
-		stickyVFSRegistry: server.NewStickyVFSRegistry(),
+		stickyVFSRegistry: fs.NewStickyRegistry(),
 		infoLog:           log.Infof,
 		warnLog:           log.Warningf,
 		shoutLog:          log.Ops.Shoutf,
@@ -355,7 +356,7 @@ func TestTenantCapabilities(t *testing.T) {
 		demoCtx:           demoCtx,
 		stopper:           stop.NewStopper(),
 		demoDir:           certsDir,
-		stickyVFSRegistry: server.NewStickyVFSRegistry(),
+		stickyVFSRegistry: fs.NewStickyRegistry(),
 		infoLog:           log.Infof,
 		warnLog:           log.Warningf,
 		shoutLog:          log.Ops.Shoutf,

pkg/kv/kvserver/client_lease_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/bootstrap"
+	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
@@ -819,7 +820,7 @@ func TestLeasePreferencesRebalance(t *testing.T) {
 func TestLeaseholderRelocate(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
-	stickyRegistry := server.NewStickyVFSRegistry()
+	stickyRegistry := fs.NewStickyRegistry()
 	ctx := context.Background()
 	manualClock := hlc.NewHybridManualClock()
 
@@ -963,7 +964,7 @@ func TestLeasePreferencesDuringOutage(t *testing.T) {
 	// out heartbeating their liveness.
 	skip.UnderStressRace(t)
 
-	stickyRegistry := server.NewStickyVFSRegistry()
+	stickyRegistry := fs.NewStickyRegistry()
 	ctx := context.Background()
 	manualClock := hlc.NewHybridManualClock()
 	// Place all the leases in the us.
@@ -1201,7 +1202,7 @@ func TestLeasesDontThrashWhenNodeBecomesSuspect(t *testing.T) {
 	kvserver.ExpirationLeasesOnly.Override(ctx, &st.SV, false) // override metamorphism
 
 	// Speed up lease transfers.
-	stickyRegistry := server.NewStickyVFSRegistry()
+	stickyRegistry := fs.NewStickyRegistry()
 	manualClock := hlc.NewHybridManualClock()
 	serverArgs := make(map[int]base.TestServerArgs)
 	numNodes := 4

pkg/kv/kvserver/client_metrics_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/storage"
+	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
@@ -241,7 +242,7 @@ func TestStoreMetrics(t *testing.T) {
 
 	ctx := context.Background()
 	const numServers int = 3
-	stickyVFSRegistry := server.NewStickyVFSRegistry()
+	stickyVFSRegistry := fs.NewStickyRegistry()
 	stickyServerArgs := make(map[int]base.TestServerArgs)
 	locs := make(map[int]storage.Location)
 	for i := 0; i < numServers; i++ {

pkg/kv/kvserver/client_raft_log_queue_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/server"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/storage"
+	"github.com/cockroachdb/cockroach/pkg/storage/fs"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/listenerutil"
 	"github.com/cockroachdb/cockroach/pkg/testutils/storageutils"
@@ -170,7 +171,7 @@ func TestCrashWhileTruncatingSideloadedEntries(t *testing.T) {
 
 	// Use sticky engine registry to "survive" a node restart. Use the strict
 	// in-memory engine to be able to stop flushes and emulate data loss.
-	vfsReg := server.NewStickyVFSRegistry(server.UseStrictMemFS)
+	vfsReg := fs.NewStickyRegistry(fs.UseStrictMemFS)
 	// Use the sticky listener registry so that server port assignments survive
 	// node restarts, and don't get erroneously used by other clusters.
 	netReg := listenerutil.NewListenerRegistry()