diff --git a/.bazelignore b/.bazelignore index 9bc1218c91f8..0dcef45291d9 100644 --- a/.bazelignore +++ b/.bazelignore @@ -10,5 +10,6 @@ pkg/ui/workspaces/db-console/src/js/node_modules pkg/ui/workspaces/db-console/ccl/src/js/node_modules pkg/ui/workspaces/e2e-tests/node_modules pkg/ui/workspaces/eslint-plugin-crdb/node_modules +pkg/ui/workspaces/crdb-api-client/node_modules tmp vendor diff --git a/BUILD.bazel b/BUILD.bazel index 458e0816c8e6..3ed6c5a6797a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -102,6 +102,7 @@ exports_files([ # gazelle:exclude pkg/ui/workspaces/db-console/src/js/node_modules # gazelle:exclude pkg/ui/workspaces/e2e-tests/node_modules # gazelle:exclude pkg/ui/workspaces/eslint-plugin-crdb/node_modules +# gazelle:exclude pkg/ui/workspaces/crdb-api-client/node_modules # gazelle:exclude vendor # gazelle:exclude .vendor.tmp.* # gazelle:exclude **/zcgo_flags.go diff --git a/WORKSPACE b/WORKSPACE index c83cb769e364..952054c833bf 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -275,6 +275,7 @@ npm_translate_lock( "//pkg/ui:pnpm-workspace.yaml", "//pkg/ui/patches:topojson@3.0.2.patch", "//pkg/ui/workspaces/cluster-ui:package.json", + "//pkg/ui/workspaces/crdb-api-client:package.json", "//pkg/ui/workspaces/db-console:package.json", "//pkg/ui/workspaces/db-console/src/js:package.json", "//pkg/ui/workspaces/e2e-tests:package.json", diff --git a/docs/generated/http/BUILD.bazel b/docs/generated/http/BUILD.bazel index e92109b20bbf..5ea9c1a990b6 100644 --- a/docs/generated/http/BUILD.bazel +++ b/docs/generated/http/BUILD.bazel @@ -1,64 +1,8 @@ +load(":defs.bzl", "PROTOBUF_TARGETS") + genrule( name = "http", - srcs = [ - "//pkg/build:build_proto", - "//pkg/clusterversion:clusterversion_proto", - "//pkg/config/zonepb:zonepb_proto", - "//pkg/geo/geopb:geopb_proto", - "//pkg/gossip:gossip_proto", - "//pkg/jobs/jobspb:jobspb_proto", - "//pkg/kv/kvpb:kvpb_proto", - "//pkg/kv/kvserver/concurrency/isolation:isolation_proto", - "//pkg/kv/kvserver/concurrency/lock:lock_proto", - "//pkg/kv/kvserver/kvserverpb:kvserverpb_proto", - "//pkg/kv/kvserver/liveness/livenesspb:livenesspb_proto", - "//pkg/kv/kvserver/loqrecovery/loqrecoverypb:loqrecoverypb_proto", - "//pkg/kv/kvserver/readsummary/rspb:rspb_proto", - "//pkg/multitenant/mtinfopb:mtinfopb_proto", - "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb:tenantcapabilitiespb_proto", - "//pkg/roachpb:roachpb_proto", - "//pkg/rpc/rpcpb:rpcpb_proto", - "//pkg/server/diagnostics/diagnosticspb:diagnosticspb_proto", - "//pkg/server/serverpb:serverpb_proto", - "//pkg/server/status/statuspb:statuspb_proto", - "//pkg/settings:settings_proto", - "//pkg/sql/appstatspb:appstatspb_proto", - "//pkg/sql/catalog/catenumpb:catenumpb_proto", - "//pkg/sql/catalog/catpb:catpb_proto", - "//pkg/sql/catalog/descpb:descpb_proto", - "//pkg/sql/catalog/fetchpb:fetchpb_proto", - "//pkg/sql/contentionpb:contentionpb_proto", - "//pkg/sql/execinfrapb:execinfrapb_proto", - "//pkg/sql/sem/semenumpb:semenumpb_proto", - "//pkg/sql/inverted:inverted_proto", - "//pkg/sql/lex:lex_proto", - "//pkg/sql/pgwire/pgerror:pgerror_proto", - "//pkg/sql/schemachanger/scpb:scpb_proto", - "//pkg/sql/sessiondatapb:sessiondatapb_proto", - "//pkg/sql/sqlstats/insights:insights_proto", - "//pkg/sql/types:types_proto", - "//pkg/storage/enginepb:enginepb_proto", - "//pkg/ts/catalog:catalog_proto", - "//pkg/ts/tspb:tspb_proto", - "//pkg/util/duration:duration_proto", - "//pkg/util/hlc:hlc_proto", - "//pkg/util/admission/admissionpb:admissionpb_proto", - "//pkg/util/log/logpb:logpb_proto", - "//pkg/util/metric:metric_proto", - "//pkg/util/timeutil/pgdate:pgdate_proto", - "//pkg/util/tracing/tracingpb:tracingpb_proto", - "//pkg/util:util_proto", - "@com_github_prometheus_client_model//io/prometheus/client:io_prometheus_client_proto", - "@com_github_cockroachdb_errors//errorspb:errorspb_proto", - "@com_github_gogo_protobuf//gogoproto:gogo_proto", - "@com_google_protobuf//:any_proto", - "@com_google_protobuf//:descriptor_proto", - "@com_google_protobuf//:duration_proto", - "@com_google_protobuf//:timestamp_proto", - "@go_googleapis//google/api:annotations_proto", - "@go_googleapis//google/api:http_proto", - "@io_etcd_go_raft_v3//raftpb:raftpb_proto", - ], + srcs = PROTOBUF_TARGETS, outs = [ "full.md", "health-other.md", diff --git a/docs/generated/http/defs.bzl b/docs/generated/http/defs.bzl new file mode 100644 index 000000000000..0ca7059c2d1f --- /dev/null +++ b/docs/generated/http/defs.bzl @@ -0,0 +1,62 @@ +# PROTOBUF_TARGETS lists the indirect dependencies needed to compile proto dependencies +# of //pkg/server/serverpb:serverpb_proto target +PROTOBUF_TARGETS = [ + "//pkg/build:build_proto", + "//pkg/clusterversion:clusterversion_proto", + "//pkg/config/zonepb:zonepb_proto", + "//pkg/geo/geopb:geopb_proto", + "//pkg/gossip:gossip_proto", + "//pkg/jobs/jobspb:jobspb_proto", + "//pkg/kv/kvpb:kvpb_proto", + "//pkg/kv/kvserver/concurrency/isolation:isolation_proto", + "//pkg/kv/kvserver/concurrency/lock:lock_proto", + "//pkg/kv/kvserver/kvserverpb:kvserverpb_proto", + "//pkg/kv/kvserver/liveness/livenesspb:livenesspb_proto", + "//pkg/kv/kvserver/loqrecovery/loqrecoverypb:loqrecoverypb_proto", + "//pkg/kv/kvserver/readsummary/rspb:rspb_proto", + "//pkg/kv/kvserver/kvflowcontrol/kvflowcontrolpb:kvflowcontrolpb_proto", + "//pkg/multitenant/mtinfopb:mtinfopb_proto", + "//pkg/multitenant/tenantcapabilities/tenantcapabilitiespb:tenantcapabilitiespb_proto", + "//pkg/roachpb:roachpb_proto", + "//pkg/rpc/rpcpb:rpcpb_proto", + "//pkg/server/diagnostics/diagnosticspb:diagnosticspb_proto", + "//pkg/server/serverpb:serverpb_proto", + "//pkg/server/status/statuspb:statuspb_proto", + "//pkg/settings:settings_proto", + "//pkg/sql/appstatspb:appstatspb_proto", + "//pkg/sql/catalog/catenumpb:catenumpb_proto", + "//pkg/sql/catalog/catpb:catpb_proto", + "//pkg/sql/catalog/descpb:descpb_proto", + "//pkg/sql/catalog/fetchpb:fetchpb_proto", + "//pkg/sql/contentionpb:contentionpb_proto", + "//pkg/sql/execinfrapb:execinfrapb_proto", + "//pkg/sql/sem/semenumpb:semenumpb_proto", + "//pkg/sql/inverted:inverted_proto", + "//pkg/sql/lex:lex_proto", + "//pkg/sql/pgwire/pgerror:pgerror_proto", + "//pkg/sql/schemachanger/scpb:scpb_proto", + "//pkg/sql/sessiondatapb:sessiondatapb_proto", + "//pkg/sql/sqlstats/insights:insights_proto", + "//pkg/sql/types:types_proto", + "//pkg/storage/enginepb:enginepb_proto", + "//pkg/ts/catalog:catalog_proto", + "//pkg/ts/tspb:tspb_proto", + "//pkg/util/duration:duration_proto", + "//pkg/util/hlc:hlc_proto", + "//pkg/util/admission/admissionpb:admissionpb_proto", + "//pkg/util/log/logpb:logpb_proto", + "//pkg/util/metric:metric_proto", + "//pkg/util/timeutil/pgdate:pgdate_proto", + "//pkg/util/tracing/tracingpb:tracingpb_proto", + "//pkg/util:util_proto", + "@com_github_prometheus_client_model//io/prometheus/client:io_prometheus_client_proto", + "@com_github_cockroachdb_errors//errorspb:errorspb_proto", + "@com_github_gogo_protobuf//gogoproto:gogo_proto", + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:timestamp_proto", + "@go_googleapis//google/api:annotations_proto", + "@go_googleapis//google/api:http_proto", + "@io_etcd_go_raft_v3//raftpb:raftpb_proto", +] diff --git a/pkg/cli/BUILD.bazel b/pkg/cli/BUILD.bazel index 46d2b90000f0..dc37a8a6a442 100644 --- a/pkg/cli/BUILD.bazel +++ b/pkg/cli/BUILD.bazel @@ -401,6 +401,7 @@ go_test( "//pkg/sql/protoreflect", "//pkg/sql/sem/catconstants", "//pkg/storage", + "//pkg/storage/fs", "//pkg/testutils", "//pkg/testutils/datapathutils", "//pkg/testutils/jobutils", diff --git a/pkg/cli/debug_recover_loss_of_quorum_test.go b/pkg/cli/debug_recover_loss_of_quorum_test.go index 71e9c869a5a2..34d3088c94f6 100644 --- a/pkg/cli/debug_recover_loss_of_quorum_test.go +++ b/pkg/cli/debug_recover_loss_of_quorum_test.go @@ -28,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/server/serverpb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -345,7 +346,7 @@ func TestStageVersionCheck(t *testing.T) { listenerReg := listenerutil.NewListenerRegistry() defer listenerReg.Close() - storeReg := server.NewStickyVFSRegistry() + storeReg := fs.NewStickyRegistry() tc := testcluster.NewTestCluster(t, 4, base.TestClusterArgs{ ServerArgs: base.TestServerArgs{ // This logic is specific to the storage layer. @@ -468,7 +469,7 @@ func TestHalfOnlineLossOfQuorumRecovery(t *testing.T) { sa[i] = base.TestServerArgs{ Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, }, StoreSpecs: []base.StoreSpec{ diff --git a/pkg/cli/democluster/BUILD.bazel b/pkg/cli/democluster/BUILD.bazel index 93a611cafa3a..4396cc551216 100644 --- a/pkg/cli/democluster/BUILD.bazel +++ b/pkg/cli/democluster/BUILD.bazel @@ -38,6 +38,7 @@ go_library( "//pkg/sql/isql", "//pkg/sql/sem/catconstants", "//pkg/sql/sqlclustersettings", + "//pkg/storage/fs", "//pkg/testutils/serverutils", "//pkg/testutils/serverutils/regionlatency", "//pkg/util/envutil", @@ -79,6 +80,7 @@ go_test( "//pkg/security/securitytest", "//pkg/server", "//pkg/sql/sqlclustersettings", + "//pkg/storage/fs", "//pkg/testutils", "//pkg/testutils/serverutils/regionlatency", "//pkg/testutils/skip", diff --git a/pkg/cli/democluster/demo_cluster.go b/pkg/cli/democluster/demo_cluster.go index 54501229eeae..6acc1b42e000 100644 --- a/pkg/cli/democluster/demo_cluster.go +++ b/pkg/cli/democluster/demo_cluster.go @@ -44,6 +44,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/isql" "github.com/cockroachdb/cockroach/pkg/sql/sem/catconstants" "github.com/cockroachdb/cockroach/pkg/sql/sqlclustersettings" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils/regionlatency" "github.com/cockroachdb/cockroach/pkg/util/envutil" @@ -89,7 +90,7 @@ type transientCluster struct { adminPassword string adminUser username.SQLUsername - stickyVFSRegistry server.StickyVFSRegistry + stickyVFSRegistry fs.StickyRegistry drainAndShutdown func(ctx context.Context, adminClient serverpb.AdminClient) error @@ -212,7 +213,7 @@ func NewDemoCluster( } } - c.stickyVFSRegistry = server.NewStickyVFSRegistry() + c.stickyVFSRegistry = fs.NewStickyRegistry() return c, nil } @@ -908,7 +909,7 @@ func (demoCtx *Context) testServerArgsForTransientCluster( serverIdx int, joinAddr string, demoDir string, - stickyVFSRegistry server.StickyVFSRegistry, + stickyVFSRegistry fs.StickyRegistry, ) base.TestServerArgs { // Assign a path to the store spec, to be saved. storeSpec := base.DefaultTestStoreSpec diff --git a/pkg/cli/democluster/demo_cluster_test.go b/pkg/cli/democluster/demo_cluster_test.go index fb1dfad720df..6b334923b468 100644 --- a/pkg/cli/democluster/demo_cluster_test.go +++ b/pkg/cli/democluster/demo_cluster_test.go @@ -28,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/security/securitytest" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/sql/sqlclustersettings" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils/regionlatency" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -54,7 +55,7 @@ func TestTestServerArgsForTransientCluster(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() testCases := []struct { serverIdx int @@ -179,7 +180,7 @@ func TestTransientClusterSimulateLatencies(t *testing.T) { demoCtx: demoCtx, stopper: stop.NewStopper(), demoDir: certsDir, - stickyVFSRegistry: server.NewStickyVFSRegistry(), + stickyVFSRegistry: fs.NewStickyRegistry(), infoLog: log.Infof, warnLog: log.Warningf, shoutLog: log.Ops.Shoutf, @@ -291,7 +292,7 @@ func TestTransientClusterMultitenant(t *testing.T) { demoCtx: demoCtx, stopper: stop.NewStopper(), demoDir: certsDir, - stickyVFSRegistry: server.NewStickyVFSRegistry(), + stickyVFSRegistry: fs.NewStickyRegistry(), infoLog: log.Infof, warnLog: log.Warningf, shoutLog: log.Ops.Shoutf, @@ -355,7 +356,7 @@ func TestTenantCapabilities(t *testing.T) { demoCtx: demoCtx, stopper: stop.NewStopper(), demoDir: certsDir, - stickyVFSRegistry: server.NewStickyVFSRegistry(), + stickyVFSRegistry: fs.NewStickyRegistry(), infoLog: log.Infof, warnLog: log.Warningf, shoutLog: log.Ops.Shoutf, diff --git a/pkg/kv/kvserver/client_lease_test.go b/pkg/kv/kvserver/client_lease_test.go index 25a6cc5fac46..942024cec95d 100644 --- a/pkg/kv/kvserver/client_lease_test.go +++ b/pkg/kv/kvserver/client_lease_test.go @@ -35,6 +35,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql/catalog/bootstrap" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -819,7 +820,7 @@ func TestLeasePreferencesRebalance(t *testing.T) { func TestLeaseholderRelocate(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyRegistry := server.NewStickyVFSRegistry() + stickyRegistry := fs.NewStickyRegistry() ctx := context.Background() manualClock := hlc.NewHybridManualClock() @@ -963,7 +964,7 @@ func TestLeasePreferencesDuringOutage(t *testing.T) { // out heartbeating their liveness. skip.UnderStressRace(t) - stickyRegistry := server.NewStickyVFSRegistry() + stickyRegistry := fs.NewStickyRegistry() ctx := context.Background() manualClock := hlc.NewHybridManualClock() // Place all the leases in the us. @@ -1201,7 +1202,7 @@ func TestLeasesDontThrashWhenNodeBecomesSuspect(t *testing.T) { kvserver.ExpirationLeasesOnly.Override(ctx, &st.SV, false) // override metamorphism // Speed up lease transfers. - stickyRegistry := server.NewStickyVFSRegistry() + stickyRegistry := fs.NewStickyRegistry() manualClock := hlc.NewHybridManualClock() serverArgs := make(map[int]base.TestServerArgs) numNodes := 4 diff --git a/pkg/kv/kvserver/client_metrics_test.go b/pkg/kv/kvserver/client_metrics_test.go index 2a6d0335692f..9cd0933f41fa 100644 --- a/pkg/kv/kvserver/client_metrics_test.go +++ b/pkg/kv/kvserver/client_metrics_test.go @@ -26,6 +26,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -241,7 +242,7 @@ func TestStoreMetrics(t *testing.T) { ctx := context.Background() const numServers int = 3 - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() stickyServerArgs := make(map[int]base.TestServerArgs) locs := make(map[int]storage.Location) for i := 0; i < numServers; i++ { diff --git a/pkg/kv/kvserver/client_raft_log_queue_test.go b/pkg/kv/kvserver/client_raft_log_queue_test.go index 093db7d7bb41..727696b76e9f 100644 --- a/pkg/kv/kvserver/client_raft_log_queue_test.go +++ b/pkg/kv/kvserver/client_raft_log_queue_test.go @@ -31,6 +31,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/storageutils" @@ -170,7 +171,7 @@ func TestCrashWhileTruncatingSideloadedEntries(t *testing.T) { // Use sticky engine registry to "survive" a node restart. Use the strict // in-memory engine to be able to stop flushes and emulate data loss. - vfsReg := server.NewStickyVFSRegistry(server.UseStrictMemFS) + vfsReg := fs.NewStickyRegistry(fs.UseStrictMemFS) // Use the sticky listener registry so that server port assignments survive // node restarts, and don't get erroneously used by other clusters. netReg := listenerutil.NewListenerRegistry() diff --git a/pkg/kv/kvserver/client_raft_test.go b/pkg/kv/kvserver/client_raft_test.go index 72e0fee21bfe..726daa33f287 100644 --- a/pkg/kv/kvserver/client_raft_test.go +++ b/pkg/kv/kvserver/client_raft_test.go @@ -47,6 +47,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -112,7 +113,7 @@ func TestStoreRecoverFromEngine(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, }, }, @@ -202,7 +203,7 @@ func TestStoreRecoverWithErrors(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() @@ -347,7 +348,7 @@ func TestRestoreReplicas(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() @@ -684,7 +685,7 @@ func TestSnapshotAfterTruncation(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, }, } @@ -1728,7 +1729,7 @@ func TestConcurrentRaftSnapshots(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, }, } @@ -1811,7 +1812,7 @@ func TestReplicateAfterRemoveAndSplit(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, Store: &kvserver.StoreTestingKnobs{ // Disable the replica GC queue so that it doesn't accidentally pick up the @@ -1905,7 +1906,7 @@ func TestLogGrowthWhenRefreshingPendingCommands(t *testing.T) { RaftConfig: raftConfig, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, Store: &kvserver.StoreTestingKnobs{ // Disable leader transfers during leaseholder changes so that we @@ -2210,7 +2211,7 @@ func TestProgressWithDownNode(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, }, } @@ -2292,7 +2293,7 @@ func runReplicateRestartAfterTruncation(t *testing.T, removeBeforeTruncateAndReA }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), WallClock: manualClock, }, }, @@ -2395,7 +2396,7 @@ func testReplicaAddRemove(t *testing.T, addFirst bool) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), WallClock: manualClock, }, Store: &kvserver.StoreTestingKnobs{ @@ -3494,7 +3495,7 @@ func TestReplicateRogueRemovedNode(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), WallClock: manualClock, }, Store: &kvserver.StoreTestingKnobs{ @@ -3838,7 +3839,7 @@ func TestReplicaTooOldGC(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, Store: &kvserver.StoreTestingKnobs{ DisableScanner: true, @@ -3935,7 +3936,7 @@ func TestReplicateReAddAfterDown(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, }, } @@ -4802,7 +4803,7 @@ func TestDefaultConnectionDisruptionDoesNotInterfereWithSystemTraffic(t *testing defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() @@ -5211,7 +5212,7 @@ func TestProcessSplitAfterRightHandSideHasBeenRemoved(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, Store: &kvserver.StoreTestingKnobs{ // Newly-started stores (including the "rogue" one) should not GC @@ -5577,7 +5578,7 @@ func TestElectionAfterRestart(t *testing.T) { const electionTimeoutTicks = 30 const raftTickInterval = 200 * time.Millisecond - r := server.NewStickyVFSRegistry() + r := fs.NewStickyRegistry() newTCArgs := func(parallel bool, replMode base.TestClusterReplicationMode, onTimeoutCampaign func(roachpb.RangeID)) base.TestClusterArgs { return base.TestClusterArgs{ ReplicationMode: replMode, diff --git a/pkg/kv/kvserver/client_replica_circuit_breaker_test.go b/pkg/kv/kvserver/client_replica_circuit_breaker_test.go index ef2cf7511cb8..83b106ba7a3d 100644 --- a/pkg/kv/kvserver/client_replica_circuit_breaker_test.go +++ b/pkg/kv/kvserver/client_replica_circuit_breaker_test.go @@ -29,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" @@ -1012,7 +1013,7 @@ func setupCircuitBreakerTest(t *testing.T) *circuitBreakerTest { raftCfg.SetDefaults() raftCfg.RaftHeartbeatIntervalTicks = 1 raftCfg.RaftElectionTimeoutTicks = 2 - reg := server.NewStickyVFSRegistry() + reg := fs.NewStickyRegistry() args := base.TestClusterArgs{ ReplicationMode: base.ReplicationManual, ServerArgs: base.TestServerArgs{ diff --git a/pkg/kv/kvserver/client_replica_test.go b/pkg/kv/kvserver/client_replica_test.go index 75836ed1f51b..0a0a63aeb77d 100644 --- a/pkg/kv/kvserver/client_replica_test.go +++ b/pkg/kv/kvserver/client_replica_test.go @@ -52,6 +52,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/isql" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/kvclientutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" @@ -2160,7 +2161,7 @@ func TestLeaseNotUsedAfterRestart(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() @@ -4702,7 +4703,7 @@ func TestTenantID(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() ctx := context.Background() // Create a config with a sticky-in-mem engine so we can restart the server. // We also configure the settings to be as robust as possible to problems diff --git a/pkg/kv/kvserver/client_split_test.go b/pkg/kv/kvserver/client_split_test.go index 2a1e773bc25e..d09756e2ac3a 100644 --- a/pkg/kv/kvserver/client_split_test.go +++ b/pkg/kv/kvserver/client_split_test.go @@ -51,6 +51,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sqlstats" "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -1492,7 +1493,7 @@ func TestStoreRangeSplitBackpressureWrites(t *testing.T) { // See https://github.com/cockroachdb/cockroach/issues/1644. func runSetupSplitSnapshotRace( t *testing.T, - stickyVFSRegistry server.StickyVFSRegistry, + stickyVFSRegistry fs.StickyRegistry, testFn func(*testcluster.TestCluster, roachpb.Key, roachpb.Key), ) { const numServers int = 6 @@ -1643,7 +1644,7 @@ func TestSplitSnapshotRace_SplitWins(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - runSetupSplitSnapshotRace(t, server.NewStickyVFSRegistry(), func(tc *testcluster.TestCluster, leftKey, rightKey roachpb.Key) { + runSetupSplitSnapshotRace(t, fs.NewStickyRegistry(), func(tc *testcluster.TestCluster, leftKey, rightKey roachpb.Key) { // Bring the left range up first so that the split happens before it sees a snapshot. for i := 1; i <= 3; i++ { require.NoError(t, tc.RestartServer(i)) @@ -1677,7 +1678,7 @@ func TestSplitSnapshotRace_SnapshotWins(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - runSetupSplitSnapshotRace(t, server.NewStickyVFSRegistry(), func(tc *testcluster.TestCluster, leftKey, rightKey roachpb.Key) { + runSetupSplitSnapshotRace(t, fs.NewStickyRegistry(), func(tc *testcluster.TestCluster, leftKey, rightKey roachpb.Key) { // Bring the right range up first. for i := 3; i <= 5; i++ { require.NoError(t, tc.RestartServer(i)) diff --git a/pkg/kv/kvserver/client_store_test.go b/pkg/kv/kvserver/client_store_test.go index daa1404c9ffb..a5ddef3d0ffa 100644 --- a/pkg/kv/kvserver/client_store_test.go +++ b/pkg/kv/kvserver/client_store_test.go @@ -22,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -119,7 +120,7 @@ func TestStoreLoadReplicaQuiescent(t *testing.T) { defer log.Scope(t).Close(t) testutils.RunTrueAndFalse(t, "kv.expiration_leases_only.enabled", func(t *testing.T, expOnly bool) { - storeReg := server.NewStickyVFSRegistry() + storeReg := fs.NewStickyRegistry() listenerReg := listenerutil.NewListenerRegistry() defer listenerReg.Close() diff --git a/pkg/kv/kvserver/consistency_queue_test.go b/pkg/kv/kvserver/consistency_queue_test.go index b1af1436f8a4..2b43c7ec0b1d 100644 --- a/pkg/kv/kvserver/consistency_queue_test.go +++ b/pkg/kv/kvserver/consistency_queue_test.go @@ -34,6 +34,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" "github.com/cockroachdb/cockroach/pkg/util/hlc" @@ -248,7 +249,7 @@ func TestCheckConsistencyInconsistent(t *testing.T) { // Test uses sticky registry to have persistent pebble state that could // be analyzed for existence of snapshots and to verify snapshot content // after failures. - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() // The cluster has 3 nodes, one store per node. The test writes a few KVs to a // range, which gets replicated to all 3 stores. Then it manually replaces an diff --git a/pkg/kv/kvserver/flow_control_integration_test.go b/pkg/kv/kvserver/flow_control_integration_test.go index 83c517e3a279..a1a97e42cb0f 100644 --- a/pkg/kv/kvserver/flow_control_integration_test.go +++ b/pkg/kv/kvserver/flow_control_integration_test.go @@ -29,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/datapathutils" "github.com/cockroachdb/cockroach/pkg/testutils/echotest" @@ -792,7 +793,7 @@ func TestFlowControlRaftSnapshot(t *testing.T) { }, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), }, Store: &kvserver.StoreTestingKnobs{ FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ diff --git a/pkg/kv/kvserver/loqrecovery/BUILD.bazel b/pkg/kv/kvserver/loqrecovery/BUILD.bazel index 9b2386be238a..376dd502de43 100644 --- a/pkg/kv/kvserver/loqrecovery/BUILD.bazel +++ b/pkg/kv/kvserver/loqrecovery/BUILD.bazel @@ -97,6 +97,7 @@ go_test( "//pkg/spanconfig", "//pkg/storage", "//pkg/storage/enginepb", + "//pkg/storage/fs", "//pkg/testutils", "//pkg/testutils/datapathutils", "//pkg/testutils/listenerutil", diff --git a/pkg/kv/kvserver/loqrecovery/server_integration_test.go b/pkg/kv/kvserver/loqrecovery/server_integration_test.go index 0d90b053b8f6..ab6256bb275a 100644 --- a/pkg/kv/kvserver/loqrecovery/server_integration_test.go +++ b/pkg/kv/kvserver/loqrecovery/server_integration_test.go @@ -29,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -666,10 +667,10 @@ func TestRejectBadVersionApplication(t *testing.T) { func prepTestCluster( t *testing.T, nodes int, -) (*testcluster.TestCluster, server.StickyVFSRegistry, map[int]loqrecovery.PlanStore) { +) (*testcluster.TestCluster, fs.StickyRegistry, map[int]loqrecovery.PlanStore) { skip.UnderStressRace(t, "cluster frequently fails to start under stress race") - reg := server.NewStickyVFSRegistry() + reg := fs.NewStickyRegistry() lReg := listenerutil.NewListenerRegistry() diff --git a/pkg/kv/kvserver/node_liveness_test.go b/pkg/kv/kvserver/node_liveness_test.go index 7dab48673506..90afbd1f2e78 100644 --- a/pkg/kv/kvserver/node_liveness_test.go +++ b/pkg/kv/kvserver/node_liveness_test.go @@ -35,6 +35,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/rpc/nodedialer" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" @@ -154,7 +155,7 @@ func TestNodeLivenessInitialIncrement(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() @@ -466,7 +467,7 @@ func TestNodeLivenessRestart(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() const numServers int = 2 stickyServerArgs := make(map[int]base.TestServerArgs) @@ -834,7 +835,7 @@ func TestNodeLivenessSetDraining(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() @@ -1165,7 +1166,7 @@ func verifyNodeIsDecommissioning(t *testing.T, tc *testcluster.TestCluster, node } func testNodeLivenessSetDecommissioning(t *testing.T, decommissionNodeIdx int) { - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() diff --git a/pkg/kv/kvserver/replicate_queue_test.go b/pkg/kv/kvserver/replicate_queue_test.go index 86314492334e..475f958aa52e 100644 --- a/pkg/kv/kvserver/replicate_queue_test.go +++ b/pkg/kv/kvserver/replicate_queue_test.go @@ -43,6 +43,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" "github.com/cockroachdb/cockroach/pkg/sql/catalog/bootstrap" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -2116,7 +2117,7 @@ func TestReplicateQueueAcquiresInvalidLeases(t *testing.T) { ScanMaxIdleTime: time.Millisecond, Knobs: base.TestingKnobs{ Server: &server.TestingKnobs{ - StickyVFSRegistry: server.NewStickyVFSRegistry(), + StickyVFSRegistry: fs.NewStickyRegistry(), DefaultZoneConfigOverride: &zcfg, }, }, diff --git a/pkg/server/BUILD.bazel b/pkg/server/BUILD.bazel index d1729400451b..f60a30fcc782 100644 --- a/pkg/server/BUILD.bazel +++ b/pkg/server/BUILD.bazel @@ -70,7 +70,6 @@ go_library( "statements.go", "status.go", "status_local_file_retrieval.go", - "sticky_vfs.go", "stop_trigger.go", "tcp_keepalive_manager.go", "tenant.go", @@ -459,7 +458,6 @@ go_test( "span_stats_test.go", "statements_test.go", "status_ext_test.go", - "sticky_vfs_test.go", "tcp_keepalive_manager_test.go", "tenant_delayed_id_set_test.go", "tenant_range_lookup_test.go", @@ -523,6 +521,7 @@ go_test( "//pkg/sql/sessiondata", "//pkg/sql/sqlstats", "//pkg/storage", + "//pkg/storage/fs", "//pkg/testutils", "//pkg/testutils/datapathutils", "//pkg/testutils/listenerutil", diff --git a/pkg/server/migration_test.go b/pkg/server/migration_test.go index a892dda8d13f..fed9bc6cbb43 100644 --- a/pkg/server/migration_test.go +++ b/pkg/server/migration_test.go @@ -25,6 +25,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -112,7 +113,7 @@ func TestSyncAllEngines(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - vfsRegistry := NewStickyVFSRegistry(UseStrictMemFS) + vfsRegistry := fs.NewStickyRegistry(fs.UseStrictMemFS) storeSpec := base.DefaultTestStoreSpec storeSpec.StickyVFSID = "sync-all-engines" testServerArgs := base.TestServerArgs{ diff --git a/pkg/server/multi_store_test.go b/pkg/server/multi_store_test.go index 596b232dd007..1b2f082e4916 100644 --- a/pkg/server/multi_store_test.go +++ b/pkg/server/multi_store_test.go @@ -20,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/kv/kvserver" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" @@ -46,7 +47,7 @@ func TestAddNewStoresToExistingNodes(t *testing.T) { ctx := context.Background() - ser := server.NewStickyVFSRegistry() + ser := fs.NewStickyRegistry() const ( numNodes = 3 diff --git a/pkg/server/server_startup_test.go b/pkg/server/server_startup_test.go index 91fef928954e..f5bc0175c4b1 100644 --- a/pkg/server/server_startup_test.go +++ b/pkg/server/server_startup_test.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/testcluster" "github.com/cockroachdb/cockroach/pkg/util/leaktest" @@ -40,7 +41,7 @@ func TestStartupInjectedFailureSingleNode(t *testing.T) { rng, seed := randutil.NewLockedTestRand() t.Log("TestStartupInjectedFailure random seed", seed) - reg := server.NewStickyVFSRegistry() + reg := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() diff --git a/pkg/server/settings_cache_test.go b/pkg/server/settings_cache_test.go index f1c85d01435c..95646bbfa84c 100644 --- a/pkg/server/settings_cache_test.go +++ b/pkg/server/settings_cache_test.go @@ -23,6 +23,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/sqlutils" @@ -67,7 +68,7 @@ func TestCachedSettingsServerRestart(t *testing.T) { defer log.Scope(t).Close(t) ctx := context.Background() - stickyVFSRegistry := NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() serverArgs := base.TestServerArgs{ DefaultTestTenant: base.TestIsSpecificToStorageLayerAndNeedsASystemTenant, diff --git a/pkg/server/testing_knobs.go b/pkg/server/testing_knobs.go index c5831fa7f945..dc076933936c 100644 --- a/pkg/server/testing_knobs.go +++ b/pkg/server/testing_knobs.go @@ -20,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/rpc" "github.com/cockroachdb/cockroach/pkg/server/diagnostics" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/hlc" ) @@ -103,7 +104,7 @@ type TestingKnobs struct { // // When supplied to a TestCluster, StickyVFSIDs will be associated auto- // matically to the StoreSpecs used. - StickyVFSRegistry StickyVFSRegistry + StickyVFSRegistry fs.StickyRegistry // WallClock is used to inject a custom clock for testing the server. It is // typically either an hlc.HybridManualClock or hlc.ManualClock. WallClock hlc.WallClock diff --git a/pkg/settings/integration_tests/BUILD.bazel b/pkg/settings/integration_tests/BUILD.bazel index ac9027e099fe..258004b52a07 100644 --- a/pkg/settings/integration_tests/BUILD.bazel +++ b/pkg/settings/integration_tests/BUILD.bazel @@ -14,6 +14,7 @@ go_test( "//pkg/security/securitytest", "//pkg/server", "//pkg/settings", + "//pkg/storage/fs", "//pkg/testutils", "//pkg/testutils/listenerutil", "//pkg/testutils/serverutils", diff --git a/pkg/settings/integration_tests/settings_test.go b/pkg/settings/integration_tests/settings_test.go index 88181b311835..8a7ddd5f11f4 100644 --- a/pkg/settings/integration_tests/settings_test.go +++ b/pkg/settings/integration_tests/settings_test.go @@ -19,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" @@ -303,7 +304,7 @@ func TestSettingsPersistenceEndToEnd(t *testing.T) { // We're going to restart the test server, but expecting storage to // persist. Define a sticky VFS for this purpose. - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() // We'll also need stable listeners to enable port reuse across restarts. lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() diff --git a/pkg/storage/fs/BUILD.bazel b/pkg/storage/fs/BUILD.bazel index 8e9c89ea9626..6284b39a3aaf 100644 --- a/pkg/storage/fs/BUILD.bazel +++ b/pkg/storage/fs/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "fs.go", "safewrite.go", + "sticky_vfs.go", "temp_dir.go", ], importpath = "github.com/cockroachdb/cockroach/pkg/storage/fs", @@ -12,6 +13,7 @@ go_library( deps = [ "//pkg/util/log", "//pkg/util/stop", + "//pkg/util/syncutil", "@com_github_cockroachdb_errors//:errors", "@com_github_cockroachdb_errors//oserror", "@com_github_cockroachdb_pebble//vfs", @@ -22,10 +24,15 @@ go_test( name = "fs_test", srcs = [ "safewrite_test.go", + "sticky_vfs_test.go", "temp_dir_test.go", ], embed = [":fs"], deps = [ + "//pkg/base", + "//pkg/roachpb", + "//pkg/settings/cluster", + "//pkg/storage", "//pkg/util/leaktest", "//pkg/util/log", "//pkg/util/stop", diff --git a/pkg/server/sticky_vfs.go b/pkg/storage/fs/sticky_vfs.go similarity index 62% rename from pkg/server/sticky_vfs.go rename to pkg/storage/fs/sticky_vfs.go index fe7ae186a773..cf3eafeac78c 100644 --- a/pkg/server/sticky_vfs.go +++ b/pkg/storage/fs/sticky_vfs.go @@ -8,47 +8,47 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package server +package fs import ( "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/pebble/vfs" ) -// StickyVFSOption is a config option for a sticky engine -// registry that can be passed to NewStickyVFSRegistry. -type StickyVFSOption func(cfg *stickyConfig) +// StickyOption is a config option for a sticky vfs +// registry that can be passed to NewStickyRegistry. +type StickyOption func(cfg *stickyConfig) type stickyConfig struct { newFS func() *vfs.MemFS // by default vfs.NewMem } -// UseStrictMemFS option instructs StickyVFSRegistry to produce strict in-memory +// UseStrictMemFS option instructs StickyRegistry to produce strict in-memory // filesystems, i.e. to use vfs.NewStrictMem instead of vfs.NewMem. -var UseStrictMemFS = StickyVFSOption(func(cfg *stickyConfig) { +var UseStrictMemFS = StickyOption(func(cfg *stickyConfig) { cfg.newFS = vfs.NewStrictMem }) -// StickyVFSRegistry manages the lifecycle of sticky in-memory filesystems. It +// StickyRegistry manages the lifecycle of sticky in-memory filesystems. It // is intended for use in demos and/or tests, where we want in-memory storage // nodes to persist between killed nodes. -type StickyVFSRegistry interface { +type StickyRegistry interface { // Get returns the named in-memory FS, constructing a new one if this is the // first time a FS with the provided ID has been requested. Get(stickyVFSID string) *vfs.MemFS } -// stickyVFSRegistryImpl is the bookkeeper for all active sticky filesystems, -// keyed by their id. It implements the StickyVFSRegistry interface. -type stickyVFSRegistryImpl struct { +// stickyRegistryImpl is the bookkeeper for all active sticky filesystems, +// keyed by their id. It implements the StickyRegistry interface. +type stickyRegistryImpl struct { entries map[string]*vfs.MemFS mu syncutil.Mutex cfg stickyConfig } -// NewStickyVFSRegistry creates a new StickyVFSRegistry. -func NewStickyVFSRegistry(opts ...StickyVFSOption) StickyVFSRegistry { - registry := &stickyVFSRegistryImpl{ +// NewStickyRegistry creates a new StickyRegistry. +func NewStickyRegistry(opts ...StickyOption) StickyRegistry { + registry := &stickyRegistryImpl{ entries: map[string]*vfs.MemFS{}, } for _, opt := range opts { @@ -61,8 +61,8 @@ func NewStickyVFSRegistry(opts ...StickyVFSOption) StickyVFSRegistry { return registry } -// Get implements the StickyVFSRegistry interface. -func (registry *stickyVFSRegistryImpl) Get(stickyVFSID string) *vfs.MemFS { +// Get implements the StickyRegistry interface. +func (registry *stickyRegistryImpl) Get(stickyVFSID string) *vfs.MemFS { registry.mu.Lock() defer registry.mu.Unlock() diff --git a/pkg/server/sticky_vfs_test.go b/pkg/storage/fs/sticky_vfs_test.go similarity index 92% rename from pkg/server/sticky_vfs_test.go rename to pkg/storage/fs/sticky_vfs_test.go index d3e057c2f205..bec649db1709 100644 --- a/pkg/server/sticky_vfs_test.go +++ b/pkg/storage/fs/sticky_vfs_test.go @@ -8,7 +8,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -package server +package fs_test import ( "context" @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/stretchr/testify/require" @@ -30,14 +31,11 @@ func TestStickyVFS(t *testing.T) { var ( ctx = context.Background() attrs = roachpb.Attributes{} - cacheSize = int64(1 << 20) /* 1 MiB */ storeSize = int64(512 << 20) /* 512 MiB */ settings = cluster.MakeTestingClusterSettings() - registry = NewStickyVFSRegistry() + registry = fs.NewStickyRegistry() ) - cfg1 := MakeConfig(ctx, settings) - cfg1.CacheSize = cacheSize spec1 := base.StoreSpec{ StickyVFSID: "engine1", Attributes: attrs, diff --git a/pkg/testutils/testcluster/BUILD.bazel b/pkg/testutils/testcluster/BUILD.bazel index 905c92ce8f2c..a0a5a818a940 100644 --- a/pkg/testutils/testcluster/BUILD.bazel +++ b/pkg/testutils/testcluster/BUILD.bazel @@ -64,6 +64,7 @@ go_test( "//pkg/server", "//pkg/server/serverpb", "//pkg/sql/catalog/desctestutils", + "//pkg/storage/fs", "//pkg/testutils", "//pkg/testutils/listenerutil", "//pkg/testutils/serverutils", diff --git a/pkg/testutils/testcluster/testcluster_test.go b/pkg/testutils/testcluster/testcluster_test.go index afe373b8ded3..eb41803fc54b 100644 --- a/pkg/testutils/testcluster/testcluster_test.go +++ b/pkg/testutils/testcluster/testcluster_test.go @@ -24,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog/desctestutils" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -307,7 +308,7 @@ func TestRestart(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - stickyVFSRegistry := server.NewStickyVFSRegistry() + stickyVFSRegistry := fs.NewStickyRegistry() lisReg := listenerutil.NewListenerRegistry() defer lisReg.Close() diff --git a/pkg/ui/pnpm-lock.yaml b/pkg/ui/pnpm-lock.yaml index 33b0fdc8eb0d..668feed13b82 100644 --- a/pkg/ui/pnpm-lock.yaml +++ b/pkg/ui/pnpm-lock.yaml @@ -459,6 +459,19 @@ importers: specifier: ^5.0.0-3 version: 5.0.0-3(webpack@4.41.5) + workspaces/crdb-api-client: + dependencies: + '@bufbuild/protobuf': + specifier: ^1.7.2 + version: 1.7.2 + devDependencies: + '@bufbuild/protoc-gen-es': + specifier: ^1.7.2 + version: 1.7.2(@bufbuild/protobuf@1.7.2) + typescript: + specifier: 5.1.6 + version: 5.1.6 + workspaces/db-console: dependencies: '@babel/runtime': @@ -5151,6 +5164,35 @@ packages: resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} dev: true + /@bufbuild/protobuf@1.7.2: + resolution: {integrity: sha512-i5GE2Dk5ekdlK1TR7SugY4LWRrKSfb5T1Qn4unpIMbfxoeGKERKQ59HG3iYewacGD10SR7UzevfPnh6my4tNmQ==} + + /@bufbuild/protoc-gen-es@1.7.2(@bufbuild/protobuf@1.7.2): + resolution: {integrity: sha512-yiRk/T+YGmpSVvIkybCjPt+QyM/pLWMO+MAiz6auvCsiAgfXfc5nFFosD4yBYXID55M6eIkgBcity1AoJ6I30A==} + engines: {node: '>=14'} + hasBin: true + peerDependencies: + '@bufbuild/protobuf': 1.7.2 + peerDependenciesMeta: + '@bufbuild/protobuf': + optional: true + dependencies: + '@bufbuild/protobuf': 1.7.2 + '@bufbuild/protoplugin': 1.7.2 + transitivePeerDependencies: + - supports-color + dev: true + + /@bufbuild/protoplugin@1.7.2: + resolution: {integrity: sha512-N3QtO8XWD4F4alMtASWtxBw6BWXp4aLz7rPBXH4KTULdjpUHnq46g15TsrG0/8szZw6pIklTO3lFe14dl6ZYdA==} + dependencies: + '@bufbuild/protobuf': 1.7.2 + '@typescript/vfs': 1.5.0 + typescript: 4.5.2 + transitivePeerDependencies: + - supports-color + dev: true + /@cockroachlabs/design-tokens@0.4.5: resolution: {integrity: sha512-az2aIuGjnD0UBN5igLmaKvVWnZJ9r7v2jQ9C4fNR6bxxiF5DkOcD1DXf8EXHIbgwJtorpB74zaic7MQhhMY45w==} @@ -8671,6 +8713,14 @@ packages: '@typescript-eslint/types': 5.26.0 eslint-visitor-keys: 3.4.1 + /@typescript/vfs@1.5.0: + resolution: {integrity: sha512-AJS307bPgbsZZ9ggCT3wwpg3VbTKMFNHfaY/uF0ahSkYYrPF2dSSKDNIDIQAHm9qJqbLvCsSJH7yN4Vs/CsMMg==} + dependencies: + debug: 4.3.4(supports-color@8.1.1) + transitivePeerDependencies: + - supports-color + dev: true + /@webassemblyjs/ast@1.8.5: resolution: {integrity: sha512-aJMfngIZ65+t71C3y2nBBg5FFG0Okt9m0XEgWZ7Ywgn1oMAT8cNwx00Uv1cQyHtidq0Xn94R4TAywO+LCQ+ZAQ==} dependencies: @@ -22969,6 +23019,12 @@ packages: dependencies: typescript-compare: 0.0.2 + /typescript@4.5.2: + resolution: {integrity: sha512-5BlMof9H1yGt0P8/WF+wPNw6GfctgGjXp5hkblpyT+8rkASSmkUKMXrxR0Xg8ThVCi/JnHQiKXeBaEwCeQwMFw==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: true + /typescript@5.1.6: resolution: {integrity: sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==} engines: {node: '>=14.17'} diff --git a/pkg/ui/pnpm-workspace.yaml b/pkg/ui/pnpm-workspace.yaml index afa72f4633e8..55fd45b002d3 100644 --- a/pkg/ui/pnpm-workspace.yaml +++ b/pkg/ui/pnpm-workspace.yaml @@ -5,3 +5,4 @@ packages: - "./workspaces/cluster-ui" - "./workspaces/db-console" - "./workspaces/e2e-tests" + - "./workspaces/crdb-api-client" diff --git a/pkg/ui/workspaces/crdb-api-client/BUILD.bazel b/pkg/ui/workspaces/crdb-api-client/BUILD.bazel new file mode 100644 index 000000000000..d5a1dcf8dac3 --- /dev/null +++ b/pkg/ui/workspaces/crdb-api-client/BUILD.bazel @@ -0,0 +1,88 @@ +load("@aspect_rules_js//js:defs.bzl", "js_binary", "js_run_binary", "js_test") +load("@aspect_rules_js//npm:defs.bzl", "npm_package") +load("@aspect_rules_ts//ts:defs.bzl", "ts_project") +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@npm//:defs.bzl", "npm_link_all_packages") +load("@npm//:defs.bzl", "npm_link_all_packages") +load("//docs/generated/http:defs.bzl", "PROTOBUF_TARGETS") +load("//pkg/ui/workspaces/db-console/src/js:defs.bzl", "proto_sources") + +npm_link_all_packages(name = "node_modules") + +genrule( + name = "protos", + srcs = [ + ":node_modules/@bufbuild/protoc-gen-es/bin/protoc-gen-es", + ":proto_sources", + ] + PROTOBUF_TARGETS, + outs = ["dist"], + cmd = """ + mkdir -p $(OUTS); + DESCRIPTOR_SET_IN=$$(for word in $(SRCS); do echo $$word; done | grep '\\.bin$$' | tr -s '\\n' ':'); + PROTO_PATHS=$$(for f in $(locations :proto_sources); + do if [[ "$$f" =~ ^.*/pkg/.*/_virtual_imports/.*_proto/.*.proto$$ ]]; + then echo $$f | sed 's|.*pkg/.*/_virtual_imports/.*_proto/||' | sed 's|.bin$$||'; + fi; done | sort -u); + export PATH=$$PATH:$$(dirname $(NODE_PATH)) + $(location @com_google_protobuf//:protoc) \ + --plugin=$(location :node_modules/@bufbuild/protoc-gen-es/bin/protoc-gen-es) \ + --es_out $(OUTS) \ + --descriptor_set_in $$DESCRIPTOR_SET_IN \ + --es_opt target=js+dts \ + $$PROTO_PATHS""", + toolchains = ["@nodejs_toolchains//:resolved_toolchain"], + tools = [ + "@com_google_protobuf//:protoc", + "@nodejs_toolchains//:resolved_toolchain", + ], +) + +proto_sources( + name = "proto_sources", + protos = [ + "//pkg/server/serverpb:serverpb_proto", + "//pkg/ts/tspb:tspb_proto", + ], +) + +genrule( + name = "index_ts", + srcs = [":protos"], + outs = ["index.ts"], + cmd = """ + ABS_ROOT_PATH=$$(realpath $(SRCS)) + for f in $$(find $$ABS_ROOT_PATH -name *.js) + do + filename=$$(basename -s .js $$f) + filepath=$${f#$$ABS_ROOT_PATH/} + filepath=$${filepath%.js} + import_name=$${filepath/-/_}; \ + echo export "*" as $${import_name////_} from \\"./$$(basename $(SRCS))/$${filepath}\\"\\; >> $@ + done + """, +) + +ts_project( + name = "ts_project", + srcs = [ + "tsconfig.json", + ":index_ts", + ":protos", + ], + tsconfig = "tsconfig.json", + validate = True, + deps = [":node_modules"], +) + +npm_package( + name = "crdb-api-client", + srcs = [ + "package.json", + "tsconfig.json", + ":index_ts", + ":protos", + ":ts_project", + ], + package = "@cockroachlabs/crdb-api-client", + visibility = ["//visibility:public"], +) diff --git a/pkg/ui/workspaces/crdb-api-client/package.json b/pkg/ui/workspaces/crdb-api-client/package.json new file mode 100644 index 000000000000..b66489bdcaf1 --- /dev/null +++ b/pkg/ui/workspaces/crdb-api-client/package.json @@ -0,0 +1,12 @@ +{ + "name": "@cockroachlabs/crdb-api-client", + "version": "1.0.0", + "main": "index.ts", + "devDependencies": { + "@bufbuild/protoc-gen-es": "^1.7.2", + "typescript": "5.1.6" + }, + "dependencies": { + "@bufbuild/protobuf": "^1.7.2" + } +} diff --git a/pkg/ui/workspaces/crdb-api-client/tsconfig.json b/pkg/ui/workspaces/crdb-api-client/tsconfig.json new file mode 100644 index 000000000000..9e21daf38803 --- /dev/null +++ b/pkg/ui/workspaces/crdb-api-client/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "es2016", + "lib": ["ESNext"], + "module": "ESNext", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": false + } +} diff --git a/pkg/ui/workspaces/db-console/src/js/defs.bzl b/pkg/ui/workspaces/db-console/src/js/defs.bzl index b5cc9e1bf0ed..daeea46599c7 100644 --- a/pkg/ui/workspaces/db-console/src/js/defs.bzl +++ b/pkg/ui/workspaces/db-console/src/js/defs.bzl @@ -10,7 +10,7 @@ def _proto_sources_impl(ctx): transitive = [p[ProtoInfo].transitive_sources for p in ctx.attr.protos], )) -_proto_sources = rule( +proto_sources = rule( doc = """Provider Adapter from ProtoInfo to DefaultInfo. Extracts the transitive_sources from the ProtoInfo provided by the proto attr. This allows a macro to access the complete set of .proto files needed during compilation. @@ -37,7 +37,7 @@ def protobufjs_library(name, out_name, protos, **kwargs): ts_target = "_%s_pbts" % name # grab the transitive .proto files needed to compile the given one - _proto_sources( + proto_sources( name = proto_target, protos = protos, ) diff --git a/pkg/util/startup/BUILD.bazel b/pkg/util/startup/BUILD.bazel index 2fe667343a1f..e3574eedd4e3 100644 --- a/pkg/util/startup/BUILD.bazel +++ b/pkg/util/startup/BUILD.bazel @@ -36,6 +36,7 @@ go_test( "//pkg/server", "//pkg/settings/cluster", "//pkg/spanconfig", + "//pkg/storage/fs", "//pkg/testutils/listenerutil", "//pkg/testutils/serverutils", "//pkg/testutils/skip", diff --git a/pkg/util/startup/startup_test.go b/pkg/util/startup/startup_test.go index 446a2c7df545..375bd10950d5 100644 --- a/pkg/util/startup/startup_test.go +++ b/pkg/util/startup/startup_test.go @@ -29,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/spanconfig" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils/listenerutil" "github.com/cockroachdb/cockroach/pkg/testutils/serverutils" "github.com/cockroachdb/cockroach/pkg/testutils/skip" @@ -139,7 +140,7 @@ func runCircuitBreakerTestForKey( lReg := listenerutil.NewListenerRegistry() defer lReg.Close() - reg := server.NewStickyVFSRegistry() + reg := fs.NewStickyRegistry() // TODO: Disable expiration based leases metamorphism since it currently // breaks closed timestamps and prevent rangefeeds from advancing checkpoint