Skip to content

Commit

Permalink
Merge #139365 #140071 #140241
Browse files Browse the repository at this point in the history 
139365: docgen: update COPY TO STDOUT portions of SQL diagram r=mgartner a=taroface

Updated `COPY` diagram so that it better reflects the `COPY TO` syntax that was added in 23.1 (this PR will be backported to every version up until 23.1). These changes accompany the docs updates in cockroachdb/docs#19310
 
Current diagram:

<img width="647" alt="image" src="https://github.com/user-attachments/assets/eb70b831-3473-40aa-823f-5432a8cf2698" />

New diagram:

<img width="829" alt="image" src="https://github.com/user-attachments/assets/314e1345-cec8-4a72-9be2-fb4b2f2885ee" />

Epic: none
Release note: none
Release justification: non-production code change

140071: storage: covert SizeSpec to protobuf r=RaduBerinde a=andrewbaptist

This commit moves SizeSpec to the storagepb directory and converts it toa protobuf. Additionally it pulls shared storage into the global StorageConfig file.

Epic: none

Release note: None

140241: multiregionccl: deflake TestColdStartLatency r=rafiss a=rafiss

Similar to what we're seeing in
#140172 and the issues linked to #140187, this test is flaky due to running with autocommit_before_ddl=true in a multitenant setup.

fixes #140174
Release note: None

Co-authored-by: Ryan Kuo <[email protected]>
Co-authored-by: Andrew Baptist <[email protected]>
Co-authored-by: Rafi Shamim <[email protected]>
  • Loading branch information
@taroface @andrewbaptist @rafiss
4 people committed Jan 31, 2025
4 parents 4493b25 + f82b7c8 + fcc23e1 + bbf71b2 commit 6a0a025
Show file tree
Hide file tree
Showing 28 changed files with 259 additions and 206 deletions.
18 changes: 9 additions & 9 deletions docs/generated/sql/bnf/copy_stmt.bnf
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
copy_stmt ::=
'COPY' table_name opt_column_list 'FROM' 'STDIN' 'WITH' copy_options ( ( copy_options ) )*
| 'COPY' table_name opt_column_list 'FROM' 'STDIN' copy_options ( ( copy_options ) )*
| 'COPY' table_name opt_column_list 'FROM' 'STDIN' 'WITH' '(' copy_generic_options_list ')'
| 'COPY' table_name opt_column_list 'FROM' 'STDIN' '(' copy_generic_options_list ')'
| 'COPY' table_name opt_column_list 'FROM' 'STDIN' 'WITH'
| 'COPY' table_name opt_column_list 'FROM' 'STDIN'
| 'COPY' table_name opt_column_list 'FROM' 'STDIN'
| 'COPY' table_name opt_column_list 'TO' 'STDOUT' 'WITH' copy_options ( ( copy_options ) )*
| 'COPY' table_name opt_column_list 'TO' 'STDOUT' copy_options ( ( copy_options ) )*
| 'COPY' table_name opt_column_list 'TO' 'STDOUT' 'WITH' '(' copy_generic_options_list ')'
| 'COPY' table_name opt_column_list 'TO' 'STDOUT' '(' copy_generic_options_list ')'
| 'COPY' table_name opt_column_list 'TO' 'STDOUT' 'WITH'
| 'COPY' table_name opt_column_list 'TO' 'STDOUT'
| 'COPY' table_name opt_column_list 'TO' 'STDOUT'
| 'COPY' '(' copy_to_stmt ')' 'TO' 'STDOUT' 'WITH' copy_options ( ( copy_options ) )*
| 'COPY' '(' copy_to_stmt ')' 'TO' 'STDOUT' copy_options ( ( copy_options ) )*
| 'COPY' '(' copy_to_stmt ')' 'TO' 'STDOUT' 'WITH' '(' copy_generic_options_list ')'
| 'COPY' '(' copy_to_stmt ')' 'TO' 'STDOUT' '(' copy_generic_options_list ')'
| 'COPY' '(' copy_to_stmt ')' 'TO' 'STDOUT'
| 'COPY' '(' query ')' 'TO' 'STDOUT' 'WITH' copy_options ( ( copy_options ) )*
| 'COPY' '(' query ')' 'TO' 'STDOUT' copy_options ( ( copy_options ) )*
| 'COPY' '(' query ')' 'TO' 'STDOUT' 'WITH'
| 'COPY' '(' query ')' 'TO' 'STDOUT'
| 'COPY' '(' query ')' 'TO' 'STDOUT'
1 change: 1 addition & 0 deletions pkg/acceptance/cluster/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ go_library(
"//pkg/security",
"//pkg/security/certnames",
"//pkg/security/username",
"//pkg/storage/storagepb",
"//pkg/testutils/datapathutils",
"//pkg/util/log",
"//pkg/util/log/logflags",
Expand Down
3 changes: 2 additions & 1 deletion pkg/acceptance/cluster/dockercluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/security"
"github.com/cockroachdb/cockroach/pkg/security/certnames"
"github.com/cockroachdb/cockroach/pkg/security/username"
"github.com/cockroachdb/cockroach/pkg/storage/storagepb"
"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/log/logflags"
Expand Down Expand Up @@ -484,7 +485,7 @@ func (l *DockerCluster) startNode(ctx context.Context, node *testNode, singleNod
for _, store := range node.stores {
storeSpec := base.StoreSpec{
Path: store.dir,
Size: base.SizeSpec{InBytes: int64(store.config.MaxRanges) * maxRangeBytes},
Size: storagepb.SizeSpec{Capacity: int64(store.config.MaxRanges) * maxRangeBytes},
}
cmd = append(cmd, fmt.Sprintf("--store=%s", storeSpec))
}
Expand Down
145 changes: 13 additions & 132 deletions pkg/base/store_spec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ import (
"net"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"unicode"

Expand Down Expand Up @@ -54,109 +52,6 @@ func GetAbsoluteFSPath(fieldName string, p string) (string, error) {
return ret, nil
}

// SizeSpec contains size in different kinds of formats supported by CLI(%age, bytes).
type SizeSpec struct {
// InBytes is used for calculating free space and making rebalancing
// decisions. Zero indicates that there is no maximum size. This value is not
// actually used by the engine and thus not enforced.
InBytes int64
Percent float64
}

type intInterval struct {
min *int64
max *int64
}

type floatInterval struct {
min *float64
max *float64
}

// NewSizeSpec parses the string passed into a --size flag and returns a
// SizeSpec if it is correctly parsed.
func NewSizeSpec(
field redact.SafeString, value string, bytesRange *intInterval, percentRange *floatInterval,
) (SizeSpec, error) {
var size SizeSpec
if fractionRegex.MatchString(value) {
percentFactor := 100.0
factorValue := value
if value[len(value)-1] == '%' {
percentFactor = 1.0
factorValue = value[:len(value)-1]
}
var err error
size.Percent, err = strconv.ParseFloat(factorValue, 64)
size.Percent *= percentFactor
if err != nil {
return SizeSpec{}, errors.Wrapf(err, "could not parse %s size (%s)", field, value)
}
if percentRange != nil {
if (percentRange.min != nil && size.Percent < *percentRange.min) ||
(percentRange.max != nil && size.Percent > *percentRange.max) {
return SizeSpec{}, errors.Newf(
"%s size (%s) must be between %f%% and %f%%",
field,
value,
*percentRange.min,
*percentRange.max,
)
}
}
} else {
var err error
size.InBytes, err = humanizeutil.ParseBytes(value)
if err != nil {
return SizeSpec{}, errors.Wrapf(err, "could not parse %s size (%s)", field, value)
}
if bytesRange != nil {
if bytesRange.min != nil && size.InBytes < *bytesRange.min {
return SizeSpec{}, errors.Newf("%s size (%s) must be larger than %s",
field, value, humanizeutil.IBytes(*bytesRange.min))
}
if bytesRange.max != nil && size.InBytes > *bytesRange.max {
return SizeSpec{}, errors.Newf("%s size (%s) must be smaller than %s",
field, value, humanizeutil.IBytes(*bytesRange.max))
}
}
}
return size, nil
}

// String returns a string representation of the SizeSpec. This is part
// of pflag's value interface.
func (ss *SizeSpec) String() string {
var buffer bytes.Buffer
if ss.InBytes != 0 {
fmt.Fprintf(&buffer, "--size=%s,", humanizeutil.IBytes(ss.InBytes))
}
if ss.Percent != 0 {
fmt.Fprintf(&buffer, "--size=%s%%,", humanize.Ftoa(ss.Percent))
}
return buffer.String()
}

// Type returns the underlying type in string form. This is part of pflag's
// value interface.
func (ss *SizeSpec) Type() string {
return "SizeSpec"
}

var _ pflag.Value = &SizeSpec{}

// Set adds a new value to the StoreSpecValue. It is the important part of
// pflag's value interface.
func (ss *SizeSpec) Set(value string) error {
spec, err := NewSizeSpec("specified", value, nil, nil)
if err != nil {
return err
}
ss.InBytes = spec.InBytes
ss.Percent = spec.Percent
return nil
}

// ProvisionedRateSpec is an optional part of the StoreSpec.
type ProvisionedRateSpec struct {
// ProvisionedBandwidth is the bandwidth provisioned for this store in bytes/s.
Expand Down Expand Up @@ -200,8 +95,8 @@ func newStoreProvisionedRateSpec(
// to the --store flag.
type StoreSpec struct {
Path string
Size SizeSpec
BallastSize *SizeSpec
Size storagepb.SizeSpec
BallastSize *storagepb.SizeSpec
InMemory bool
Attributes roachpb.Attributes
// StickyVFSID is a unique identifier associated with a given store which
Expand Down Expand Up @@ -231,15 +126,15 @@ func (ss StoreSpec) String() string {
if ss.InMemory {
fmt.Fprint(&buffer, "type=mem,")
}
if ss.Size.InBytes > 0 {
fmt.Fprintf(&buffer, "size=%s,", humanizeutil.IBytes(ss.Size.InBytes))
if ss.Size.Capacity > 0 {
fmt.Fprintf(&buffer, "size=%s,", humanizeutil.IBytes(ss.Size.Capacity))
}
if ss.Size.Percent > 0 {
fmt.Fprintf(&buffer, "size=%s%%,", humanize.Ftoa(ss.Size.Percent))
}
if ss.BallastSize != nil {
if ss.BallastSize.InBytes > 0 {
fmt.Fprintf(&buffer, "ballast-size=%s,", humanizeutil.IBytes(ss.BallastSize.InBytes))
if ss.BallastSize.Capacity > 0 {
fmt.Fprintf(&buffer, "ballast-size=%s,", humanizeutil.IBytes(ss.BallastSize.Capacity))
}
if ss.BallastSize.Percent > 0 {
fmt.Fprintf(&buffer, "ballast-size=%s%%,", humanize.Ftoa(ss.BallastSize.Percent))
Expand Down Expand Up @@ -277,20 +172,6 @@ func (ss StoreSpec) IsEncrypted() bool {
return ss.EncryptionOptions != nil
}

// fractionRegex is the regular expression that recognizes whether
// the specified size is a fraction of the total available space.
// Proportional sizes can be expressed as fractional numbers, either
// in absolute value or with a trailing "%" sign. A fractional number
// without a trailing "%" must be recognized by the presence of a
// decimal separator; numbers without decimal separators are plain
// sizes in bytes (separate case in the parsing).
// The first part of the regexp matches NNN.[MMM]; the second part
// [NNN].MMM, and the last part matches explicit percentages with or
// without a decimal separator.
// Values smaller than 1% and 100% are rejected after parsing using
// a separate check.
var fractionRegex = regexp.MustCompile(`^([-]?([0-9]+\.[0-9]*|[0-9]*\.[0-9]+|[0-9]+(\.[0-9]*)?%))$`)

// NewStoreSpec parses the string passed into a --store flag and returns a
// StoreSpec if it is correctly parsed.
// There are five possible fields that can be passed in, comma separated:
Expand Down Expand Up @@ -353,11 +234,11 @@ func NewStoreSpec(value string) (StoreSpec, error) {
var minBytesAllowed int64 = MinimumStoreSize
var minPercent float64 = 1
var maxPercent float64 = 100
ss.Size, err = NewSizeSpec(
ss.Size, err = storagepb.NewSizeSpec(
"store",
value,
&intInterval{min: &minBytesAllowed},
&floatInterval{min: &minPercent, max: &maxPercent},
&storagepb.IntInterval{Min: &minBytesAllowed},
&storagepb.FloatInterval{Min: &minPercent, Max: &maxPercent},
)
if err != nil {
return StoreSpec{}, err
Expand All @@ -366,11 +247,11 @@ func NewStoreSpec(value string) (StoreSpec, error) {
var minBytesAllowed int64
var minPercent float64 = 0
var maxPercent float64 = 50
ballastSize, err := NewSizeSpec(
ballastSize, err := storagepb.NewSizeSpec(
"ballast",
value,
&intInterval{min: &minBytesAllowed},
&floatInterval{min: &minPercent, max: &maxPercent},
&storagepb.IntInterval{Min: &minBytesAllowed},
&storagepb.FloatInterval{Min: &minPercent, Max: &maxPercent},
)
if err != nil {
return StoreSpec{}, err
Expand Down Expand Up @@ -447,7 +328,7 @@ func NewStoreSpec(value string) (StoreSpec, error) {
if ss.Path != "" {
return StoreSpec{}, fmt.Errorf("path specified for in memory store")
}
if ss.Size.Percent == 0 && ss.Size.InBytes == 0 {
if ss.Size.Percent == 0 && ss.Size.Capacity == 0 {
return StoreSpec{}, fmt.Errorf("size must be specified for an in memory store")
}
if ss.BallastSize != nil {
Expand Down
35 changes: 18 additions & 17 deletions pkg/base/store_spec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/cockroachdb/cockroach/pkg/base"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/storage/storagepb"
"github.com/cockroachdb/cockroach/pkg/testutils"
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
"github.com/cockroachdb/errors"
Expand Down Expand Up @@ -92,13 +93,13 @@ target_file_size=2097152`
{"path=/mnt/hda1,attrs=hdd,attrs=ssd", "attrs field was used twice in store definition", StoreSpec{}},

// size
{"path=/mnt/hda1,size=671088640", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{InBytes: 671088640}}},
{"path=/mnt/hda1,size=20GB", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{InBytes: 20000000000}}},
{"size=20GiB,path=/mnt/hda1", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{InBytes: 21474836480}}},
{"size=0.1TiB,path=/mnt/hda1", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{InBytes: 109951162777}}},
{"path=/mnt/hda1,size=.1TiB", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{InBytes: 109951162777}}},
{"path=/mnt/hda1,size=123TB", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{InBytes: 123000000000000}}},
{"path=/mnt/hda1,size=123TiB", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{InBytes: 135239930216448}}},
{"path=/mnt/hda1,size=671088640", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Capacity: 671088640}}},
{"path=/mnt/hda1,size=20GB", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Capacity: 20000000000}}},
{"size=20GiB,path=/mnt/hda1", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Capacity: 21474836480}}},
{"size=0.1TiB,path=/mnt/hda1", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Capacity: 109951162777}}},
{"path=/mnt/hda1,size=.1TiB", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Capacity: 109951162777}}},
{"path=/mnt/hda1,size=123TB", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Capacity: 123000000000000}}},
{"path=/mnt/hda1,size=123TiB", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Capacity: 135239930216448}}},
// %
{"path=/mnt/hda1,size=50.5%", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Percent: 50.5}}},
{"path=/mnt/hda1,size=100%", "", StoreSpec{Path: "/mnt/hda1", Size: SizeSpec{Percent: 100}}},
Expand All @@ -123,18 +124,18 @@ target_file_size=2097152`
{"size=123TB", "no path specified", StoreSpec{}},

// ballast size
{"path=/mnt/hda1,ballast-size=671088640", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{InBytes: 671088640}}},
{"path=/mnt/hda1,ballast-size=20GB", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{InBytes: 20000000000}}},
{"path=/mnt/hda1,ballast-size=671088640", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{Capacity: 671088640}}},
{"path=/mnt/hda1,ballast-size=20GB", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{Capacity: 20000000000}}},
{"path=/mnt/hda1,ballast-size=1%", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{Percent: 1}}},
{"path=/mnt/hda1,ballast-size=100.000%", "ballast size (100.000%) must be between 0.000000% and 50.000000%", StoreSpec{}},
{"ballast-size=20GiB,path=/mnt/hda1,ballast-size=20GiB", "ballast-size field was used twice in store definition", StoreSpec{}},

// type
{"type=mem,size=20GiB", "", StoreSpec{Size: SizeSpec{InBytes: 21474836480}, InMemory: true}},
{"size=20GiB,type=mem", "", StoreSpec{Size: SizeSpec{InBytes: 21474836480}, InMemory: true}},
{"size=20.5GiB,type=mem", "", StoreSpec{Size: SizeSpec{InBytes: 22011707392}, InMemory: true}},
{"type=mem,size=20GiB", "", StoreSpec{Size: SizeSpec{Capacity: 21474836480}, InMemory: true}},
{"size=20GiB,type=mem", "", StoreSpec{Size: SizeSpec{Capacity: 21474836480}, InMemory: true}},
{"size=20.5GiB,type=mem", "", StoreSpec{Size: SizeSpec{Capacity: 22011707392}, InMemory: true}},
{"size=20GiB,type=mem,attrs=mem", "", StoreSpec{
Size: SizeSpec{InBytes: 21474836480},
Size: SizeSpec{Capacity: 21474836480},
InMemory: true,
Attributes: roachpb.Attributes{Attrs: []string{"mem"}},
}},
Expand All @@ -161,11 +162,11 @@ target_file_size=2097152`
// all together
{"path=/mnt/hda1,attrs=hdd:ssd,size=20GiB", "", StoreSpec{
Path: "/mnt/hda1",
Size: SizeSpec{InBytes: 21474836480},
Size: SizeSpec{Capacity: 21474836480},
Attributes: roachpb.Attributes{Attrs: []string{"hdd", "ssd"}},
}},
{"type=mem,attrs=hdd:ssd,size=20GiB", "", StoreSpec{
Size: SizeSpec{InBytes: 21474836480},
Size: SizeSpec{Capacity: 21474836480},
InMemory: true,
Attributes: roachpb.Attributes{Attrs: []string{"hdd", "ssd"}},
}},
Expand Down Expand Up @@ -218,8 +219,8 @@ target_file_size=2097152`
// StoreSpec aliases base.StoreSpec for convenience.
type StoreSpec = base.StoreSpec

// SizeSpec aliases base.SizeSpec for convenience.
type SizeSpec = base.SizeSpec
// SizeSpec aliases storagepb.SizeSpec for convenience.
type SizeSpec = storagepb.SizeSpec

func TestJoinListType(t *testing.T) {
defer leaktest.AfterTest(t)()
Expand Down
5 changes: 3 additions & 2 deletions pkg/base/test_server_args.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/storage/storagepb"
"github.com/cockroachdb/cockroach/pkg/testutils/listenerutil"
"github.com/cockroachdb/cockroach/pkg/util/mon"
"github.com/cockroachdb/cockroach/pkg/util/retry"
Expand Down Expand Up @@ -490,8 +491,8 @@ var (
// with no special attributes.
DefaultTestStoreSpec = StoreSpec{
InMemory: true,
Size: SizeSpec{
InBytes: 512 << 20,
Size: storagepb.SizeSpec{
Capacity: 512 << 20,
},
}
)
Expand Down
1 change: 1 addition & 0 deletions pkg/ccl/multiregionccl/cold_start_latency_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ func TestColdStartLatency(t *testing.T) {
} else {
stmts = []string{`
BEGIN;
SET LOCAL autocommit_before_ddl = false;
ALTER DATABASE system PRIMARY REGION "us-east1";
ALTER DATABASE system ADD REGION "us-west1";
ALTER DATABASE system ADD REGION "europe-west1";
Expand Down
6 changes: 3 additions & 3 deletions pkg/ccl/storageccl/engineccl/encrypted_fs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ func TestPebbleEncryption(t *testing.T) {
base.StoreSpec{
InMemory: true,
Attributes: roachpb.Attributes{},
Size: base.SizeSpec{InBytes: 512 << 20},
Size: storagepb.SizeSpec{Capacity: 512 << 20},
EncryptionOptions: encOptions,
StickyVFSID: stickyVFSID,
},
Expand Down Expand Up @@ -301,7 +301,7 @@ func TestPebbleEncryption(t *testing.T) {
base.StoreSpec{
InMemory: true,
Attributes: roachpb.Attributes{},
Size: base.SizeSpec{InBytes: 512 << 20},
Size: storagepb.SizeSpec{Capacity: 512 << 20},
EncryptionOptions: encOptions,
StickyVFSID: stickyVFSID,
},
Expand Down Expand Up @@ -389,7 +389,7 @@ func TestPebbleEncryption2(t *testing.T) {
base.StoreSpec{
InMemory: true,
Attributes: roachpb.Attributes{},
Size: base.SizeSpec{InBytes: 512 << 20},
Size: storagepb.SizeSpec{Capacity: 512 << 20},
EncryptionOptions: encOptions,
StickyVFSID: stickyVFSID,
},
Expand Down
Loading

0 comments on commit 6a0a025

Please sign in to comment.