Skip to content

Commit

Permalink
objstorage: improve test and add read ahead test
Browse files Browse the repository at this point in the history 
This change improves the objstorage test to allow reading arbitrary
parts of objects. We add a test that verifies that the read-ahead
implementation does the Prefetch and the reopen that is expected.

We also improve logging FS to log `ReadAt` and `Prefetch` calls.

Informs cockroachdb#2531.
  • Loading branch information
@RaduBerinde
RaduBerinde committed May 16, 2023
1 parent 2bc4319 commit 5a6b91b
Show file tree
Hide file tree
Showing 14 changed files with 498 additions and 130 deletions.
73 changes: 60 additions & 13 deletions objstorage/objstorageprovider/provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,14 @@ func TestProvider(t *testing.T) {
opts := objstorage.CreateOptions{
SharedCleanupMethod: objstorage.SharedRefTracking,
}
if len(d.CmdArgs) == 3 && d.CmdArgs[2].Key == "no-ref-tracking" {
d.CmdArgs = d.CmdArgs[:2]
if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" {
d.CmdArgs = d.CmdArgs[:4]
opts.SharedCleanupMethod = objstorage.SharedNoCleanup
}
var fileNum base.FileNum
var typ string
scanArgs("<file-num> <local|shared> [no-ref-tracking]", &fileNum, &typ)
var salt, size int
scanArgs("<file-num> <local|shared> <salt> <size> [no-ref-tracking]", &fileNum, &typ, &salt, &size)
switch typ {
case "local":
case "shared":
Expand All @@ -116,7 +117,10 @@ func TestProvider(t *testing.T) {
if err != nil {
return err.Error()
}
require.NoError(t, w.Write([]byte(d.Input)))
data := make([]byte, size)
// TODO(radu): write in chunks?
genData(byte(salt), 0, data)
require.NoError(t, w.Write(data))
require.NoError(t, w.Finish())

return log.String()
Expand All @@ -125,13 +129,14 @@ func TestProvider(t *testing.T) {
opts := objstorage.CreateOptions{
SharedCleanupMethod: objstorage.SharedRefTracking,
}
if len(d.CmdArgs) == 3 && d.CmdArgs[2].Key == "no-ref-tracking" {
d.CmdArgs = d.CmdArgs[:2]
if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" {
d.CmdArgs = d.CmdArgs[:4]
opts.SharedCleanupMethod = objstorage.SharedNoCleanup
}
var fileNum base.FileNum
var typ string
scanArgs("<file-num> <local|shared> [no-ref-tracking]", &fileNum, &typ)
var salt, size int
scanArgs("<file-num> <local|shared> <salt> <size> [no-ref-tracking]", &fileNum, &typ, &salt, &size)
switch typ {
case "local":
case "shared":
Expand All @@ -144,8 +149,10 @@ func TestProvider(t *testing.T) {
tmpFilename := fmt.Sprintf("temp-file-%d", tmpFileCounter)
f, err := fs.Create(tmpFilename)
require.NoError(t, err)
n, err := f.Write([]byte(d.Input))
require.Equal(t, len(d.Input), n)
data := make([]byte, size)
genData(byte(salt), 0, data)
n, err := f.Write(data)
require.Equal(t, len(data), n)
require.NoError(t, err)
require.NoError(t, f.Close())

Expand All @@ -162,10 +169,23 @@ func TestProvider(t *testing.T) {
if err != nil {
return err.Error()
}
data := make([]byte, int(r.Size()))
err = r.ReadAt(ctx, data, 0)
require.NoError(t, err)
return log.String() + fmt.Sprintf("data: %s\n", string(data))
rh := r.NewReadHandle(ctx)
log.Infof("size: %d", r.Size())
for _, l := range strings.Split(d.Input, "\n") {
var offset, size int
fmt.Sscanf(l, "%d %d", &offset, &size)
data := make([]byte, size)
err := rh.ReadAt(ctx, data, int64(offset))
if err != nil {
log.Infof("%d %d: %v", offset, size, err)
} else {
salt := checkData(d, t, offset, data)
log.Infof("%d %d: ok (salt %d)", offset, size, salt)
}
}
require.NoError(t, rh.Close())
require.NoError(t, r.Close())
return log.String()

case "remove":
var fileNum base.FileNum
Expand Down Expand Up @@ -289,3 +309,30 @@ func TestNotExistError(t *testing.T) {
})
}
}

// genData generates object data that can be checked later with checkData.
func genData(salt byte, offset int, p []byte) {
for i := range p {
p[i] = salt ^ xor(offset+i)
}
}

func checkData(d *datadriven.TestData, t *testing.T, offset int, p []byte) (salt byte) {
t.Helper()
salt = p[0] ^ xor(offset)
for i := range p {
if p[i]^xor(offset+i) != salt {
d.Fatalf(t, "invalid data")
}
}
return salt
}

// xor returns the XOR of all bytes representing the integer.
func xor(n int) byte {
v := uint64(n)
v ^= v >> 32
v ^= v >> 16
v ^= v >> 8
return byte(v)
}
53 changes: 42 additions & 11 deletions objstorage/objstorageprovider/testdata/provider/local
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,52 @@ open p0 0
<local fs> mkdir-all: p0 0755
<local fs> open-dir: p0

create 1 local
create 1 local 1 1024
foo
----
<local fs> create: p0/000001.sst
<local fs> sync-data: p0/000001.sst
<local fs> close: p0/000001.sst

read 1
0 512
0 1024
512 1024
----
data: foo
<local fs> open: p0/000001.sst
size: 1024
<local fs> read-at(0, 512): p0/000001.sst
0 512: ok (salt 1)
<local fs> read-at(0, 1024): p0/000001.sst
0 1024: ok (salt 1)
<local fs> prefetch(512, 65536): p0/000001.sst
<local fs> read-at(512, 1024): p0/000001.sst
512 1024: EOF
<local fs> close: p0/000001.sst

# A provider without shared storage creates object with shared preference
# locally.
create 2 shared
bar
create 2 shared 2 1024
----
<local fs> create: p0/000002.sst
<local fs> sync-data: p0/000002.sst
<local fs> close: p0/000002.sst

read 2
0 512
0 1024
512 1024
----
data: bar
<local fs> open: p0/000002.sst
size: 1024
<local fs> read-at(0, 512): p0/000002.sst
0 512: ok (salt 2)
<local fs> read-at(0, 1024): p0/000002.sst
0 1024: ok (salt 2)
<local fs> prefetch(512, 65536): p0/000002.sst
<local fs> read-at(512, 1024): p0/000002.sst
512 1024: EOF
<local fs> close: p0/000002.sst

remove 1
----
Expand All @@ -41,27 +64,35 @@ read 1
----
file 000001 (type 2) unknown to the objstorage provider: file does not exist

link-or-copy 3 local
three
link-or-copy 3 local 3 100
----
<local fs> create: temp-file-1
<local fs> close: temp-file-1
<local fs> link: temp-file-1 -> p0/000003.sst

read 3
0 100
----
data: three
<local fs> open: p0/000003.sst
size: 100
<local fs> read-at(0, 100): p0/000003.sst
0 100: ok (salt 3)
<local fs> close: p0/000003.sst

link-or-copy 4 shared
four
link-or-copy 4 shared 4 1234
----
<local fs> create: temp-file-2
<local fs> close: temp-file-2
<local fs> link: temp-file-2 -> p0/000004.sst

read 4
0 1234
----
data: four
<local fs> open: p0/000004.sst
size: 1234
<local fs> read-at(0, 1234): p0/000004.sst
0 1234: ok (salt 4)
<local fs> close: p0/000004.sst

close
----
Expand Down
51 changes: 51 additions & 0 deletions objstorage/objstorageprovider/testdata/provider/local_readahead
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
open p1 1
----
<local fs> mkdir-all: p1 0755
<local fs> open-dir: p1
<local fs> open-dir: p1
<local fs> create: p1/SHARED-CATALOG-000001
<local fs> sync: p1/SHARED-CATALOG-000001
<local fs> create: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001
<local fs> close: p1/marker.shared-catalog.000001.SHARED-CATALOG-000001
<local fs> sync: p1
<local fs> sync: p1/SHARED-CATALOG-000001

create 1 local 1 2000000
----
<local fs> create: p1/000001.sst
<local fs> sync-data: p1/000001.sst
<local fs> sync-data: p1/000001.sst
<local fs> close: p1/000001.sst

# We should see prefetch calls, and eventually a reopen
# (with sequential reads option).
read 1
0 1000
1000 15000
16000 30000
46000 10000
56000 50000
106000 30000
140000 80000
----
<local fs> open: p1/000001.sst
size: 2000000
<local fs> read-at(0, 1000): p1/000001.sst
0 1000: ok (salt 1)
<local fs> read-at(1000, 15000): p1/000001.sst
1000 15000: ok (salt 1)
<local fs> prefetch(16000, 65536): p1/000001.sst
<local fs> read-at(16000, 30000): p1/000001.sst
16000 30000: ok (salt 1)
<local fs> read-at(46000, 10000): p1/000001.sst
46000 10000: ok (salt 1)
<local fs> prefetch(56000, 131072): p1/000001.sst
<local fs> read-at(56000, 50000): p1/000001.sst
56000 50000: ok (salt 1)
<local fs> read-at(106000, 30000): p1/000001.sst
106000 30000: ok (salt 1)
<local fs> open: p1/000001.sst
<local fs> read-at(140000, 80000): p1/000001.sst
140000 80000: ok (salt 1)
<local fs> close: p1/000001.sst
<local fs> close: p1/000001.sst
53 changes: 31 additions & 22 deletions objstorage/objstorageprovider/testdata/provider/shared_attach
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,32 +13,28 @@ open p1 1
<local fs> sync: p1
<local fs> sync: p1/SHARED-CATALOG-000001

create 1 shared
obj-one
create 1 shared 1 100
----
<shared> create object "61a6-1-000001.sst"
<shared> close writer for "61a6-1-000001.sst" after 7 bytes
<shared> close writer for "61a6-1-000001.sst" after 100 bytes
<shared> create object "61a6-1-000001.sst.ref.1.000001"
<shared> close writer for "61a6-1-000001.sst.ref.1.000001" after 0 bytes

create 2 shared
obj-two
create 2 shared 2 200
----
<shared> create object "a629-1-000002.sst"
<shared> close writer for "a629-1-000002.sst" after 7 bytes
<shared> close writer for "a629-1-000002.sst" after 200 bytes
<shared> create object "a629-1-000002.sst.ref.1.000002"
<shared> close writer for "a629-1-000002.sst.ref.1.000002" after 0 bytes

create 3 shared
obj-three
create 3 shared 3 300
----
<shared> create object "eaac-1-000003.sst"
<shared> close writer for "eaac-1-000003.sst" after 9 bytes
<shared> close writer for "eaac-1-000003.sst" after 300 bytes
<shared> create object "eaac-1-000003.sst.ref.1.000003"
<shared> close writer for "eaac-1-000003.sst.ref.1.000003" after 0 bytes

create 100 local
obj-one
create 100 local 100 15
----
<local fs> create: p1/000100.sst
<local fs> sync-data: p1/000100.sst
Expand Down Expand Up @@ -83,8 +79,7 @@ open p2 2
<local fs> sync: p2
<local fs> sync: p2/SHARED-CATALOG-000001

create 100 shared
obj-one-hundred
create 100 shared 100 15
----
<shared> create object "fd72-2-000100.sst"
<shared> close writer for "fd72-2-000100.sst" after 15 bytes
Expand Down Expand Up @@ -118,25 +113,39 @@ list
000103 -> shared://eaac-1-000003.sst

read 101
0 100
15 10
----
<shared> size of object "61a6-1-000001.sst.ref.2.000101": 0
<shared> size of object "61a6-1-000001.sst": 7
<shared> read object "61a6-1-000001.sst" at 0: 7 bytes
<shared> size of object "61a6-1-000001.sst": 100
size: 100
<shared> read object "61a6-1-000001.sst" at 0: 100 bytes
0 100: ok (salt 1)
<shared> close reader for "61a6-1-000001.sst"
<shared> read object "61a6-1-000001.sst" at 15: 85 bytes
15 10: ok (salt 1)
<shared> close reader for "61a6-1-000001.sst"
data: obj-one

read 102
0 200
90 100
----
<shared> size of object "a629-1-000002.sst.ref.2.000102": 0
<shared> size of object "a629-1-000002.sst": 7
<shared> read object "a629-1-000002.sst" at 0: 7 bytes
<shared> size of object "a629-1-000002.sst": 200
size: 200
<shared> read object "a629-1-000002.sst" at 0: 200 bytes
0 200: ok (salt 2)
<shared> close reader for "a629-1-000002.sst"
<shared> read object "a629-1-000002.sst" at 90: 110 bytes
90 100: ok (salt 2)
<shared> close reader for "a629-1-000002.sst"
data: obj-two

read 103
0 300
----
<shared> size of object "eaac-1-000003.sst.ref.2.000103": 0
<shared> size of object "eaac-1-000003.sst": 9
<shared> read object "eaac-1-000003.sst" at 0: 9 bytes
<shared> size of object "eaac-1-000003.sst": 300
size: 300
<shared> read object "eaac-1-000003.sst" at 0: 300 bytes
0 300: ok (salt 3)
<shared> close reader for "eaac-1-000003.sst"
data: obj-three
10 changes: 4 additions & 6 deletions objstorage/objstorageprovider/testdata/provider/shared_attach_after_unref
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,10 @@ open p5 5
<local fs> sync: p5
<local fs> sync: p5/SHARED-CATALOG-000001

create 1 shared
obj-one
create 1 shared 1 100
----
<shared> create object "d632-5-000001.sst"
<shared> close writer for "d632-5-000001.sst" after 7 bytes
<shared> close writer for "d632-5-000001.sst" after 100 bytes
<shared> create object "d632-5-000001.sst.ref.5.000001"
<shared> close writer for "d632-5-000001.sst.ref.5.000001" after 0 bytes

Expand Down Expand Up @@ -56,11 +55,10 @@ switch p5
close-backing p5b1
----

create 2 shared
obj-two
create 2 shared 2 100
----
<shared> create object "1ab5-5-000002.sst"
<shared> close writer for "1ab5-5-000002.sst" after 7 bytes
<shared> close writer for "1ab5-5-000002.sst" after 100 bytes
<shared> create object "1ab5-5-000002.sst.ref.5.000002"
<shared> close writer for "1ab5-5-000002.sst.ref.5.000002" after 0 bytes

Expand Down
Loading

0 comments on commit 5a6b91b

Please sign in to comment.