Skip to content

Commit

Permalink
seekable: use private options
Browse files Browse the repository at this point in the history
  • Loading branch information
SaveTheRbtz committed Jul 4, 2022
1 parent 7d1e377 commit 6bc0c21
Show file tree
Hide file tree
Showing 14 changed files with 68 additions and 111 deletions.
4 changes: 2 additions & 2 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,15 @@ go_library(
"doc.go",
"encoder.go",
"reader.go",
"reader_options.go",
"seekable.go",
"writer.go",
"writer_options.go",
],
importpath = "github.com/SaveTheRbtz/zstd-seekable-format-go",
visibility = ["//visibility:public"],
deps = [
"//env",
"//options",
"@com_github_cespare_xxhash_v2//:xxhash",
"@com_github_google_btree//:btree",
"@org_uber_go_atomic//:atomic",
Expand Down Expand Up @@ -58,7 +59,6 @@ go_test(
race = "on",
deps = [
"//env",
"//options",
"@com_github_klauspost_compress//zstd",
"@com_github_stretchr_testify//assert",
],
Expand Down
1 change: 0 additions & 1 deletion cmd/zstdseek/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ go_library(
visibility = ["//visibility:private"],
deps = [
"//:zstd-seekable-format-go",
"//options",
"@com_github_klauspost_compress//zstd",
"@com_github_restic_chunker//:chunker",
"@org_uber_go_zap//:zap",
Expand Down
5 changes: 2 additions & 3 deletions cmd/zstdseek/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"go.uber.org/zap"

seekable "github.com/SaveTheRbtz/zstd-seekable-format-go"
"github.com/SaveTheRbtz/zstd-seekable-format-go/options"
)

type readCloser struct {
Expand Down Expand Up @@ -107,7 +106,7 @@ func main() {
logger.Fatal("failed to create zstd encoder", zap.Error(err))
}

w, err := seekable.NewWriter(output, enc, options.WithWLogger(logger))
w, err := seekable.NewWriter(output, enc, seekable.WithWLogger(logger))
if err != nil {
logger.Fatal("failed to create compressed writer", zap.Error(err))
}
Expand Down Expand Up @@ -169,7 +168,7 @@ func main() {
}
defer dec.Close()

reader, err := seekable.NewReader(verify, dec, options.WithRLogger(logger))
reader, err := seekable.NewReader(verify, dec, seekable.WithRLogger(logger))
if err != nil {
logger.Fatal("failed to create new seekable reader", zap.Error(err))
}
Expand Down
7 changes: 3 additions & 4 deletions decoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package seekable

import (
"github.com/SaveTheRbtz/zstd-seekable-format-go/env"
"github.com/SaveTheRbtz/zstd-seekable-format-go/options"
)

// Decoder is a byte-oriented API that is useful for cases where wrapping io.ReadSeeker is not desirable.
Expand All @@ -28,16 +27,16 @@ type Decoder interface {
// NewDecoder creates a byte-oriented Decode interface from a given seektable index.
// This index can either be produced by either Writer's WriteSeekTable or Encoder's EndStream.
// Decoder can be used concurrently.
func NewDecoder(seekTable []byte, decoder ZSTDDecoder, opts ...options.ROption) (Decoder, error) {
opts = append(opts, options.WithREnvironment(&decoderEnv{seekTable: seekTable}))
func NewDecoder(seekTable []byte, decoder ZSTDDecoder, opts ...rOption) (Decoder, error) {
opts = append(opts, WithREnvironment(&decoderEnv{seekTable: seekTable}))

sr, err := NewReader(nil, decoder, opts...)
if err != nil {
return nil, err
}

// Release seekTable reference to not leak memory.
sr.(*readerImpl).o.Env = nil
sr.(*readerImpl).env = nil

return sr.(*readerImpl), err
}
Expand Down
6 changes: 2 additions & 4 deletions encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import (

"github.com/cespare/xxhash/v2"
"go.uber.org/zap"

"github.com/SaveTheRbtz/zstd-seekable-format-go/options"
)

// Encoder is a byte-oriented API that is useful where wrapping io.Writer is not desirable.
Expand All @@ -19,7 +17,7 @@ type Encoder interface {
EndStream() ([]byte, error)
}

func NewEncoder(encoder ZSTDEncoder, opts ...options.WOption) (Encoder, error) {
func NewEncoder(encoder ZSTDEncoder, opts ...wOption) (Encoder, error) {
sw, err := NewWriter(nil, encoder, opts...)
if err != nil {
return nil, err
Expand Down Expand Up @@ -51,7 +49,7 @@ func (s *writerImpl) Encode(src []byte) ([]byte, error) {
Checksum: uint32((xxhash.Sum64(src) << 32) >> 32),
}

s.o.Logger.Debug("appending frame", zap.Object("frame", &entry))
s.logger.Debug("appending frame", zap.Object("frame", &entry))
s.frameEntries = append(s.frameEntries, entry)

return dst, nil
Expand Down
15 changes: 0 additions & 15 deletions options/BUILD.bazel

This file was deleted.

28 changes: 0 additions & 28 deletions options/reader_options.go

This file was deleted.

28 changes: 0 additions & 28 deletions options/writer_options.go

This file was deleted.

24 changes: 12 additions & 12 deletions reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"go.uber.org/zap"

"github.com/SaveTheRbtz/zstd-seekable-format-go/env"
"github.com/SaveTheRbtz/zstd-seekable-format-go/options"
)

type cachedFrame struct {
Expand Down Expand Up @@ -105,7 +104,8 @@ type readerImpl struct {
numFrames int64
endOffset int64

o options.ReaderOptions
logger *zap.Logger
env env.REnvironment

closed atomic.Bool

Expand Down Expand Up @@ -147,21 +147,21 @@ type ZSTDDecoder interface {

// NewReader returns ZSTD stream reader that can be randomly accessed using uncompressed data offset.
// Ideally, passed io.ReadSeeker should implement io.ReaderAt interface.
func NewReader(rs io.ReadSeeker, decoder ZSTDDecoder, opts ...options.ROption) (Reader, error) {
func NewReader(rs io.ReadSeeker, decoder ZSTDDecoder, opts ...rOption) (Reader, error) {
sr := readerImpl{
dec: decoder,
}

sr.o.SetDefault()
sr.logger = zap.NewNop()
for _, o := range opts {
err := o(&sr.o)
err := o(&sr)
if err != nil {
return nil, err
}
}

if sr.o.Env == nil {
sr.o.Env = &readSeekerEnvImpl{
if sr.env == nil {
sr.env = &readSeekerEnvImpl{
rs: rs,
}
}
Expand Down Expand Up @@ -244,7 +244,7 @@ func (r *readerImpl) read(dst []byte, off int64) (int64, int, error) {
index.CompSize, maxDecoderFrameSize)
}

src, err := r.o.Env.GetFrameByIndex(*index)
src, err := r.env.GetFrameByIndex(*index)
if err != nil {
return 0, 0, fmt.Errorf("failed to read compressed data at: %d, %w", index.CompOffset, err)
}
Expand Down Expand Up @@ -280,7 +280,7 @@ func (r *readerImpl) read(dst []byte, off int64) (int64, int, error) {
size = uint64(len(dst))
}

r.o.Logger.Debug("decompressed", zap.Uint64("offsetWithinFrame", offsetWithinFrame), zap.Uint64("end", offsetWithinFrame+size),
r.logger.Debug("decompressed", zap.Uint64("offsetWithinFrame", offsetWithinFrame), zap.Uint64("end", offsetWithinFrame+size),
zap.Uint64("size", size), zap.Int("lenDecompressed", len(decompressed)), zap.Int("lenDst", len(dst)), zap.Object("index", index))
copy(dst, decompressed[offsetWithinFrame:offsetWithinFrame+size])

Expand Down Expand Up @@ -311,7 +311,7 @@ func (r *readerImpl) Seek(offset int64, whence int) (int64, error) {

func (r *readerImpl) indexFooter() (*btree.BTreeG[*env.FrameOffsetEntry], *env.FrameOffsetEntry, error) {
// read seekTableFooter
buf, err := r.o.Env.ReadFooter()
buf, err := r.env.ReadFooter()
if err != nil {
return nil, nil, fmt.Errorf("failed to read footer: %w", err)
}
Expand All @@ -325,7 +325,7 @@ func (r *readerImpl) indexFooter() (*btree.BTreeG[*env.FrameOffsetEntry], *env.F
if err != nil {
return nil, nil, fmt.Errorf("failed to parse footer %+v: %w", buf, err)
}
r.o.Logger.Debug("loaded", zap.Object("footer", &footer))
r.logger.Debug("loaded", zap.Object("footer", &footer))

r.checksums = footer.SeekTableDescriptor.ChecksumFlag

Expand All @@ -344,7 +344,7 @@ func (r *readerImpl) indexFooter() (*btree.BTreeG[*env.FrameOffsetEntry], *env.F
skippableFrameOffset, maxDecoderFrameSize)
}

buf, err = r.o.Env.ReadSkipFrame(skippableFrameOffset)
buf, err = r.env.ReadSkipFrame(skippableFrameOffset)
if err != nil {
return nil, nil, fmt.Errorf("failed to read footer: %w", err)
}
Expand Down
17 changes: 17 additions & 0 deletions reader_options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package seekable

import (
"go.uber.org/zap"

"github.com/SaveTheRbtz/zstd-seekable-format-go/env"
)

type rOption func(*readerImpl) error

func WithRLogger(l *zap.Logger) rOption {
return func(r *readerImpl) error { r.logger = l; return nil }
}

func WithREnvironment(e env.REnvironment) rOption {
return func(r *readerImpl) error { r.env = e; return nil }
}
3 changes: 1 addition & 2 deletions reader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/stretchr/testify/assert"

"github.com/SaveTheRbtz/zstd-seekable-format-go/env"
"github.com/SaveTheRbtz/zstd-seekable-format-go/options"
)

const sourceString = "testtest2"
Expand Down Expand Up @@ -391,7 +390,7 @@ func TestReadEnvironment(t *testing.T) {
assert.NoError(t, err)
defer dec.Close()

r, err := NewReader(nil, dec, options.WithREnvironment(&fakeReadEnvironment{}))
r, err := NewReader(nil, dec, WithREnvironment(&fakeReadEnvironment{}))
assert.NoError(t, err)
defer func() { assert.NoError(t, r.Close()) }()

Expand Down
20 changes: 11 additions & 9 deletions writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@ import (
"sync"

"go.uber.org/multierr"
"go.uber.org/zap"

"github.com/SaveTheRbtz/zstd-seekable-format-go/options"
"github.com/SaveTheRbtz/zstd-seekable-format-go/env"
)

// writerEnvImpl is the environment implementation of for the underlying WriteCloser.
Expand All @@ -27,7 +28,8 @@ type writerImpl struct {
enc ZSTDEncoder
frameEntries []seekTableEntry

o options.WriterOptions
logger *zap.Logger
env env.WEnvironment

once *sync.Once
}
Expand Down Expand Up @@ -58,22 +60,22 @@ type ZSTDEncoder interface {

// NewWriter wraps the passed io.Writer and Encoder into and indexed ZSTD stream.
// Resulting stream then can be randomly accessed through the Reader and Decoder interfaces.
func NewWriter(w io.Writer, encoder ZSTDEncoder, opts ...options.WOption) (Writer, error) {
func NewWriter(w io.Writer, encoder ZSTDEncoder, opts ...wOption) (Writer, error) {
sw := writerImpl{
once: &sync.Once{},
enc: encoder,
}

sw.o.SetDefault()
sw.logger = zap.NewNop()
for _, o := range opts {
err := o(&sw.o)
err := o(&sw)
if err != nil {
return nil, err
}
}

if sw.o.Env == nil {
sw.o.Env = &writerEnvImpl{
if sw.env == nil {
sw.env = &writerEnvImpl{
w: w,
}
}
Expand All @@ -87,7 +89,7 @@ func (s *writerImpl) Write(src []byte) (int, error) {
return 0, err
}

n, err := s.o.Env.WriteFrame(dst)
n, err := s.env.WriteFrame(dst)
if err != nil {
return 0, err
}
Expand All @@ -111,6 +113,6 @@ func (s *writerImpl) writeSeekTable() error {
return err
}

_, err = s.o.Env.WriteSeekTable(seekTableBytes)
_, err = s.env.WriteSeekTable(seekTableBytes)
return err
}
17 changes: 17 additions & 0 deletions writer_options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package seekable

import (
"go.uber.org/zap"

"github.com/SaveTheRbtz/zstd-seekable-format-go/env"
)

type wOption func(*writerImpl) error

func WithWLogger(l *zap.Logger) wOption {
return func(w *writerImpl) error { w.logger = l; return nil }
}

func WithWEnvironment(e env.WEnvironment) wOption {
return func(w *writerImpl) error { w.env = e; return nil }
}
Loading

0 comments on commit 6bc0c21

Please sign in to comment.