From eed779eb0d47eac6de2c38ab5c8cf526c10a4fba Mon Sep 17 00:00:00 2001 From: Filip Petkovski Date: Sun, 23 Jul 2023 16:34:54 +0200 Subject: [PATCH] tmp --- inmem.go | 2 +- objstore.go | 13 ++++++++----- prefixed_bucket.go | 2 +- providers/azure/azure.go | 2 +- providers/bos/bos.go | 2 +- providers/cos/cos.go | 2 +- providers/filesystem/filesystem.go | 2 +- providers/gcs/gcs.go | 2 +- providers/obs/obs.go | 2 +- providers/oci/oci.go | 2 +- providers/oss/oss.go | 2 +- providers/s3/s3.go | 2 +- providers/swift/swift.go | 2 +- testing.go | 2 +- tracing/opentelemetry/opentelemetry.go | 2 +- tracing/opentracing/opentracing.go | 2 +- 16 files changed, 23 insertions(+), 20 deletions(-) diff --git a/inmem.go b/inmem.go index da1d8ea0..55e28de3 100644 --- a/inmem.go +++ b/inmem.go @@ -50,7 +50,7 @@ func (b *InMemBucket) Objects() map[string][]byte { // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. -func (b *InMemBucket) Iter(ctx context.Context, dir string, f func(name string, attrs ObjectAttributes) error, options ...IterOption) error { +func (b *InMemBucket) Iter(ctx context.Context, dir string, f func(attrs ObjectAttributes) error, options ...IterOption) error { unique := map[string]struct{}{} params := ApplyIterOptions(options...) diff --git a/objstore.go b/objstore.go index 1f364ca7..84384743 100644 --- a/objstore.go +++ b/objstore.go @@ -75,7 +75,7 @@ type BucketReader interface { // returned by the underlying provider. Attributes can be requested using various IterOption's. // // Entries are passed to function in sorted order. - Iter(ctx context.Context, dir string, f func(name string, attrs ObjectAttributes) error, options ...IterOption) error + Iter(ctx context.Context, dir string, f func(attrs ObjectAttributes) error, options ...IterOption) error // Get returns a reader for the given object name. Get(ctx context.Context, name string) (io.ReadCloser, error) @@ -194,6 +194,9 @@ func applyUploadOptions(options ...UploadOption) uploadParams { } type ObjectAttributes struct { + // Name is the object name. + Name string `json:"name"` + // Size is the object size in bytes. Size int64 `json:"size"` @@ -359,10 +362,10 @@ func DownloadDir(ctx context.Context, logger log.Logger, bkt BucketReader, origi var downloadedFiles []string var m sync.Mutex - err := bkt.Iter(ctx, src, func(name string, _ ObjectAttributes) error { + err := bkt.Iter(ctx, src, func(attrs ObjectAttributes) error { g.Go(func() error { - dst := filepath.Join(dst, filepath.Base(name)) - if strings.HasSuffix(name, DirDelim) { + dst := filepath.Join(dst, filepath.Base(attrs.Name)) + if strings.HasSuffix(attrs.Name, DirDelim) { if err := DownloadDir(ctx, logger, bkt, originalSrc, name, dst, options...); err != nil { return err } @@ -543,7 +546,7 @@ func (b *metricBucket) ReaderWithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket return b.WithExpectedErrs(fn) } -func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string, _ ObjectAttributes) error, options ...IterOption) error { +func (b *metricBucket) Iter(ctx context.Context, dir string, f func(attrs ObjectAttributes) error, options ...IterOption) error { const op = OpIter b.metrics.ops.WithLabelValues(op).Inc() diff --git a/prefixed_bucket.go b/prefixed_bucket.go index 8ce69461..a848bac4 100644 --- a/prefixed_bucket.go +++ b/prefixed_bucket.go @@ -46,7 +46,7 @@ func (p *PrefixedBucket) Close() error { // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full // object name including the prefix of the inspected directory. // Entries are passed to function in sorted order. -func (p *PrefixedBucket) Iter(ctx context.Context, dir string, f func(name string, attrs ObjectAttributes) error, options ...IterOption) error { +func (p *PrefixedBucket) Iter(ctx context.Context, dir string, f func(attrs ObjectAttributes) error, options ...IterOption) error { pdir := withPrefix(p.prefix, dir) return p.bkt.Iter(ctx, pdir, func(s string, _ ObjectAttributes) error { diff --git a/providers/azure/azure.go b/providers/azure/azure.go index d18ea5aa..2b13b069 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -195,7 +195,7 @@ func NewBucketWithConfig(logger log.Logger, conf Config, component string, wrapR // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, _ objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { prefix := dir if prefix != "" && !strings.HasSuffix(prefix, DirDelim) { prefix += DirDelim diff --git a/providers/bos/bos.go b/providers/bos/bos.go index 0a223730..07740d96 100644 --- a/providers/bos/bos.go +++ b/providers/bos/bos.go @@ -178,7 +178,7 @@ func (b *Bucket) Upload(_ context.Context, name string, r io.Reader) error { // Iter calls f for each entry in the given directory (not recursive). The argument to f is the full // object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, _ objstore.ObjectAttributes) error, opt ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { if dir != "" { dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim } diff --git a/providers/cos/cos.go b/providers/cos/cos.go index fa9eb8eb..f28eb26c 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -278,7 +278,7 @@ func (b *Bucket) Delete(ctx context.Context, name string) error { // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full // object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { if dir != "" { dir = strings.TrimSuffix(dir, dirDelim) + dirDelim } diff --git a/providers/filesystem/filesystem.go b/providers/filesystem/filesystem.go index 0522fad8..473a72de 100644 --- a/providers/filesystem/filesystem.go +++ b/providers/filesystem/filesystem.go @@ -52,7 +52,7 @@ func NewBucket(rootDir string) (*Bucket, error) { // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { if ctx.Err() != nil { return ctx.Err() } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 37ae4cf9..aacd572c 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -183,7 +183,7 @@ func (b *Bucket) Name() string { // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the // object itself as one prefix item. if dir != "" { diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 1cb5c699..f991e9f9 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -233,7 +233,7 @@ func (b *Bucket) multipartUpload(size int64, key, uploadId string, body io.Reade func (b *Bucket) Close() error { return nil } // Iter calls f for each entry in the given directory (not recursive.) -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, _ objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { if dir != "" { dir = strings.TrimSuffix(dir, DirDelim) + DirDelim } diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 0349047d..eb85ad81 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -103,7 +103,7 @@ func (b *Bucket) Name() string { // Iter calls f for each entry in the given directory (not recursive). The argument to f is the full // object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, _ objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the // object itself as one prefix item. if dir != "" { diff --git a/providers/oss/oss.go b/providers/oss/oss.go index 9ef01751..70c864a0 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -218,7 +218,7 @@ func validate(config Config) error { // Iter calls f for each entry in the given directory (not recursive). The argument to f is the full // object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, _ objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { if dir != "" { dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index f7298988..5e018c2f 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -389,7 +389,7 @@ func ValidateForTests(conf Config) error { // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. -func (b *Bucket) Iter(ctx context.Context, dir string, f func(name string, attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (b *Bucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the // object itself as one prefix item. if dir != "" { diff --git a/providers/swift/swift.go b/providers/swift/swift.go index 9b51cfbb..0bde791a 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -224,7 +224,7 @@ func (c *Container) Name() string { // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. -func (c *Container) Iter(ctx context.Context, dir string, f func(name string, _ objstore.ObjectAttributes) error, options ...objstore.IterOption) error { +func (c *Container) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) error { if dir != "" { dir = strings.TrimSuffix(dir, string(DirDelim)) + string(DirDelim) } diff --git a/testing.go b/testing.go index 6f987b88..0acffbf0 100644 --- a/testing.go +++ b/testing.go @@ -290,7 +290,7 @@ func (d *delayingBucket) Attributes(ctx context.Context, name string) (ObjectAtt return d.bkt.Attributes(ctx, name) } -func (d *delayingBucket) Iter(ctx context.Context, dir string, f func(name string, attrs ObjectAttributes) error, options ...IterOption) error { +func (d *delayingBucket) Iter(ctx context.Context, dir string, f func(attrs ObjectAttributes) error, options ...IterOption) error { time.Sleep(d.delay) return d.bkt.Iter(ctx, dir, f, options...) } diff --git a/tracing/opentelemetry/opentelemetry.go b/tracing/opentelemetry/opentelemetry.go index d6dd1879..f49e9f15 100644 --- a/tracing/opentelemetry/opentelemetry.go +++ b/tracing/opentelemetry/opentelemetry.go @@ -23,7 +23,7 @@ func WrapWithTraces(bkt objstore.Bucket, tracer trace.Tracer) objstore.Instrumen return TracingBucket{tracer: tracer, bkt: bkt} } -func (t TracingBucket) Iter(ctx context.Context, dir string, f func(name string, attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) (err error) { +func (t TracingBucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) (err error) { ctx, span := t.tracer.Start(ctx, "bucket_iter") defer span.End() span.SetAttributes(attribute.String("dir", dir)) diff --git a/tracing/opentracing/opentracing.go b/tracing/opentracing/opentracing.go index 111a0c7d..7eeebc01 100644 --- a/tracing/opentracing/opentracing.go +++ b/tracing/opentracing/opentracing.go @@ -44,7 +44,7 @@ func WrapWithTraces(bkt objstore.Bucket) objstore.InstrumentedBucket { return TracingBucket{bkt: bkt} } -func (t TracingBucket) Iter(ctx context.Context, dir string, f func(name string, _ objstore.ObjectAttributes) error, options ...objstore.IterOption) (err error) { +func (t TracingBucket) Iter(ctx context.Context, dir string, f func(attrs objstore.ObjectAttributes) error, options ...objstore.IterOption) (err error) { doWithSpan(ctx, "bucket_iter", func(spanCtx context.Context, span opentracing.Span) { span.LogKV("dir", dir) err = t.bkt.Iter(spanCtx, dir, f, options...)