Skip to content

Commit

Permalink
Add filestorage, update interface, fix typo error
Browse files Browse the repository at this point in the history
  • Loading branch information
visill committed Aug 5, 2024
1 parent edeb821 commit e4c2682
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 6 deletions.
3 changes: 3 additions & 0 deletions config/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,7 @@ type Storage struct {
StorageConcurrency int64 `json:"storage_concurrency" toml:"storage_concurrency" yaml:"storage_concurrency"`

StorageRegion string `json:"storage_region" toml:"storage_region" yaml:"storage_region"`

// File storage default s3. Available: s3, fs
StorageType string `json:"storage_type" toml:"storage_type" yaml:"storage_type"`
}
71 changes: 71 additions & 0 deletions pkg/storage/filestorage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package storage

import (
"fmt"
"io"
"io/fs"
"os"
"path/filepath"

"github.com/yezzey-gp/yproxy/config"
)

// Storage prefix uses as path to folder.
// "/path/to/folder/" + "path/to/file.txt"
type FileStorageInteractor struct {
StorageInteractor
cnf *config.Storage
}

func (s *FileStorageInteractor) CatFileFromStorage(name string, offset int64) (io.ReadCloser, error) {
file, err := os.Open(s.cnf.StoragePrefix + name)
if err != nil {
return nil, err
}
_, err = io.CopyN(io.Discard, file, offset)
return file, err
}
func (s *FileStorageInteractor) ListPath(prefix string) ([]*ObjectInfo, error) {
var data []*ObjectInfo
err := filepath.WalkDir(s.cnf.StoragePrefix+prefix, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
file, err := os.Open(path)
if err != nil {
return err
}
fileinfo, err := file.Stat()
if err != nil {
return err
}
data = append(data, &ObjectInfo{fileinfo.Name(), fileinfo.Size()})
return nil
})
return data, err
}

func (s *FileStorageInteractor) PutFileToDest(name string, r io.Reader) error {
file, err := os.Create(s.cnf.StoragePrefix + name)
if err != nil {
return err
}
_, err = io.Copy(file, r)
return err
}

func (s *FileStorageInteractor) PatchFile(name string, r io.ReadSeeker, startOffset int64) error {
//UNUSED TODO
return fmt.Errorf("TODO")
}

func (s *FileStorageInteractor) MoveObject(from string, to string) error {
return os.Rename(s.cnf.StoragePrefix+from, s.cnf.StoragePrefix+to)
}

func (s *FileStorageInteractor) DeleteObject(key string) error {
return os.Remove(s.cnf.StoragePrefix + key)
}
4 changes: 2 additions & 2 deletions pkg/storage/s3storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func (s *S3StorageInteractor) PutFileToDest(name string, r io.Reader) error {
return err
}

func (s *S3StorageInteractor) PatchFile(name string, r io.ReadSeeker, startOffste int64) error {
func (s *S3StorageInteractor) PatchFile(name string, r io.ReadSeeker, startOffset int64) error {
sess, err := s.pool.GetSession(context.TODO())
if err != nil {
ylogger.Zero.Err(err).Msg("failed to acquire s3 session")
Expand All @@ -82,7 +82,7 @@ func (s *S3StorageInteractor) PatchFile(name string, r io.ReadSeeker, startOffst
Bucket: &s.cnf.StorageBucket,
Key: aws.String(objectPath),
Body: r,
ContentRange: aws.String(fmt.Sprintf("bytes %d-18446744073709551615", startOffste)),
ContentRange: aws.String(fmt.Sprintf("bytes %d-18446744073709551615", startOffset)),
}

_, err = sess.PatchObject(input)
Expand Down
20 changes: 16 additions & 4 deletions pkg/storage/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,34 @@ type StorageReader interface {

type StorageWriter interface {
PutFileToDest(name string, r io.Reader) error
PatchFile(name string, r io.ReadSeeker, startOffste int64) error
PatchFile(name string, r io.ReadSeeker, startOffset int64) error
}

type StorageLister interface {
ListPath(prefix string) ([]*ObjectInfo, error)
}

type StorageMover interface {
MoveObject(from string, to string) error
DeleteObject(key string) error
}
type StorageInteractor interface {
StorageReader
StorageWriter
StorageLister
StorageMover
}

func NewStorage(cnf *config.Storage) StorageInteractor {
return &S3StorageInteractor{
pool: NewSessionPool(cnf),
cnf: cnf,
switch cnf.StorageType {
case "fs":
return &FileStorageInteractor{
cnf: cnf,
}
default:
return &S3StorageInteractor{
pool: NewSessionPool(cnf),
cnf: cnf,
}
}
}

0 comments on commit e4c2682

Please sign in to comment.