diff --git a/cmd/client/main.go b/cmd/client/main.go index 3e0a9ed..af9e309 100644 --- a/cmd/client/main.go +++ b/cmd/client/main.go @@ -24,9 +24,11 @@ var ( decrypt bool /* Put command flags */ - encrypt bool - storageClass string - tableSpace string + encrypt bool + storageClass string + tableSpace string + multipart_chunksize int64 + multipart_threshold int64 offset uint64 @@ -115,6 +117,14 @@ func putFunc(con net.Conn, instanceCnf *config.Instance, args []string) error { Name: message.TableSpaceSetting, Value: tableSpace, }, + { + Name: message.MultipartChunksize, + Value: fmt.Sprintf("%s", multipart_chunksize), + }, + { + Name: message.MultipartThreshold, + Value: fmt.Sprintf("%s", multipart_threshold), + }, }).Encode() _, err := con.Write(msg) if err != nil { diff --git a/pkg/message/put_message_v2.go b/pkg/message/put_message_v2.go index 28fc247..48c3519 100644 --- a/pkg/message/put_message_v2.go +++ b/pkg/message/put_message_v2.go @@ -8,6 +8,8 @@ import ( const StorageClassSetting = "StorageClass" const TableSpaceSetting = "TableSpace" +const MultipartChunksize = "MultipartChunksize" +const MultipartThreshold = "MultipartThreshold" type PutMessageV2 struct { Encrypt bool diff --git a/pkg/storage/s3storage.go b/pkg/storage/s3storage.go index 5ab15cf..131cd5b 100644 --- a/pkg/storage/s3storage.go +++ b/pkg/storage/s3storage.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "path" + "strconv" "strings" "github.com/yezzey-gp/aws-sdk-go/aws" @@ -69,14 +70,19 @@ func (s *S3StorageInteractor) PutFileToDest(name string, r io.Reader, settings [ objectPath := path.Join(s.cnf.StoragePrefix, name) + storageClass := ResolveStorageSetting(settings, message.StorageClassSetting, "STANDARD") + tableSpace := ResolveStorageSetting(settings, message.TableSpaceSetting, tablespace.DefaultTableSpace) + multipartChunksizeStr := ResolveStorageSetting(settings, message.MultipartChunksize, "") + multipartChunksize, err := strconv.ParseInt(multipartChunksizeStr, 10, 64) + if err != nil { + return err + } + up := s3manager.NewUploaderWithClient(sess, func(uploader *s3manager.Uploader) { - uploader.PartSize = int64(1 << 24) + uploader.PartSize = int64(multipartChunksize) uploader.Concurrency = 1 }) - storageClass := ResolveStorageSetting(settings, message.StorageClassSetting, "STANDARD") - tableSpace := ResolveStorageSetting(settings, message.TableSpaceSetting, tablespace.DefaultTableSpace) - bucket, ok := s.bucketMap[tableSpace] if !ok { err := fmt.Errorf("failed to match tablespace %s to s3 bucket.", tableSpace)