From e7e894dd7f47dfcd7dde1765ccb347f76bd9a35a Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Tue, 19 Mar 2019 11:02:39 +0100 Subject: [PATCH 1/7] Add Put Function without MultiPart Upload --- server/storage.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/server/storage.go b/server/storage.go index f8125d2..b938482 100644 --- a/server/storage.go +++ b/server/storage.go @@ -27,6 +27,7 @@ import ( type Storage interface { Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) Head(token string, filename string) (contentType string, contentLength uint64, err error) + PutMulti(token string, filename string, reader io.Reader, contentType string, contentLength uint64) error Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) error Delete(token string, filename string) error IsNotExist(err error) bool @@ -150,7 +151,6 @@ func (s *S3Storage) Head(token string, filename string) (contentType string, con if err != nil { return } - contentType = response.Header.Get("Content-Type") contentLength, err = strconv.ParseUint(response.Header.Get("Content-Length"), 10, 0) @@ -202,7 +202,7 @@ func (s *S3Storage) Delete(token string, filename string) (err error) { return } -func (s *S3Storage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { +func (s *S3Storage) PutMulti(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { key := fmt.Sprintf("%s/%s", token, filename) var ( @@ -316,6 +316,22 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content return } +func (s *S3Storage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { + key := fmt.Sprintf("%s/%s", token, filename) + + s.logger.Printf("Uploading file %s to S3 Bucket", filename) + + err = s.bucket.PutReader(key, reader, contentType, s3.Private, s3.Options{}) + + if err != nil { + return + } + + s.logger.Printf("Completed uploading %s", filename) + + return +} + type GDrive struct { service *drive.Service rootId string From 7be8c21044c89c35df824abec717da97c64207be Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Tue, 19 Mar 2019 11:35:30 +0100 Subject: [PATCH 2/7] Add CLI Flag and update Function Flow --- cmd/cmd.go | 10 ++++++---- server/storage.go | 26 ++++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 348c05c..6e97180 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -2,15 +2,13 @@ package cmd import ( "fmt" - + "log" "os" - "strings" "github.com/dutchcoders/transfer.sh/server" "github.com/fatih/color" "github.com/minio/cli" - "log" "google.golang.org/api/googleapi" ) @@ -117,6 +115,10 @@ var globalFlags = []cli.Flag{ Value: "", EnvVar: "BUCKET", }, + cli.BoolFlag{ + Name: "no-multipart", + Usage: "Disables Multipart Puts", + }, cli.StringFlag{ Name: "gdrive-client-json-filepath", Usage: "", @@ -294,7 +296,7 @@ func New() *Cmd { panic("secret-key not set.") } else if bucket := c.String("bucket"); bucket == "" { panic("bucket not set.") - } else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger); err != nil { + } else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger, c.Bool("no-multipart")); err != nil { panic(err) } else { options = append(options, server.UseStorage(storage)) diff --git a/server/storage.go b/server/storage.go index b938482..85169c9 100644 --- a/server/storage.go +++ b/server/storage.go @@ -2,32 +2,30 @@ package server import ( "bytes" + "encoding/json" "fmt" "io" + "io/ioutil" "log" "mime" + "net/http" "os" "path/filepath" "strconv" + "strings" "sync" - "encoding/json" "github.com/goamz/goamz/s3" - "golang.org/x/net/context" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" - "io/ioutil" - "net/http" - "strings" ) type Storage interface { Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) Head(token string, filename string) (contentType string, contentLength uint64, err error) - PutMulti(token string, filename string, reader io.Reader, contentType string, contentLength uint64) error Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) error Delete(token string, filename string) error IsNotExist(err error) bool @@ -126,17 +124,18 @@ func (s *LocalStorage) Put(token string, filename string, reader io.Reader, cont type S3Storage struct { Storage - bucket *s3.Bucket - logger *log.Logger + bucket *s3.Bucket + logger *log.Logger + noMultipart bool } -func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger) (*S3Storage, error) { +func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, multipart bool) (*S3Storage, error) { bucket, err := getBucket(accessKey, secretKey, bucketName, endpoint) if err != nil { return nil, err } - return &S3Storage{bucket: bucket, logger: logger}, nil + return &S3Storage{bucket: bucket, logger: logger, noMultipart: multipart}, nil } func (s *S3Storage) Type() string { @@ -320,8 +319,11 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content key := fmt.Sprintf("%s/%s", token, filename) s.logger.Printf("Uploading file %s to S3 Bucket", filename) - - err = s.bucket.PutReader(key, reader, contentType, s3.Private, s3.Options{}) + if s.noMultipart { + err = s.PutMulti(token, filename, reader, contentType, contentLength) + } else { + err = s.bucket.PutReader(key, reader, int64(contentLength), contentType, s3.Private, s3.Options{}) + } if err != nil { return From 9f7a123047dfa6bd6e374bbd25ab2a1e0e20c5c8 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Tue, 19 Mar 2019 13:57:34 +0100 Subject: [PATCH 3/7] Add Goland Exceptions --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ca60910..fc27d44 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ src/ bin/ *.pyc *.egg-info/ +.idea/ .tmp .vagrant From f644a930dc7dcbb454dca56ec5ba74957dfc9d85 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Tue, 19 Mar 2019 18:48:28 +0100 Subject: [PATCH 4/7] Refactor and update Readme --- README.md | 1 + cmd/cmd.go | 6 +++--- server/storage.go | 5 ++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ecf694d..f12de8f 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ provider | which storage provider to use | (s3, grdrive or local) | aws-access-key | aws access key | | AWS_ACCESS_KEY aws-secret-key | aws access key | | AWS_SECRET_KEY bucket | aws bucket | | BUCKET +s3-no-multipart | disables s3 multipart upload | false | | basedir | path storage for local/gdrive provider| | gdrive-client-json-filepath | path to oauth client json config for gdrive provider| | gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider| | diff --git a/cmd/cmd.go b/cmd/cmd.go index 6e97180..d6ba374 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -116,8 +116,8 @@ var globalFlags = []cli.Flag{ EnvVar: "BUCKET", }, cli.BoolFlag{ - Name: "no-multipart", - Usage: "Disables Multipart Puts", + Name: "s3-no-multipart", + Usage: "Disables S3 Multipart Puts", }, cli.StringFlag{ Name: "gdrive-client-json-filepath", @@ -296,7 +296,7 @@ func New() *Cmd { panic("secret-key not set.") } else if bucket := c.String("bucket"); bucket == "" { panic("bucket not set.") - } else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger, c.Bool("no-multipart")); err != nil { + } else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger, c.Bool("s3-no-multipart")); err != nil { panic(err) } else { options = append(options, server.UseStorage(storage)) diff --git a/server/storage.go b/server/storage.go index 85169c9..5f8425c 100644 --- a/server/storage.go +++ b/server/storage.go @@ -201,8 +201,7 @@ func (s *S3Storage) Delete(token string, filename string) (err error) { return } -func (s *S3Storage) PutMulti(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { - key := fmt.Sprintf("%s/%s", token, filename) +func (s *S3Storage) putMulti(key string, reader io.Reader, contentType string, contentLength uint64) (err error) { var ( multi *s3.Multi @@ -320,7 +319,7 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content s.logger.Printf("Uploading file %s to S3 Bucket", filename) if s.noMultipart { - err = s.PutMulti(token, filename, reader, contentType, contentLength) + err = s.putMulti(key, reader, contentType, contentLength) } else { err = s.bucket.PutReader(key, reader, int64(contentLength), contentType, s3.Private, s3.Options{}) } From f3e49e39133e5dc19a960027b3ced0222af7da75 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Tue, 19 Mar 2019 21:20:15 +0100 Subject: [PATCH 5/7] Revert Logic, due to refactoring --- server/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/storage.go b/server/storage.go index 5f8425c..d0fa59f 100644 --- a/server/storage.go +++ b/server/storage.go @@ -318,7 +318,7 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content key := fmt.Sprintf("%s/%s", token, filename) s.logger.Printf("Uploading file %s to S3 Bucket", filename) - if s.noMultipart { + if !s.noMultipart { err = s.putMulti(key, reader, contentType, contentLength) } else { err = s.bucket.PutReader(key, reader, int64(contentLength), contentType, s3.Private, s3.Options{}) From f448fc7206ff4bf15f918f2d2b34ecb4254b18d0 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Mon, 25 Mar 2019 18:03:16 +0100 Subject: [PATCH 6/7] Variable Naming --- server/storage.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/storage.go b/server/storage.go index d0fa59f..f348497 100644 --- a/server/storage.go +++ b/server/storage.go @@ -129,13 +129,13 @@ type S3Storage struct { noMultipart bool } -func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, multipart bool) (*S3Storage, error) { +func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, nomultipart bool) (*S3Storage, error) { bucket, err := getBucket(accessKey, secretKey, bucketName, endpoint) if err != nil { return nil, err } - return &S3Storage{bucket: bucket, logger: logger, noMultipart: multipart}, nil + return &S3Storage{bucket: bucket, logger: logger, noMultipart: nomultipart}, nil } func (s *S3Storage) Type() string { From bb6edbdacadc62fb4dc7ca0a34a84d26c2d6fc35 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Mon, 25 Mar 2019 18:04:26 +0100 Subject: [PATCH 7/7] Renaming --- server/storage.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/storage.go b/server/storage.go index f348497..eb0daaf 100644 --- a/server/storage.go +++ b/server/storage.go @@ -129,13 +129,13 @@ type S3Storage struct { noMultipart bool } -func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, nomultipart bool) (*S3Storage, error) { +func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, disableMultipart bool) (*S3Storage, error) { bucket, err := getBucket(accessKey, secretKey, bucketName, endpoint) if err != nil { return nil, err } - return &S3Storage{bucket: bucket, logger: logger, noMultipart: nomultipart}, nil + return &S3Storage{bucket: bucket, logger: logger, noMultipart: disableMultipart}, nil } func (s *S3Storage) Type() string {