diff --git a/.gitignore b/.gitignore index ca60910..fc27d44 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ src/ bin/ *.pyc *.egg-info/ +.idea/ .tmp .vagrant diff --git a/README.md b/README.md index ecf694d..f12de8f 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ provider | which storage provider to use | (s3, grdrive or local) | aws-access-key | aws access key | | AWS_ACCESS_KEY aws-secret-key | aws access key | | AWS_SECRET_KEY bucket | aws bucket | | BUCKET +s3-no-multipart | disables s3 multipart upload | false | | basedir | path storage for local/gdrive provider| | gdrive-client-json-filepath | path to oauth client json config for gdrive provider| | gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider| | diff --git a/cmd/cmd.go b/cmd/cmd.go index 348c05c..d6ba374 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -2,15 +2,13 @@ package cmd import ( "fmt" - + "log" "os" - "strings" "github.com/dutchcoders/transfer.sh/server" "github.com/fatih/color" "github.com/minio/cli" - "log" "google.golang.org/api/googleapi" ) @@ -117,6 +115,10 @@ var globalFlags = []cli.Flag{ Value: "", EnvVar: "BUCKET", }, + cli.BoolFlag{ + Name: "s3-no-multipart", + Usage: "Disables S3 Multipart Puts", + }, cli.StringFlag{ Name: "gdrive-client-json-filepath", Usage: "", @@ -294,7 +296,7 @@ func New() *Cmd { panic("secret-key not set.") } else if bucket := c.String("bucket"); bucket == "" { panic("bucket not set.") - } else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger); err != nil { + } else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger, c.Bool("s3-no-multipart")); err != nil { panic(err) } else { options = append(options, server.UseStorage(storage)) diff --git a/server/storage.go b/server/storage.go index f8125d2..eb0daaf 100644 --- a/server/storage.go +++ b/server/storage.go @@ -2,26 +2,25 @@ package server import ( "bytes" + "encoding/json" "fmt" "io" + "io/ioutil" "log" "mime" + "net/http" "os" "path/filepath" "strconv" + "strings" "sync" - "encoding/json" "github.com/goamz/goamz/s3" - "golang.org/x/net/context" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" - "io/ioutil" - "net/http" - "strings" ) type Storage interface { @@ -125,17 +124,18 @@ func (s *LocalStorage) Put(token string, filename string, reader io.Reader, cont type S3Storage struct { Storage - bucket *s3.Bucket - logger *log.Logger + bucket *s3.Bucket + logger *log.Logger + noMultipart bool } -func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger) (*S3Storage, error) { +func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, disableMultipart bool) (*S3Storage, error) { bucket, err := getBucket(accessKey, secretKey, bucketName, endpoint) if err != nil { return nil, err } - return &S3Storage{bucket: bucket, logger: logger}, nil + return &S3Storage{bucket: bucket, logger: logger, noMultipart: disableMultipart}, nil } func (s *S3Storage) Type() string { @@ -150,7 +150,6 @@ func (s *S3Storage) Head(token string, filename string) (contentType string, con if err != nil { return } - contentType = response.Header.Get("Content-Type") contentLength, err = strconv.ParseUint(response.Header.Get("Content-Length"), 10, 0) @@ -202,8 +201,7 @@ func (s *S3Storage) Delete(token string, filename string) (err error) { return } -func (s *S3Storage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { - key := fmt.Sprintf("%s/%s", token, filename) +func (s *S3Storage) putMulti(key string, reader io.Reader, contentType string, contentLength uint64) (err error) { var ( multi *s3.Multi @@ -316,6 +314,25 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content return } +func (s *S3Storage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { + key := fmt.Sprintf("%s/%s", token, filename) + + s.logger.Printf("Uploading file %s to S3 Bucket", filename) + if !s.noMultipart { + err = s.putMulti(key, reader, contentType, contentLength) + } else { + err = s.bucket.PutReader(key, reader, int64(contentLength), contentType, s3.Private, s3.Options{}) + } + + if err != nil { + return + } + + s.logger.Printf("Completed uploading %s", filename) + + return +} + type GDrive struct { service *drive.Service rootId string