Browse Source

Merge pull request #201 from stefanbenten/sb/put-full

Add second S3 Handler for Non-Multipart Uploads
tags/v1.0.0
Andrea Spacca 5 years ago
committed by GitHub
parent
commit
058f6e581c
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 37 additions and 16 deletions
  1. +1
    -0
      .gitignore
  2. +1
    -0
      README.md
  3. +6
    -4
      cmd/cmd.go
  4. +29
    -12
      server/storage.go

+ 1
- 0
.gitignore View File

@@ -5,6 +5,7 @@ src/
bin/
*.pyc
*.egg-info/
.idea/

.tmp
.vagrant


+ 1
- 0
README.md View File

@@ -126,6 +126,7 @@ provider | which storage provider to use | (s3, grdrive or local) |
aws-access-key | aws access key | | AWS_ACCESS_KEY
aws-secret-key | aws access key | | AWS_SECRET_KEY
bucket | aws bucket | | BUCKET
s3-no-multipart | disables s3 multipart upload | false | |
basedir | path storage for local/gdrive provider| |
gdrive-client-json-filepath | path to oauth client json config for gdrive provider| |
gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider| |


+ 6
- 4
cmd/cmd.go View File

@@ -2,15 +2,13 @@ package cmd

import (
"fmt"
"log"
"os"

"strings"

"github.com/dutchcoders/transfer.sh/server"
"github.com/fatih/color"
"github.com/minio/cli"
"log"
"google.golang.org/api/googleapi"
)

@@ -117,6 +115,10 @@ var globalFlags = []cli.Flag{
Value: "",
EnvVar: "BUCKET",
},
cli.BoolFlag{
Name: "s3-no-multipart",
Usage: "Disables S3 Multipart Puts",
},
cli.StringFlag{
Name: "gdrive-client-json-filepath",
Usage: "",
@@ -294,7 +296,7 @@ func New() *Cmd {
panic("secret-key not set.")
} else if bucket := c.String("bucket"); bucket == "" {
panic("bucket not set.")
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger); err != nil {
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger, c.Bool("s3-no-multipart")); err != nil {
panic(err)
} else {
options = append(options, server.UseStorage(storage))


+ 29
- 12
server/storage.go View File

@@ -2,26 +2,25 @@ package server

import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"mime"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"

"encoding/json"
"github.com/goamz/goamz/s3"

"golang.org/x/net/context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
"io/ioutil"
"net/http"
"strings"
)

type Storage interface {
@@ -125,17 +124,18 @@ func (s *LocalStorage) Put(token string, filename string, reader io.Reader, cont

type S3Storage struct {
Storage
bucket *s3.Bucket
logger *log.Logger
bucket *s3.Bucket
logger *log.Logger
noMultipart bool
}

func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger) (*S3Storage, error) {
func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, disableMultipart bool) (*S3Storage, error) {
bucket, err := getBucket(accessKey, secretKey, bucketName, endpoint)
if err != nil {
return nil, err
}

return &S3Storage{bucket: bucket, logger: logger}, nil
return &S3Storage{bucket: bucket, logger: logger, noMultipart: disableMultipart}, nil
}

func (s *S3Storage) Type() string {
@@ -150,7 +150,6 @@ func (s *S3Storage) Head(token string, filename string) (contentType string, con
if err != nil {
return
}

contentType = response.Header.Get("Content-Type")

contentLength, err = strconv.ParseUint(response.Header.Get("Content-Length"), 10, 0)
@@ -202,8 +201,7 @@ func (s *S3Storage) Delete(token string, filename string) (err error) {
return
}

func (s *S3Storage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
key := fmt.Sprintf("%s/%s", token, filename)
func (s *S3Storage) putMulti(key string, reader io.Reader, contentType string, contentLength uint64) (err error) {

var (
multi *s3.Multi
@@ -316,6 +314,25 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content
return
}

func (s *S3Storage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
key := fmt.Sprintf("%s/%s", token, filename)

s.logger.Printf("Uploading file %s to S3 Bucket", filename)
if !s.noMultipart {
err = s.putMulti(key, reader, contentType, contentLength)
} else {
err = s.bucket.PutReader(key, reader, int64(contentLength), contentType, s3.Private, s3.Options{})
}

if err != nil {
return
}

s.logger.Printf("Completed uploading %s", filename)

return
}

type GDrive struct {
service *drive.Service
rootId string


Loading…
Cancel
Save