@@ -7,6 +7,7 @@ import ( | |||||
"strings" | "strings" | ||||
"github.com/dutchcoders/transfer.sh/server" | "github.com/dutchcoders/transfer.sh/server" | ||||
"github.com/dutchcoders/transfer.sh/server/storage" | |||||
"github.com/fatih/color" | "github.com/fatih/color" | ||||
"github.com/urfave/cli" | "github.com/urfave/cli" | ||||
"google.golang.org/api/googleapi" | "google.golang.org/api/googleapi" | ||||
@@ -353,7 +354,7 @@ func New() *Cmd { | |||||
panic("secret-key not set.") | panic("secret-key not set.") | ||||
} else if bucket := c.String("bucket"); bucket == "" { | } else if bucket := c.String("bucket"); bucket == "" { | ||||
panic("bucket not set.") | panic("bucket not set.") | ||||
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-region"), c.String("s3-endpoint"), logger, c.Bool("s3-no-multipart"), c.Bool("s3-path-style")); err != nil { | |||||
} else if storage, err := storage.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-region"), c.String("s3-endpoint"), logger, c.Bool("s3-no-multipart"), c.Bool("s3-path-style")); err != nil { | |||||
panic(err) | panic(err) | ||||
} else { | } else { | ||||
options = append(options, server.UseStorage(storage)) | options = append(options, server.UseStorage(storage)) | ||||
@@ -367,7 +368,7 @@ func New() *Cmd { | |||||
panic("local-config-path not set.") | panic("local-config-path not set.") | ||||
} else if basedir := c.String("basedir"); basedir == "" { | } else if basedir := c.String("basedir"); basedir == "" { | ||||
panic("basedir not set.") | panic("basedir not set.") | ||||
} else if storage, err := server.NewGDriveStorage(clientJsonFilepath, localConfigPath, basedir, chunkSize, logger); err != nil { | |||||
} else if storage, err := storage.NewGDriveStorage(clientJsonFilepath, localConfigPath, basedir, chunkSize, logger); err != nil { | |||||
panic(err) | panic(err) | ||||
} else { | } else { | ||||
options = append(options, server.UseStorage(storage)) | options = append(options, server.UseStorage(storage)) | ||||
@@ -375,7 +376,7 @@ func New() *Cmd { | |||||
case "local": | case "local": | ||||
if v := c.String("basedir"); v == "" { | if v := c.String("basedir"); v == "" { | ||||
panic("basedir not set.") | panic("basedir not set.") | ||||
} else if storage, err := server.NewLocalStorage(v, logger); err != nil { | |||||
} else if storage, err := storage.NewLocalStorage(v, logger); err != nil { | |||||
panic(err) | panic(err) | ||||
} else { | } else { | ||||
options = append(options, server.UseStorage(storage)) | options = append(options, server.UseStorage(storage)) | ||||
@@ -32,9 +32,9 @@ import ( | |||||
"archive/zip" | "archive/zip" | ||||
"bytes" | "bytes" | ||||
"compress/gzip" | "compress/gzip" | ||||
"encoding/base64" | |||||
"errors" | "errors" | ||||
"fmt" | "fmt" | ||||
blackfriday "github.com/russross/blackfriday/v2" | |||||
"html" | "html" | ||||
html_template "html/template" | html_template "html/template" | ||||
"io" | "io" | ||||
@@ -42,6 +42,7 @@ import ( | |||||
"log" | "log" | ||||
"math/rand" | "math/rand" | ||||
"mime" | "mime" | ||||
"net" | |||||
"net/http" | "net/http" | ||||
"net/url" | "net/url" | ||||
"os" | "os" | ||||
@@ -53,12 +54,11 @@ import ( | |||||
text_template "text/template" | text_template "text/template" | ||||
"time" | "time" | ||||
"net" | |||||
"encoding/base64" | |||||
web "github.com/dutchcoders/transfer.sh-web" | web "github.com/dutchcoders/transfer.sh-web" | ||||
"github.com/dutchcoders/transfer.sh/server/storage" | |||||
"github.com/gorilla/mux" | "github.com/gorilla/mux" | ||||
"github.com/microcosm-cc/bluemonday" | "github.com/microcosm-cc/bluemonday" | ||||
blackfriday "github.com/russross/blackfriday/v2" | |||||
"github.com/skip2/go-qrcode" | "github.com/skip2/go-qrcode" | ||||
) | ) | ||||
@@ -335,8 +335,8 @@ func cleanTmpFile(f *os.File) { | |||||
} | } | ||||
} | } | ||||
func (s *Server) metadataForRequest(contentType string, contentLength int64, r *http.Request) Metadata { | |||||
metadata := Metadata{ | |||||
func (s *Server) metadataForRequest(contentType string, contentLength int64, r *http.Request) storage.Metadata { | |||||
metadata := storage.Metadata{ | |||||
ContentType: contentType, | ContentType: contentType, | ||||
ContentLength: contentLength, | ContentLength: contentLength, | ||||
MaxDate: time.Now().Add(s.lifetime), | MaxDate: time.Now().Add(s.lifetime), | ||||
@@ -522,23 +522,6 @@ func getURL(r *http.Request) *url.URL { | |||||
return u | return u | ||||
} | } | ||||
func (metadata Metadata) remainingLimitHeaderValues() (remainingDownloads, remainingDays string) { | |||||
if metadata.MaxDate.IsZero() { | |||||
remainingDays = "n/a" | |||||
} else { | |||||
timeDifference := metadata.MaxDate.Sub(time.Now()) | |||||
remainingDays = strconv.Itoa(int(timeDifference.Hours()/24) + 1) | |||||
} | |||||
if metadata.MaxDownloads == -1 { | |||||
remainingDownloads = "n/a" | |||||
} else { | |||||
remainingDownloads = strconv.Itoa(metadata.MaxDownloads - metadata.Downloads) | |||||
} | |||||
return remainingDownloads, remainingDays | |||||
} | |||||
func (s *Server) Lock(token, filename string) error { | func (s *Server) Lock(token, filename string) error { | ||||
key := path.Join(token, filename) | key := path.Join(token, filename) | ||||
@@ -558,13 +541,11 @@ func (s *Server) Unlock(token, filename string) error { | |||||
return nil | return nil | ||||
} | } | ||||
func (s *Server) CheckMetadata(token, filename string, increaseDownload bool) (Metadata, error) { | |||||
func (s *Server) CheckMetadata(token, filename string, increaseDownload bool) (metadata storage.Metadata, err error) { | |||||
s.Lock(token, filename) | s.Lock(token, filename) | ||||
defer s.Unlock(token, filename) | defer s.Unlock(token, filename) | ||||
var metadata Metadata | |||||
metadata, err := s.storage.Head(token, filename) | |||||
metadata, err = s.storage.Head(token, filename) | |||||
if s.storage.IsNotExist(err) { | if s.storage.IsNotExist(err) { | ||||
return metadata, nil | return metadata, nil | ||||
} else if err != nil { | } else if err != nil { | ||||
@@ -595,8 +576,6 @@ func (s *Server) CheckDeletionToken(deletionToken, token, filename string) error | |||||
s.Lock(token, filename) | s.Lock(token, filename) | ||||
defer s.Unlock(token, filename) | defer s.Unlock(token, filename) | ||||
var metadata Metadata | |||||
metadata, err := s.storage.Head(token, filename) | metadata, err := s.storage.Head(token, filename) | ||||
if s.storage.IsNotExist(err) { | if s.storage.IsNotExist(err) { | ||||
return nil | return nil | ||||
@@ -848,7 +827,7 @@ func (s *Server) headHandler(w http.ResponseWriter, r *http.Request) { | |||||
return | return | ||||
} | } | ||||
remainingDownloads, remainingDays := metadata.remainingLimitHeaderValues() | |||||
remainingDownloads, remainingDays := metadata.RemainingLimitHeaderValues() | |||||
w.Header().Set("Content-Type", metadata.ContentType) | w.Header().Set("Content-Type", metadata.ContentType) | ||||
w.Header().Set("Content-Length", strconv.FormatInt(metadata.ContentLength, 10)) | w.Header().Set("Content-Length", strconv.FormatInt(metadata.ContentLength, 10)) | ||||
@@ -892,7 +871,7 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) { | |||||
disposition = "attachment" | disposition = "attachment" | ||||
} | } | ||||
remainingDownloads, remainingDays := metadata.remainingLimitHeaderValues() | |||||
remainingDownloads, remainingDays := metadata.RemainingLimitHeaderValues() | |||||
w.Header().Set("Content-Type", metadata.ContentType) | w.Header().Set("Content-Type", metadata.ContentType) | ||||
w.Header().Set("Content-Length", strconv.FormatInt(metadata.ContentLength, 10)) | w.Header().Set("Content-Length", strconv.FormatInt(metadata.ContentLength, 10)) | ||||
@@ -25,6 +25,7 @@ THE SOFTWARE. | |||||
package server | package server | ||||
import ( | import ( | ||||
"context" | |||||
"errors" | "errors" | ||||
"log" | "log" | ||||
"math/rand" | "math/rand" | ||||
@@ -38,11 +39,10 @@ import ( | |||||
"syscall" | "syscall" | ||||
"time" | "time" | ||||
context "golang.org/x/net/context" | |||||
"github.com/PuerkitoBio/ghost/handlers" | "github.com/PuerkitoBio/ghost/handlers" | ||||
"github.com/VojtechVitek/ratelimit" | "github.com/VojtechVitek/ratelimit" | ||||
"github.com/VojtechVitek/ratelimit/memory" | "github.com/VojtechVitek/ratelimit/memory" | ||||
"github.com/dutchcoders/transfer.sh/server/storage" | |||||
"github.com/gorilla/mux" | "github.com/gorilla/mux" | ||||
_ "net/http/pprof" | _ "net/http/pprof" | ||||
@@ -183,7 +183,7 @@ func LifeTime(lifetime int) OptionFn { | |||||
} | } | ||||
} | } | ||||
func UseStorage(s Storage) OptionFn { | |||||
func UseStorage(s storage.Storage) OptionFn { | |||||
return func(srvr *Server) { | return func(srvr *Server) { | ||||
srvr.storage = s | srvr.storage = s | ||||
} | } | ||||
@@ -263,7 +263,7 @@ type Server struct { | |||||
rateLimitRequests int | rateLimitRequests int | ||||
storage Storage | |||||
storage storage.Storage | |||||
lifetime time.Duration | lifetime time.Duration | ||||
@@ -1,736 +0,0 @@ | |||||
package server | |||||
import ( | |||||
"bytes" | |||||
"encoding/json" | |||||
"fmt" | |||||
"io" | |||||
"io/ioutil" | |||||
"log" | |||||
"net/http" | |||||
"os" | |||||
"path/filepath" | |||||
"strconv" | |||||
"strings" | |||||
"time" | |||||
"github.com/aws/aws-sdk-go/aws" | |||||
"github.com/aws/aws-sdk-go/aws/awserr" | |||||
"github.com/aws/aws-sdk-go/aws/session" | |||||
"github.com/aws/aws-sdk-go/service/s3" | |||||
"github.com/aws/aws-sdk-go/service/s3/s3manager" | |||||
"golang.org/x/net/context" | |||||
"golang.org/x/oauth2" | |||||
"golang.org/x/oauth2/google" | |||||
"google.golang.org/api/drive/v3" | |||||
"google.golang.org/api/googleapi" | |||||
) | |||||
type Metadata struct { | |||||
// ContentType is the original uploading content type | |||||
ContentType string | |||||
// ContentLength contains the length of the actual object | |||||
ContentLength int64 | |||||
// Downloads is the actual number of downloads | |||||
Downloads int | |||||
// MaxDownloads contains the maximum numbers of downloads | |||||
MaxDownloads int | |||||
// MaxDate contains the max age of the file | |||||
MaxDate time.Time | |||||
// DeletionToken contains the token to match against for deletion | |||||
DeletionToken string | |||||
// Secret as knowledge to delete file | |||||
Secret string | |||||
} | |||||
type Storage interface { | |||||
Get(token string, filename string) (reader io.ReadCloser, metaData Metadata, err error) | |||||
Head(token string, filename string) (metadata Metadata, err error) | |||||
Meta(token string, filename string, metadata Metadata) error | |||||
Put(token string, filename string, reader io.Reader, metadata Metadata) error | |||||
Delete(token string, filename string) error | |||||
IsNotExist(err error) bool | |||||
DeleteExpired() error | |||||
Type() string | |||||
} | |||||
type LocalStorage struct { | |||||
Storage | |||||
basedir string | |||||
logger *log.Logger | |||||
} | |||||
func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) { | |||||
return &LocalStorage{basedir: basedir, logger: logger}, nil | |||||
} | |||||
func (s *LocalStorage) Type() string { | |||||
return "local" | |||||
} | |||||
func (s *LocalStorage) Get(token string, filename string) (reader io.ReadCloser, metadata Metadata, err error) { | |||||
path := filepath.Join(s.basedir, token, filename) | |||||
// content type , content length | |||||
reader, err = os.Open(path) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
metadata, err = s.Head(token, filename) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
return reader, metadata, nil | |||||
} | |||||
func (s *LocalStorage) Head(token string, filename string) (metadata Metadata, err error) { | |||||
path := filepath.Join(s.basedir, token, filename) | |||||
fi, err := os.Open(path) | |||||
if err != nil { | |||||
return | |||||
} | |||||
err = json.NewDecoder(fi).Decode(&metadata) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
return metadata, nil | |||||
} | |||||
func (s *LocalStorage) Meta(token string, filename string, metadata Metadata) error { | |||||
return s.putMetadata(token, filename, metadata) | |||||
} | |||||
func (s *LocalStorage) Put(token string, filename string, reader io.Reader, metadata Metadata) error { | |||||
err := s.putMetadata(token, filename, metadata) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
err = s.put(token, filename, reader) | |||||
if err != nil { | |||||
//Delete the metadata if the put failed | |||||
_ = s.Delete(token, fmt.Sprintf("%s.metadata", filename)) | |||||
} | |||||
return err | |||||
} | |||||
func (s *LocalStorage) put(token string, filename string, reader io.Reader) error { | |||||
var f io.WriteCloser | |||||
var err error | |||||
path := filepath.Join(s.basedir, token) | |||||
if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) { | |||||
return err | |||||
} | |||||
if f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600); err != nil { | |||||
return err | |||||
} | |||||
defer f.Close() | |||||
_, err = io.Copy(f, reader) | |||||
return err | |||||
} | |||||
func (s *LocalStorage) putMetadata(token string, filename string, metadata Metadata) error { | |||||
buffer := &bytes.Buffer{} | |||||
if err := json.NewEncoder(buffer).Encode(metadata); err != nil { | |||||
log.Printf("%s", err.Error()) | |||||
return err | |||||
} else if err := s.put(token, filename, buffer); err != nil { | |||||
log.Printf("%s", err.Error()) | |||||
return nil | |||||
} | |||||
return nil | |||||
} | |||||
func (s *LocalStorage) Delete(token string, filename string) (err error) { | |||||
metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename)) | |||||
_ = os.Remove(metadata) | |||||
path := filepath.Join(s.basedir, token, filename) | |||||
err = os.Remove(path) | |||||
return | |||||
} | |||||
func (s *LocalStorage) IsNotExist(err error) bool { | |||||
if err == nil { | |||||
return false | |||||
} | |||||
return os.IsNotExist(err) | |||||
} | |||||
func (s *LocalStorage) DeleteExpired() error { | |||||
return nil | |||||
} | |||||
type S3Storage struct { | |||||
Storage | |||||
bucket string | |||||
session *session.Session | |||||
s3 *s3.S3 | |||||
logger *log.Logger | |||||
noMultipart bool | |||||
} | |||||
func NewS3Storage(accessKey, secretKey, bucketName, region, endpoint string, logger *log.Logger, disableMultipart bool, forcePathStyle bool) (*S3Storage, error) { | |||||
sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle) | |||||
return &S3Storage{bucket: bucketName, s3: s3.New(sess), session: sess, logger: logger, noMultipart: disableMultipart}, nil | |||||
} | |||||
func (s *S3Storage) Type() string { | |||||
return "s3" | |||||
} | |||||
func (s *S3Storage) Head(token string, filename string) (metadata Metadata, err error) { | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
headRequest := &s3.HeadObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | |||||
// content type , content length | |||||
response, err := s.s3.HeadObject(headRequest) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
downloads, err := strconv.Atoi(*response.Metadata["downloads"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
maxdownloads, err := strconv.Atoi(*response.Metadata["maxDownloads"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
expires, err := time.Parse("2020-02-02 02:02:02", *response.Expires) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
metadata = Metadata{ | |||||
ContentType: "", | |||||
ContentLength: *response.ContentLength, | |||||
Downloads: downloads, | |||||
MaxDownloads: maxdownloads, | |||||
MaxDate: expires, | |||||
DeletionToken: *response.Metadata["deletionToken"], | |||||
Secret: *response.Metadata["deletionSecret"], | |||||
} | |||||
return metadata, nil | |||||
} | |||||
func (s *S3Storage) Meta(token string, filename string, metadata Metadata) error { | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
input := &s3.CopyObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
CopySource: aws.String(key), | |||||
Key: aws.String(key), | |||||
MetadataDirective: aws.String("REPLACE"), | |||||
Metadata: map[string]*string{ | |||||
"downloads": aws.String(strconv.Itoa(metadata.Downloads)), | |||||
"maxDownloads": aws.String(strconv.Itoa(metadata.MaxDownloads)), | |||||
"deletionToken": aws.String(metadata.DeletionToken), | |||||
"deletionSecret": aws.String(metadata.Secret), | |||||
}, | |||||
ContentType: aws.String(metadata.ContentType), | |||||
Expires: aws.Time(metadata.MaxDate), | |||||
} | |||||
_, err := s.s3.CopyObject(input) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
return nil | |||||
} | |||||
func (s *S3Storage) Get(token string, filename string) (reader io.ReadCloser, metadata Metadata, err error) { | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
getRequest := &s3.GetObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | |||||
response, err := s.s3.GetObject(getRequest) | |||||
if err != nil { | |||||
return | |||||
} | |||||
downloads, err := strconv.Atoi(*response.Metadata["downloads"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
maxdownloads, err := strconv.Atoi(*response.Metadata["maxDownloads"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
expires, err := time.Parse("2020-02-02 02:02:02", *response.Expires) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
metadata = Metadata{ | |||||
ContentType: "", | |||||
ContentLength: *response.ContentLength, | |||||
Downloads: downloads, | |||||
MaxDownloads: maxdownloads, | |||||
MaxDate: expires, | |||||
DeletionToken: *response.Metadata["deletionToken"], | |||||
Secret: *response.Metadata["deletionSecret"], | |||||
} | |||||
reader = response.Body | |||||
return | |||||
} | |||||
func (s *S3Storage) Delete(token string, filename string) (err error) { | |||||
metadata := fmt.Sprintf("%s/%s.metadata", token, filename) | |||||
deleteRequest := &s3.DeleteObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(metadata), | |||||
} | |||||
_, err = s.s3.DeleteObject(deleteRequest) | |||||
if err != nil { | |||||
return | |||||
} | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
deleteRequest = &s3.DeleteObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | |||||
_, err = s.s3.DeleteObject(deleteRequest) | |||||
return | |||||
} | |||||
func (s *S3Storage) Put(token string, filename string, reader io.Reader, metadata Metadata) (err error) { | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
s.logger.Printf("Uploading file %s to S3 Bucket", filename) | |||||
var concurrency int | |||||
if !s.noMultipart { | |||||
concurrency = 20 | |||||
} else { | |||||
concurrency = 1 | |||||
} | |||||
// Create an uploader with the session and custom options | |||||
uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) { | |||||
u.Concurrency = concurrency // default is 5 | |||||
u.LeavePartsOnError = false | |||||
}) | |||||
_, err = uploader.Upload(&s3manager.UploadInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
Body: reader, | |||||
Metadata: map[string]*string{ | |||||
"downloads": aws.String(strconv.Itoa(metadata.Downloads)), | |||||
"maxDownloads": aws.String(strconv.Itoa(metadata.MaxDownloads)), | |||||
"deletionToken": aws.String(metadata.DeletionToken), | |||||
"deletionSecret": aws.String(metadata.Secret), | |||||
}, | |||||
ContentType: aws.String(metadata.ContentType), | |||||
Expires: aws.Time(metadata.MaxDate), | |||||
}) | |||||
return | |||||
} | |||||
func (s *S3Storage) IsNotExist(err error) bool { | |||||
if err == nil { | |||||
return false | |||||
} | |||||
if aerr, ok := err.(awserr.Error); ok { | |||||
switch aerr.Code() { | |||||
case s3.ErrCodeNoSuchKey: | |||||
return true | |||||
} | |||||
} | |||||
return false | |||||
} | |||||
func (s *S3Storage) DeleteExpired() error { | |||||
// not necessary, as S3 has expireDate on files to automatically delete the them | |||||
return nil | |||||
} | |||||
type GDrive struct { | |||||
service *drive.Service | |||||
rootId string | |||||
basedir string | |||||
localConfigPath string | |||||
chunkSize int | |||||
logger *log.Logger | |||||
} | |||||
func NewGDriveStorage(clientJsonFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) { | |||||
b, err := ioutil.ReadFile(clientJsonFilepath) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
// If modifying these scopes, delete your previously saved client_secret.json. | |||||
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
srv, err := drive.New(getGDriveClient(config, localConfigPath, logger)) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
chunkSize = chunkSize * 1024 * 1024 | |||||
storage := &GDrive{service: srv, basedir: basedir, rootId: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger} | |||||
err = storage.setupRoot() | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
return storage, nil | |||||
} | |||||
const GDriveRootConfigFile = "root_id.conf" | |||||
const GDriveTokenJsonFile = "token.json" | |||||
const GDriveDirectoryMimeType = "application/vnd.google-apps.folder" | |||||
func (s *GDrive) setupRoot() error { | |||||
rootFileConfig := filepath.Join(s.localConfigPath, GDriveRootConfigFile) | |||||
rootId, err := ioutil.ReadFile(rootFileConfig) | |||||
if err != nil && !os.IsNotExist(err) { | |||||
return err | |||||
} | |||||
if string(rootId) != "" { | |||||
s.rootId = string(rootId) | |||||
return nil | |||||
} | |||||
dir := &drive.File{ | |||||
Name: s.basedir, | |||||
MimeType: GDriveDirectoryMimeType, | |||||
} | |||||
di, err := s.service.Files.Create(dir).Fields("id").Do() | |||||
if err != nil { | |||||
return err | |||||
} | |||||
s.rootId = di.Id | |||||
err = ioutil.WriteFile(rootFileConfig, []byte(s.rootId), os.FileMode(0600)) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
return nil | |||||
} | |||||
func (s *GDrive) hasChecksum(f *drive.File) bool { | |||||
return f.Md5Checksum != "" | |||||
} | |||||
func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) { | |||||
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do() | |||||
} | |||||
func (s *GDrive) findId(filename string, token string) (string, error) { | |||||
filename = strings.Replace(filename, `'`, `\'`, -1) | |||||
filename = strings.Replace(filename, `"`, `\"`, -1) | |||||
fileId, tokenId, nextPageToken := "", "", "" | |||||
q := fmt.Sprintf("'%s' in parents and name='%s' and mimeType='%s' and trashed=false", s.rootId, token, GDriveDirectoryMimeType) | |||||
l, err := s.list(nextPageToken, q) | |||||
if err != nil { | |||||
return "", err | |||||
} | |||||
for 0 < len(l.Files) { | |||||
for _, fi := range l.Files { | |||||
tokenId = fi.Id | |||||
break | |||||
} | |||||
if l.NextPageToken == "" { | |||||
break | |||||
} | |||||
l, err = s.list(l.NextPageToken, q) | |||||
} | |||||
if filename == "" { | |||||
return tokenId, nil | |||||
} else if tokenId == "" { | |||||
return "", fmt.Errorf("Cannot find file %s/%s", token, filename) | |||||
} | |||||
q = fmt.Sprintf("'%s' in parents and name='%s' and mimeType!='%s' and trashed=false", tokenId, filename, GDriveDirectoryMimeType) | |||||
l, err = s.list(nextPageToken, q) | |||||
if err != nil { | |||||
return "", err | |||||
} | |||||
for 0 < len(l.Files) { | |||||
for _, fi := range l.Files { | |||||
fileId = fi.Id | |||||
break | |||||
} | |||||
if l.NextPageToken == "" { | |||||
break | |||||
} | |||||
l, err = s.list(l.NextPageToken, q) | |||||
} | |||||
if fileId == "" { | |||||
return "", fmt.Errorf("Cannot find file %s/%s", token, filename) | |||||
} | |||||
return fileId, nil | |||||
} | |||||
func (s *GDrive) Type() string { | |||||
return "gdrive" | |||||
} | |||||
func (s *GDrive) Get(token string, filename string) (reader io.ReadCloser, metadata Metadata, err error) { | |||||
var fileId string | |||||
fileId, err = s.findId(filename, token) | |||||
if err != nil { | |||||
return | |||||
} | |||||
var fi *drive.File | |||||
fi, err = s.service.Files.Get(fileId).Do() | |||||
if !s.hasChecksum(fi) { | |||||
err = fmt.Errorf("Cannot find file %s/%s", token, filename) | |||||
return | |||||
} | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
downloads, err := strconv.Atoi(fi.Properties["downloads"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
maxdownloads, err := strconv.Atoi(fi.Properties["maxDownloads"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
expires, err := time.Parse("2020-02-02 02:02:02", fi.Properties["expires"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
metadata = Metadata{ | |||||
ContentType: "", | |||||
ContentLength: fi.Size, | |||||
Downloads: downloads, | |||||
MaxDownloads: maxdownloads, | |||||
MaxDate: expires, | |||||
DeletionToken: fi.Properties["deletionToken"], | |||||
Secret: fi.Properties["deletionSecret"], | |||||
} | |||||
ctx := context.Background() | |||||
var res *http.Response | |||||
res, err = s.service.Files.Get(fileId).Context(ctx).Download() | |||||
if err != nil { | |||||
return | |||||
} | |||||
reader = res.Body | |||||
return | |||||
} | |||||
func (s *GDrive) Head(token string, filename string) (metadata Metadata, err error) { | |||||
var fileId string | |||||
fileId, err = s.findId(filename, token) | |||||
if err != nil { | |||||
return | |||||
} | |||||
var fi *drive.File | |||||
if fi, err = s.service.Files.Get(fileId).Do(); err != nil { | |||||
return | |||||
} | |||||
downloads, err := strconv.Atoi(fi.Properties["downloads"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
maxdownloads, err := strconv.Atoi(fi.Properties["maxDownloads"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
expires, err := time.Parse("2020-02-02 02:02:02", fi.Properties["expires"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
metadata = Metadata{ | |||||
ContentType: "", | |||||
ContentLength: fi.Size, | |||||
Downloads: downloads, | |||||
MaxDownloads: maxdownloads, | |||||
MaxDate: expires, | |||||
DeletionToken: fi.Properties["deletionToken"], | |||||
Secret: fi.Properties["deletionSecret"], | |||||
} | |||||
return | |||||
} | |||||
func (s *GDrive) Meta(token string, filename string, metadata Metadata) error { | |||||
return nil | |||||
} | |||||
func (s *GDrive) Put(token string, filename string, reader io.Reader, metadata Metadata) error { | |||||
dirId, err := s.findId("", token) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
if dirId == "" { | |||||
dir := &drive.File{ | |||||
Name: token, | |||||
Parents: []string{s.rootId}, | |||||
MimeType: GDriveDirectoryMimeType, | |||||
} | |||||
di, err := s.service.Files.Create(dir).Fields("id").Do() | |||||
if err != nil { | |||||
return err | |||||
} | |||||
dirId = di.Id | |||||
} | |||||
// Instantiate empty drive file | |||||
dst := &drive.File{ | |||||
Name: filename, | |||||
Parents: []string{dirId}, | |||||
MimeType: metadata.ContentType, | |||||
Properties: map[string]string{ | |||||
"downloads": strconv.Itoa(metadata.Downloads), | |||||
"maxDownloads": strconv.Itoa(metadata.MaxDownloads), | |||||
"deletionToken": metadata.DeletionToken, | |||||
"deletionSecret": metadata.Secret, | |||||
"expires": metadata.MaxDate.String(), | |||||
}, | |||||
} | |||||
ctx := context.Background() | |||||
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do() | |||||
if err != nil { | |||||
return err | |||||
} | |||||
return nil | |||||
} | |||||
func (s *GDrive) Delete(token string, filename string) (err error) { | |||||
metadata, _ := s.findId(fmt.Sprintf("%s.metadata", filename), token) | |||||
s.service.Files.Delete(metadata).Do() | |||||
var fileId string | |||||
fileId, err = s.findId(filename, token) | |||||
if err != nil { | |||||
return | |||||
} | |||||
err = s.service.Files.Delete(fileId).Do() | |||||
return | |||||
} | |||||
func (s *GDrive) IsNotExist(err error) bool { | |||||
if err != nil { | |||||
if e, ok := err.(*googleapi.Error); ok { | |||||
return e.Code == http.StatusNotFound | |||||
} | |||||
} | |||||
return false | |||||
} | |||||
func (s *GDrive) DeleteExpired() error { | |||||
return nil | |||||
} | |||||
// Retrieve a token, saves the token, then returns the generated client. | |||||
func getGDriveClient(config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client { | |||||
tokenFile := filepath.Join(localConfigPath, GDriveTokenJsonFile) | |||||
tok, err := gDriveTokenFromFile(tokenFile) | |||||
if err != nil { | |||||
tok = getGDriveTokenFromWeb(config, logger) | |||||
saveGDriveToken(tokenFile, tok, logger) | |||||
} | |||||
return config.Client(context.Background(), tok) | |||||
} | |||||
// Request a token from the web, then returns the retrieved token. | |||||
func getGDriveTokenFromWeb(config *oauth2.Config, logger *log.Logger) *oauth2.Token { | |||||
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline) | |||||
fmt.Printf("Go to the following link in your browser then type the "+ | |||||
"authorization code: \n%v\n", authURL) | |||||
var authCode string | |||||
if _, err := fmt.Scan(&authCode); err != nil { | |||||
logger.Fatalf("Unable to read authorization code %v", err) | |||||
} | |||||
tok, err := config.Exchange(context.TODO(), authCode) | |||||
if err != nil { | |||||
logger.Fatalf("Unable to retrieve token from web %v", err) | |||||
} | |||||
return tok | |||||
} | |||||
// Retrieves a token from a local file. | |||||
func gDriveTokenFromFile(file string) (*oauth2.Token, error) { | |||||
f, err := os.Open(file) | |||||
defer f.Close() | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
tok := &oauth2.Token{} | |||||
err = json.NewDecoder(f).Decode(tok) | |||||
return tok, err | |||||
} | |||||
// Saves a token to a file path. | |||||
func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) { | |||||
logger.Printf("Saving credential file to: %s\n", path) | |||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) | |||||
defer f.Close() | |||||
if err != nil { | |||||
logger.Fatalf("Unable to cache oauth token: %v", err) | |||||
} | |||||
json.NewEncoder(f).Encode(token) | |||||
} |
@@ -0,0 +1,35 @@ | |||||
package storage | |||||
import ( | |||||
"io" | |||||
"time" | |||||
) | |||||
type Metadata struct { | |||||
// ContentType is the original uploading content type | |||||
ContentType string | |||||
// ContentLength contains the length of the actual object | |||||
ContentLength int64 | |||||
// Downloads is the actual number of downloads | |||||
Downloads int | |||||
// MaxDownloads contains the maximum numbers of downloads | |||||
MaxDownloads int | |||||
// MaxDate contains the max age of the file | |||||
MaxDate time.Time | |||||
// DeletionToken contains the token to match against for deletion | |||||
DeletionToken string | |||||
// Secret as knowledge to delete file | |||||
Secret string | |||||
} | |||||
type Storage interface { | |||||
Get(token string, filename string) (reader io.ReadCloser, metaData Metadata, err error) | |||||
Head(token string, filename string) (metadata Metadata, err error) | |||||
Meta(token string, filename string, metadata Metadata) error | |||||
Put(token string, filename string, reader io.Reader, metadata Metadata) error | |||||
Delete(token string, filename string) error | |||||
IsNotExist(err error) bool | |||||
DeleteExpired() error | |||||
Type() string | |||||
} |
@@ -0,0 +1,384 @@ | |||||
package storage | |||||
import ( | |||||
"context" | |||||
"encoding/json" | |||||
"fmt" | |||||
"io" | |||||
"io/ioutil" | |||||
"log" | |||||
"net/http" | |||||
"os" | |||||
"path/filepath" | |||||
"strconv" | |||||
"strings" | |||||
"time" | |||||
"golang.org/x/oauth2" | |||||
"golang.org/x/oauth2/google" | |||||
"google.golang.org/api/drive/v3" | |||||
"google.golang.org/api/googleapi" | |||||
) | |||||
const GDriveRootConfigFile = "root_id.conf" | |||||
const GDriveTokenJsonFile = "token.json" | |||||
const GDriveDirectoryMimeType = "application/vnd.google-apps.folder" | |||||
type GDrive struct { | |||||
service *drive.Service | |||||
rootId string | |||||
basedir string | |||||
localConfigPath string | |||||
chunkSize int | |||||
logger *log.Logger | |||||
} | |||||
func NewGDriveStorage(clientJsonFilepath string, localConfigPath string, basedir string, chunkSize int, logger *log.Logger) (*GDrive, error) { | |||||
b, err := ioutil.ReadFile(clientJsonFilepath) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
// If modifying these scopes, delete your previously saved client_secret.json. | |||||
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
srv, err := drive.New(getGDriveClient(config, localConfigPath, logger)) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
chunkSize = chunkSize * 1024 * 1024 | |||||
storage := &GDrive{service: srv, basedir: basedir, rootId: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger} | |||||
err = storage.setupRoot() | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
return storage, nil | |||||
} | |||||
func (s *GDrive) setupRoot() error { | |||||
rootFileConfig := filepath.Join(s.localConfigPath, GDriveRootConfigFile) | |||||
rootId, err := ioutil.ReadFile(rootFileConfig) | |||||
if err != nil && !os.IsNotExist(err) { | |||||
return err | |||||
} | |||||
if string(rootId) != "" { | |||||
s.rootId = string(rootId) | |||||
return nil | |||||
} | |||||
dir := &drive.File{ | |||||
Name: s.basedir, | |||||
MimeType: GDriveDirectoryMimeType, | |||||
} | |||||
di, err := s.service.Files.Create(dir).Fields("id").Do() | |||||
if err != nil { | |||||
return err | |||||
} | |||||
s.rootId = di.Id | |||||
err = ioutil.WriteFile(rootFileConfig, []byte(s.rootId), os.FileMode(0600)) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
return nil | |||||
} | |||||
func (s *GDrive) hasChecksum(f *drive.File) bool { | |||||
return f.Md5Checksum != "" | |||||
} | |||||
func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) { | |||||
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do() | |||||
} | |||||
func (s *GDrive) findId(filename string, token string) (string, error) { | |||||
filename = strings.Replace(filename, `'`, `\'`, -1) | |||||
filename = strings.Replace(filename, `"`, `\"`, -1) | |||||
fileId, tokenId, nextPageToken := "", "", "" | |||||
q := fmt.Sprintf("'%s' in parents and name='%s' and mimeType='%s' and trashed=false", s.rootId, token, GDriveDirectoryMimeType) | |||||
l, err := s.list(nextPageToken, q) | |||||
if err != nil { | |||||
return "", err | |||||
} | |||||
for 0 < len(l.Files) { | |||||
for _, fi := range l.Files { | |||||
tokenId = fi.Id | |||||
break | |||||
} | |||||
if l.NextPageToken == "" { | |||||
break | |||||
} | |||||
l, err = s.list(l.NextPageToken, q) | |||||
} | |||||
if filename == "" { | |||||
return tokenId, nil | |||||
} else if tokenId == "" { | |||||
return "", fmt.Errorf("Cannot find file %s/%s", token, filename) | |||||
} | |||||
q = fmt.Sprintf("'%s' in parents and name='%s' and mimeType!='%s' and trashed=false", tokenId, filename, GDriveDirectoryMimeType) | |||||
l, err = s.list(nextPageToken, q) | |||||
if err != nil { | |||||
return "", err | |||||
} | |||||
for 0 < len(l.Files) { | |||||
for _, fi := range l.Files { | |||||
fileId = fi.Id | |||||
break | |||||
} | |||||
if l.NextPageToken == "" { | |||||
break | |||||
} | |||||
l, err = s.list(l.NextPageToken, q) | |||||
} | |||||
if fileId == "" { | |||||
return "", fmt.Errorf("Cannot find file %s/%s", token, filename) | |||||
} | |||||
return fileId, nil | |||||
} | |||||
func (s *GDrive) Type() string { | |||||
return "gdrive" | |||||
} | |||||
func (s *GDrive) Get(token string, filename string) (reader io.ReadCloser, metadata Metadata, err error) { | |||||
var fileId string | |||||
fileId, err = s.findId(filename, token) | |||||
if err != nil { | |||||
return | |||||
} | |||||
var fi *drive.File | |||||
fi, err = s.service.Files.Get(fileId).Do() | |||||
if !s.hasChecksum(fi) { | |||||
err = fmt.Errorf("Cannot find file %s/%s", token, filename) | |||||
return | |||||
} | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
downloads, err := strconv.Atoi(fi.Properties["downloads"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
maxdownloads, err := strconv.Atoi(fi.Properties["maxDownloads"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
expires, err := time.Parse("2020-02-02 02:02:02", fi.Properties["expires"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
metadata = Metadata{ | |||||
ContentType: "", | |||||
ContentLength: fi.Size, | |||||
Downloads: downloads, | |||||
MaxDownloads: maxdownloads, | |||||
MaxDate: expires, | |||||
DeletionToken: fi.Properties["deletionToken"], | |||||
Secret: fi.Properties["deletionSecret"], | |||||
} | |||||
ctx := context.Background() | |||||
var res *http.Response | |||||
res, err = s.service.Files.Get(fileId).Context(ctx).Download() | |||||
if err != nil { | |||||
return | |||||
} | |||||
reader = res.Body | |||||
return | |||||
} | |||||
func (s *GDrive) Head(token string, filename string) (metadata Metadata, err error) { | |||||
var fileId string | |||||
fileId, err = s.findId(filename, token) | |||||
if err != nil { | |||||
return | |||||
} | |||||
var fi *drive.File | |||||
if fi, err = s.service.Files.Get(fileId).Do(); err != nil { | |||||
return | |||||
} | |||||
downloads, err := strconv.Atoi(fi.Properties["downloads"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
maxdownloads, err := strconv.Atoi(fi.Properties["maxDownloads"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
expires, err := time.Parse("2020-02-02 02:02:02", fi.Properties["expires"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
metadata = Metadata{ | |||||
ContentType: "", | |||||
ContentLength: fi.Size, | |||||
Downloads: downloads, | |||||
MaxDownloads: maxdownloads, | |||||
MaxDate: expires, | |||||
DeletionToken: fi.Properties["deletionToken"], | |||||
Secret: fi.Properties["deletionSecret"], | |||||
} | |||||
return | |||||
} | |||||
func (s *GDrive) Meta(token string, filename string, metadata Metadata) error { | |||||
return nil | |||||
} | |||||
func (s *GDrive) Put(token string, filename string, reader io.Reader, metadata Metadata) error { | |||||
dirId, err := s.findId("", token) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
if dirId == "" { | |||||
dir := &drive.File{ | |||||
Name: token, | |||||
Parents: []string{s.rootId}, | |||||
MimeType: GDriveDirectoryMimeType, | |||||
} | |||||
di, err := s.service.Files.Create(dir).Fields("id").Do() | |||||
if err != nil { | |||||
return err | |||||
} | |||||
dirId = di.Id | |||||
} | |||||
// Instantiate empty drive file | |||||
dst := &drive.File{ | |||||
Name: filename, | |||||
Parents: []string{dirId}, | |||||
MimeType: metadata.ContentType, | |||||
Properties: map[string]string{ | |||||
"downloads": strconv.Itoa(metadata.Downloads), | |||||
"maxDownloads": strconv.Itoa(metadata.MaxDownloads), | |||||
"deletionToken": metadata.DeletionToken, | |||||
"deletionSecret": metadata.Secret, | |||||
"expires": metadata.MaxDate.String(), | |||||
}, | |||||
} | |||||
ctx := context.Background() | |||||
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do() | |||||
if err != nil { | |||||
return err | |||||
} | |||||
return nil | |||||
} | |||||
func (s *GDrive) Delete(token string, filename string) (err error) { | |||||
metadata, _ := s.findId(fmt.Sprintf("%s.metadata", filename), token) | |||||
s.service.Files.Delete(metadata).Do() | |||||
var fileId string | |||||
fileId, err = s.findId(filename, token) | |||||
if err != nil { | |||||
return | |||||
} | |||||
err = s.service.Files.Delete(fileId).Do() | |||||
return | |||||
} | |||||
func (s *GDrive) IsNotExist(err error) bool { | |||||
if err != nil { | |||||
if e, ok := err.(*googleapi.Error); ok { | |||||
return e.Code == http.StatusNotFound | |||||
} | |||||
} | |||||
return false | |||||
} | |||||
func (s *GDrive) DeleteExpired() error { | |||||
return nil | |||||
} | |||||
// Retrieve a token, saves the token, then returns the generated client. | |||||
func getGDriveClient(config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client { | |||||
tokenFile := filepath.Join(localConfigPath, GDriveTokenJsonFile) | |||||
tok, err := gDriveTokenFromFile(tokenFile) | |||||
if err != nil { | |||||
tok = getGDriveTokenFromWeb(config, logger) | |||||
saveGDriveToken(tokenFile, tok, logger) | |||||
} | |||||
return config.Client(context.Background(), tok) | |||||
} | |||||
// Request a token from the web, then returns the retrieved token. | |||||
func getGDriveTokenFromWeb(config *oauth2.Config, logger *log.Logger) *oauth2.Token { | |||||
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline) | |||||
fmt.Printf("Go to the following link in your browser then type the "+ | |||||
"authorization code: \n%v\n", authURL) | |||||
var authCode string | |||||
if _, err := fmt.Scan(&authCode); err != nil { | |||||
logger.Fatalf("Unable to read authorization code %v", err) | |||||
} | |||||
tok, err := config.Exchange(context.TODO(), authCode) | |||||
if err != nil { | |||||
logger.Fatalf("Unable to retrieve token from web %v", err) | |||||
} | |||||
return tok | |||||
} | |||||
// Retrieves a token from a local file. | |||||
func gDriveTokenFromFile(file string) (*oauth2.Token, error) { | |||||
f, err := os.Open(file) | |||||
defer f.Close() | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
tok := &oauth2.Token{} | |||||
err = json.NewDecoder(f).Decode(tok) | |||||
return tok, err | |||||
} | |||||
// Saves a token to a file path. | |||||
func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) { | |||||
logger.Printf("Saving credential file to: %s\n", path) | |||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) | |||||
defer f.Close() | |||||
if err != nil { | |||||
logger.Fatalf("Unable to cache oauth token: %v", err) | |||||
} | |||||
json.NewEncoder(f).Encode(token) | |||||
} |
@@ -0,0 +1,128 @@ | |||||
package storage | |||||
import ( | |||||
"bytes" | |||||
"encoding/json" | |||||
"fmt" | |||||
"io" | |||||
"log" | |||||
"os" | |||||
"path/filepath" | |||||
) | |||||
type LocalStorage struct { | |||||
Storage | |||||
basedir string | |||||
logger *log.Logger | |||||
} | |||||
func NewLocalStorage(basedir string, logger *log.Logger) (*LocalStorage, error) { | |||||
return &LocalStorage{basedir: basedir, logger: logger}, nil | |||||
} | |||||
func (s *LocalStorage) Type() string { | |||||
return "local" | |||||
} | |||||
func (s *LocalStorage) Get(token string, filename string) (reader io.ReadCloser, metadata Metadata, err error) { | |||||
path := filepath.Join(s.basedir, token, filename) | |||||
// content type , content length | |||||
reader, err = os.Open(path) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
metadata, err = s.Head(token, filename) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
return reader, metadata, nil | |||||
} | |||||
func (s *LocalStorage) Head(token string, filename string) (metadata Metadata, err error) { | |||||
path := filepath.Join(s.basedir, token, filename) | |||||
fi, err := os.Open(path) | |||||
if err != nil { | |||||
return | |||||
} | |||||
err = json.NewDecoder(fi).Decode(&metadata) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
return metadata, nil | |||||
} | |||||
func (s *LocalStorage) Meta(token string, filename string, metadata Metadata) error { | |||||
return s.putMetadata(token, filename, metadata) | |||||
} | |||||
func (s *LocalStorage) Put(token string, filename string, reader io.Reader, metadata Metadata) error { | |||||
err := s.putMetadata(token, filename, metadata) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
err = s.put(token, filename, reader) | |||||
if err != nil { | |||||
//Delete the metadata if the put failed | |||||
_ = s.Delete(token, fmt.Sprintf("%s.metadata", filename)) | |||||
} | |||||
return err | |||||
} | |||||
func (s *LocalStorage) put(token string, filename string, reader io.Reader) error { | |||||
var f io.WriteCloser | |||||
var err error | |||||
path := filepath.Join(s.basedir, token) | |||||
if err = os.MkdirAll(path, 0700); err != nil && !os.IsExist(err) { | |||||
return err | |||||
} | |||||
if f, err = os.OpenFile(filepath.Join(path, filename), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600); err != nil { | |||||
return err | |||||
} | |||||
defer f.Close() | |||||
_, err = io.Copy(f, reader) | |||||
return err | |||||
} | |||||
func (s *LocalStorage) putMetadata(token string, filename string, metadata Metadata) error { | |||||
buffer := &bytes.Buffer{} | |||||
if err := json.NewEncoder(buffer).Encode(metadata); err != nil { | |||||
log.Printf("%s", err.Error()) | |||||
return err | |||||
} else if err := s.put(token, filename, buffer); err != nil { | |||||
log.Printf("%s", err.Error()) | |||||
return nil | |||||
} | |||||
return nil | |||||
} | |||||
func (s *LocalStorage) Delete(token string, filename string) (err error) { | |||||
metadata := filepath.Join(s.basedir, token, fmt.Sprintf("%s.metadata", filename)) | |||||
_ = os.Remove(metadata) | |||||
path := filepath.Join(s.basedir, token, filename) | |||||
err = os.Remove(path) | |||||
return | |||||
} | |||||
func (s *LocalStorage) IsNotExist(err error) bool { | |||||
if err == nil { | |||||
return false | |||||
} | |||||
return os.IsNotExist(err) | |||||
} | |||||
func (s *LocalStorage) DeleteExpired() error { | |||||
return nil | |||||
} |
@@ -0,0 +1,23 @@ | |||||
package storage | |||||
import ( | |||||
"strconv" | |||||
"time" | |||||
) | |||||
func (metadata Metadata) RemainingLimitHeaderValues() (remainingDownloads, remainingDays string) { | |||||
if metadata.MaxDate.IsZero() { | |||||
remainingDays = "n/a" | |||||
} else { | |||||
timeDifference := metadata.MaxDate.Sub(time.Now()) | |||||
remainingDays = strconv.Itoa(int(timeDifference.Hours()/24) + 1) | |||||
} | |||||
if metadata.MaxDownloads == -1 { | |||||
remainingDownloads = "n/a" | |||||
} else { | |||||
remainingDownloads = strconv.Itoa(metadata.MaxDownloads - metadata.Downloads) | |||||
} | |||||
return remainingDownloads, remainingDays | |||||
} |
@@ -0,0 +1,225 @@ | |||||
package storage | |||||
import ( | |||||
"fmt" | |||||
"io" | |||||
"log" | |||||
"strconv" | |||||
"time" | |||||
"github.com/aws/aws-sdk-go/aws" | |||||
"github.com/aws/aws-sdk-go/aws/awserr" | |||||
"github.com/aws/aws-sdk-go/aws/credentials" | |||||
"github.com/aws/aws-sdk-go/aws/session" | |||||
"github.com/aws/aws-sdk-go/service/s3" | |||||
"github.com/aws/aws-sdk-go/service/s3/s3manager" | |||||
) | |||||
type S3Storage struct { | |||||
Storage | |||||
bucket string | |||||
session *session.Session | |||||
s3 *s3.S3 | |||||
logger *log.Logger | |||||
noMultipart bool | |||||
} | |||||
func NewS3Storage(accessKey, secretKey, bucketName, region, endpoint string, logger *log.Logger, disableMultipart bool, forcePathStyle bool) (*S3Storage, error) { | |||||
sess := getAwsSession(accessKey, secretKey, region, endpoint, forcePathStyle) | |||||
return &S3Storage{bucket: bucketName, s3: s3.New(sess), session: sess, logger: logger, noMultipart: disableMultipart}, nil | |||||
} | |||||
func (s *S3Storage) Type() string { | |||||
return "s3" | |||||
} | |||||
func (s *S3Storage) Head(token string, filename string) (metadata Metadata, err error) { | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
headRequest := &s3.HeadObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | |||||
// content type , content length | |||||
response, err := s.s3.HeadObject(headRequest) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
downloads, err := strconv.Atoi(*response.Metadata["downloads"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
maxdownloads, err := strconv.Atoi(*response.Metadata["maxDownloads"]) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
expires, err := time.Parse("2020-02-02 02:02:02", *response.Expires) | |||||
if err != nil { | |||||
return Metadata{}, err | |||||
} | |||||
metadata = Metadata{ | |||||
ContentType: "", | |||||
ContentLength: *response.ContentLength, | |||||
Downloads: downloads, | |||||
MaxDownloads: maxdownloads, | |||||
MaxDate: expires, | |||||
DeletionToken: *response.Metadata["deletionToken"], | |||||
Secret: *response.Metadata["deletionSecret"], | |||||
} | |||||
return metadata, nil | |||||
} | |||||
func (s *S3Storage) Meta(token string, filename string, metadata Metadata) error { | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
input := &s3.CopyObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
CopySource: aws.String(key), | |||||
Key: aws.String(key), | |||||
MetadataDirective: aws.String("REPLACE"), | |||||
Metadata: map[string]*string{ | |||||
"downloads": aws.String(strconv.Itoa(metadata.Downloads)), | |||||
"maxDownloads": aws.String(strconv.Itoa(metadata.MaxDownloads)), | |||||
"deletionToken": aws.String(metadata.DeletionToken), | |||||
"deletionSecret": aws.String(metadata.Secret), | |||||
}, | |||||
ContentType: aws.String(metadata.ContentType), | |||||
Expires: aws.Time(metadata.MaxDate), | |||||
} | |||||
_, err := s.s3.CopyObject(input) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
return nil | |||||
} | |||||
func (s *S3Storage) Get(token string, filename string) (reader io.ReadCloser, metadata Metadata, err error) { | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
getRequest := &s3.GetObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | |||||
response, err := s.s3.GetObject(getRequest) | |||||
if err != nil { | |||||
return | |||||
} | |||||
downloads, err := strconv.Atoi(*response.Metadata["downloads"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
maxdownloads, err := strconv.Atoi(*response.Metadata["maxDownloads"]) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
expires, err := time.Parse("2020-02-02 02:02:02", *response.Expires) | |||||
if err != nil { | |||||
return nil, Metadata{}, err | |||||
} | |||||
metadata = Metadata{ | |||||
ContentType: "", | |||||
ContentLength: *response.ContentLength, | |||||
Downloads: downloads, | |||||
MaxDownloads: maxdownloads, | |||||
MaxDate: expires, | |||||
DeletionToken: *response.Metadata["deletionToken"], | |||||
Secret: *response.Metadata["deletionSecret"], | |||||
} | |||||
reader = response.Body | |||||
return | |||||
} | |||||
func (s *S3Storage) Delete(token string, filename string) (err error) { | |||||
metadata := fmt.Sprintf("%s/%s.metadata", token, filename) | |||||
deleteRequest := &s3.DeleteObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(metadata), | |||||
} | |||||
_, err = s.s3.DeleteObject(deleteRequest) | |||||
if err != nil { | |||||
return | |||||
} | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
deleteRequest = &s3.DeleteObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | |||||
_, err = s.s3.DeleteObject(deleteRequest) | |||||
return | |||||
} | |||||
func (s *S3Storage) Put(token string, filename string, reader io.Reader, metadata Metadata) (err error) { | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
s.logger.Printf("Uploading file %s to S3 Bucket", filename) | |||||
var concurrency int | |||||
if !s.noMultipart { | |||||
concurrency = 20 | |||||
} else { | |||||
concurrency = 1 | |||||
} | |||||
// Create an uploader with the session and custom options | |||||
uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) { | |||||
u.Concurrency = concurrency // default is 5 | |||||
u.LeavePartsOnError = false | |||||
}) | |||||
_, err = uploader.Upload(&s3manager.UploadInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
Body: reader, | |||||
Metadata: map[string]*string{ | |||||
"downloads": aws.String(strconv.Itoa(metadata.Downloads)), | |||||
"maxDownloads": aws.String(strconv.Itoa(metadata.MaxDownloads)), | |||||
"deletionToken": aws.String(metadata.DeletionToken), | |||||
"deletionSecret": aws.String(metadata.Secret), | |||||
}, | |||||
ContentType: aws.String(metadata.ContentType), | |||||
Expires: aws.Time(metadata.MaxDate), | |||||
}) | |||||
return | |||||
} | |||||
func (s *S3Storage) IsNotExist(err error) bool { | |||||
if err == nil { | |||||
return false | |||||
} | |||||
if aerr, ok := err.(awserr.Error); ok { | |||||
switch aerr.Code() { | |||||
case s3.ErrCodeNoSuchKey: | |||||
return true | |||||
} | |||||
} | |||||
return false | |||||
} | |||||
func (s *S3Storage) DeleteExpired() error { | |||||
// not necessary, as S3 has expireDate on files to automatically delete the them | |||||
return nil | |||||
} | |||||
func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session { | |||||
return session.Must(session.NewSession(&aws.Config{ | |||||
Region: aws.String(region), | |||||
Endpoint: aws.String(endpoint), | |||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), | |||||
S3ForcePathStyle: aws.Bool(forcePathStyle), | |||||
})) | |||||
} |
@@ -31,21 +31,9 @@ import ( | |||||
"strconv" | "strconv" | ||||
"strings" | "strings" | ||||
"github.com/aws/aws-sdk-go/aws" | |||||
"github.com/aws/aws-sdk-go/aws/credentials" | |||||
"github.com/aws/aws-sdk-go/aws/session" | |||||
"github.com/golang/gddo/httputil/header" | "github.com/golang/gddo/httputil/header" | ||||
) | ) | ||||
func getAwsSession(accessKey, secretKey, region, endpoint string, forcePathStyle bool) *session.Session { | |||||
return session.Must(session.NewSession(&aws.Config{ | |||||
Region: aws.String(region), | |||||
Endpoint: aws.String(endpoint), | |||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), | |||||
S3ForcePathStyle: aws.Bool(forcePathStyle), | |||||
})) | |||||
} | |||||
func formatNumber(format string, s uint64) string { | func formatNumber(format string, s uint64) string { | ||||
return RenderFloat(format, float64(s)) | return RenderFloat(format, float64(s)) | ||||