From 59014f21109fd1d7f9744d6b30279be0f9e77653 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Wed, 28 Aug 2019 15:18:00 +0200 Subject: [PATCH] Add Storj Storage Adapter - Base Functions --- go.mod | 2 +- server/storage.go | 103 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 63151ea..4b95401 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/golang/gddo v0.0.0-20190815223733-287de01127ef github.com/gorilla/mux v1.7.3 github.com/gorilla/securecookie v1.1.1 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-isatty v0.0.9 // indirect github.com/microcosm-cc/bluemonday v1.0.2 @@ -31,6 +30,7 @@ require ( google.golang.org/api v0.9.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/russross/blackfriday.v2 v2.0.1 + storj.io/storj v0.19.0 ) replace gopkg.in/russross/blackfriday.v2 v2.0.1 => github.com/russross/blackfriday/v2 v2.0.1 diff --git a/server/storage.go b/server/storage.go index e256136..6d9ce95 100644 --- a/server/storage.go +++ b/server/storage.go @@ -22,6 +22,8 @@ import ( "golang.org/x/oauth2/google" "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" + "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/storj" ) type Storage interface { @@ -561,3 +563,104 @@ func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) { json.NewEncoder(f).Encode(token) } + +type StorjStorage struct { + Storage + uplink *uplink.Uplink + project *uplink.Project + bucket *uplink.Bucket + logger *log.Logger +} + +func NewStorjStorage(endpoint, apiKey, bucket, encKey string) (*StorjStorage, error) { + var instance StorjStorage + var err error + + ctx := context.Background() + + instance.uplink, err = uplink.NewUplink(ctx, nil) + if err != nil { + return nil, fmt.Errorf("could not create new Uplink Instance: %v", err) + } + + key, err := uplink.ParseAPIKey(apiKey) + if err != nil { + return nil, fmt.Errorf("could not parse api key %v", err) + } + + instance.project, err = instance.uplink.OpenProject(ctx, endpoint, key) + if err != nil { + return nil, fmt.Errorf("could not open project: %v", err) + } + + access := uplink.NewEncryptionAccessWithDefaultKey(toStorjKey(encKey)) + instance.bucket, err = instance.project.OpenBucket(ctx, bucket, access) + if err != nil { + return nil, fmt.Errorf("could not open bucket %q: %v", bucket, err) + } + + return &instance, nil +} + +func (s *StorjStorage) Type() string { + return "storj" +} + +func (s *StorjStorage) Head(token string, filename string) (contentType string, contentLength uint64, err error) { + key := fmt.Sprintf("%s/%s", token, filename) + + ctx := context.TODO() + + obj, err := s.bucket.OpenObject(ctx, key) + if err != nil { + return "", 0, fmt.Errorf("unable to open object %v", err) + } + contentType = obj.Meta.ContentType + contentLength = uint64(obj.Meta.Size) + + return +} + +func (s *StorjStorage) Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) { + key := fmt.Sprintf("%s/%s", token, filename) + ctx := context.TODO() + + obj, err := s.bucket.OpenObject(ctx, key) + if err != nil { + return nil, "", 0, fmt.Errorf("unable to open object %v", err) + } + contentType = obj.Meta.ContentType + contentLength = uint64(obj.Meta.Size) + reader, err = obj.DownloadRange(ctx, 0, -1) + return +} + +func (s *StorjStorage) Delete(token string, filename string) (err error) { + key := fmt.Sprintf("%s/%s", token, filename) + + ctx := context.TODO() + + err = s.bucket.DeleteObject(ctx, key) + + return +} + +func (s *StorjStorage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { + key := fmt.Sprintf("%s/%s", token, filename) + + s.logger.Printf("Uploading file %s to S3 Bucket", filename) + + ctx := context.TODO() + + err = s.bucket.UploadObject(ctx, key, reader, &uplink.UploadOptions{ContentType: contentType}) + if err != nil { + return fmt.Errorf("could not upload: %v", err) + } + return nil +} + +func toStorjKey(key string) (newKey storj.Key) { + var encryptionKey storj.Key + copy(encryptionKey[:], []byte(key)) + return encryptionKey +}