From 59014f21109fd1d7f9744d6b30279be0f9e77653 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Wed, 28 Aug 2019 15:18:00 +0200 Subject: [PATCH 1/6] Add Storj Storage Adapter - Base Functions --- go.mod | 2 +- server/storage.go | 103 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 63151ea..4b95401 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/golang/gddo v0.0.0-20190815223733-287de01127ef github.com/gorilla/mux v1.7.3 github.com/gorilla/securecookie v1.1.1 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-isatty v0.0.9 // indirect github.com/microcosm-cc/bluemonday v1.0.2 @@ -31,6 +30,7 @@ require ( google.golang.org/api v0.9.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/russross/blackfriday.v2 v2.0.1 + storj.io/storj v0.19.0 ) replace gopkg.in/russross/blackfriday.v2 v2.0.1 => github.com/russross/blackfriday/v2 v2.0.1 diff --git a/server/storage.go b/server/storage.go index e256136..6d9ce95 100644 --- a/server/storage.go +++ b/server/storage.go @@ -22,6 +22,8 @@ import ( "golang.org/x/oauth2/google" "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" + "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/storj" ) type Storage interface { @@ -561,3 +563,104 @@ func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) { json.NewEncoder(f).Encode(token) } + +type StorjStorage struct { + Storage + uplink *uplink.Uplink + project *uplink.Project + bucket *uplink.Bucket + logger *log.Logger +} + +func NewStorjStorage(endpoint, apiKey, bucket, encKey string) (*StorjStorage, error) { + var instance StorjStorage + var err error + + ctx := context.Background() + + instance.uplink, err = uplink.NewUplink(ctx, nil) + if err != nil { + return nil, fmt.Errorf("could not create new Uplink Instance: %v", err) + } + + key, err := uplink.ParseAPIKey(apiKey) + if err != nil { + return nil, fmt.Errorf("could not parse api key %v", err) + } + + instance.project, err = instance.uplink.OpenProject(ctx, endpoint, key) + if err != nil { + return nil, fmt.Errorf("could not open project: %v", err) + } + + access := uplink.NewEncryptionAccessWithDefaultKey(toStorjKey(encKey)) + instance.bucket, err = instance.project.OpenBucket(ctx, bucket, access) + if err != nil { + return nil, fmt.Errorf("could not open bucket %q: %v", bucket, err) + } + + return &instance, nil +} + +func (s *StorjStorage) Type() string { + return "storj" +} + +func (s *StorjStorage) Head(token string, filename string) (contentType string, contentLength uint64, err error) { + key := fmt.Sprintf("%s/%s", token, filename) + + ctx := context.TODO() + + obj, err := s.bucket.OpenObject(ctx, key) + if err != nil { + return "", 0, fmt.Errorf("unable to open object %v", err) + } + contentType = obj.Meta.ContentType + contentLength = uint64(obj.Meta.Size) + + return +} + +func (s *StorjStorage) Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) { + key := fmt.Sprintf("%s/%s", token, filename) + ctx := context.TODO() + + obj, err := s.bucket.OpenObject(ctx, key) + if err != nil { + return nil, "", 0, fmt.Errorf("unable to open object %v", err) + } + contentType = obj.Meta.ContentType + contentLength = uint64(obj.Meta.Size) + reader, err = obj.DownloadRange(ctx, 0, -1) + return +} + +func (s *StorjStorage) Delete(token string, filename string) (err error) { + key := fmt.Sprintf("%s/%s", token, filename) + + ctx := context.TODO() + + err = s.bucket.DeleteObject(ctx, key) + + return +} + +func (s *StorjStorage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { + key := fmt.Sprintf("%s/%s", token, filename) + + s.logger.Printf("Uploading file %s to S3 Bucket", filename) + + ctx := context.TODO() + + err = s.bucket.UploadObject(ctx, key, reader, &uplink.UploadOptions{ContentType: contentType}) + if err != nil { + return fmt.Errorf("could not upload: %v", err) + } + return nil +} + +func toStorjKey(key string) (newKey storj.Key) { + var encryptionKey storj.Key + copy(encryptionKey[:], []byte(key)) + return encryptionKey +} From 874268d80e913eeccae0348d1eb947560a7ebbc5 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Wed, 28 Aug 2019 15:29:35 +0200 Subject: [PATCH 2/6] Add Storj Storage Adapter - add Binding --- cmd/cmd.go | 34 ++++++++++++++++++++++++++++++++++ server/storage.go | 3 ++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 1d8fdb9..3cb3a9f 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -149,6 +149,26 @@ var globalFlags = []cli.Flag{ Usage: "", Value: googleapi.DefaultUploadChunkSize / 1024 / 1024, }, + cli.StringFlag{ + Name: "storj-endpoint", + Usage: "Satellite Address including Port.", + Value: "", + }, + cli.StringFlag{ + Name: "storj-apikey", + Usage: "", + Value: "", + }, + cli.StringFlag{ + Name: "storj-bucket", + Usage: "", + Value: "", + }, + cli.StringFlag{ + Name: "storj-enckey", + Usage: "Encryption Key for local file encryption", + Value: "", + }, cli.IntFlag{ Name: "rate-limit", Usage: "requests per minute", @@ -362,6 +382,20 @@ func New() *Cmd { } else { options = append(options, server.UseStorage(storage)) } + case "storj": + if endpoint := c.String("storj-endpoint"); endpoint == "" { + panic("storj-endpoint not set.") + } else if apiKey := c.String("storj-apikey"); apiKey == "" { + panic("storj-apikey not set.") + } else if bucket := c.String("storj-bucket"); bucket == "" { + panic("storj-enckey not set.") + } else if encKey := c.String("storj-enckey"); encKey == "" { + panic("storj-bucket not set.") + } else if storage, err := server.NewStorjStorage(endpoint, apiKey, bucket, encKey, logger); err != nil { + panic(err) + } else { + options = append(options, server.UseStorage(storage)) + } case "local": if v := c.String("basedir"); v == "" { panic("basedir not set.") diff --git a/server/storage.go b/server/storage.go index 6d9ce95..1445c4a 100644 --- a/server/storage.go +++ b/server/storage.go @@ -572,7 +572,7 @@ type StorjStorage struct { logger *log.Logger } -func NewStorjStorage(endpoint, apiKey, bucket, encKey string) (*StorjStorage, error) { +func NewStorjStorage(endpoint, apiKey, bucket, encKey string, logger *log.Logger) (*StorjStorage, error) { var instance StorjStorage var err error @@ -598,6 +598,7 @@ func NewStorjStorage(endpoint, apiKey, bucket, encKey string) (*StorjStorage, er if err != nil { return nil, fmt.Errorf("could not open bucket %q: %v", bucket, err) } + instance.logger = logger return &instance, nil } From d5e07fe2f496087b183661161cdb699839691c4e Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Wed, 28 Aug 2019 16:57:05 +0200 Subject: [PATCH 3/6] Add Env Vars --- cmd/cmd.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 3cb3a9f..991cf02 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -150,24 +150,28 @@ var globalFlags = []cli.Flag{ Value: googleapi.DefaultUploadChunkSize / 1024 / 1024, }, cli.StringFlag{ - Name: "storj-endpoint", - Usage: "Satellite Address including Port.", - Value: "", + Name: "storj-endpoint", + Usage: "Satellite Address including Port.", + Value: "", + EnvVar: "STORJ_ENDPOINT", }, cli.StringFlag{ - Name: "storj-apikey", - Usage: "", - Value: "", + Name: "storj-apikey", + Usage: "", + Value: "", + EnvVar: "STORJ_API_KEY", }, cli.StringFlag{ - Name: "storj-bucket", - Usage: "", - Value: "", + Name: "storj-bucket", + Usage: "", + Value: "", + EnvVar: "STORJ_BUCKET", }, cli.StringFlag{ - Name: "storj-enckey", - Usage: "Encryption Key for local file encryption", - Value: "", + Name: "storj-enckey", + Usage: "Encryption Key for local file encryption", + Value: "", + EnvVar: "STORJ_ENC_KEY", }, cli.IntFlag{ Name: "rate-limit", From c1691c58dcf49673cde9ed1b250e7415fb9e4731 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Thu, 29 Aug 2019 16:33:37 +0200 Subject: [PATCH 4/6] Switch to Salted Key Generation --- server/storage.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/storage.go b/server/storage.go index 1445c4a..6a80399 100644 --- a/server/storage.go +++ b/server/storage.go @@ -576,7 +576,7 @@ func NewStorjStorage(endpoint, apiKey, bucket, encKey string, logger *log.Logger var instance StorjStorage var err error - ctx := context.Background() + ctx := context.TODO() instance.uplink, err = uplink.NewUplink(ctx, nil) if err != nil { @@ -585,7 +585,7 @@ func NewStorjStorage(endpoint, apiKey, bucket, encKey string, logger *log.Logger key, err := uplink.ParseAPIKey(apiKey) if err != nil { - return nil, fmt.Errorf("could not parse api key %v", err) + return nil, fmt.Errorf("could not parse api key: %v", err) } instance.project, err = instance.uplink.OpenProject(ctx, endpoint, key) @@ -593,7 +593,12 @@ func NewStorjStorage(endpoint, apiKey, bucket, encKey string, logger *log.Logger return nil, fmt.Errorf("could not open project: %v", err) } - access := uplink.NewEncryptionAccessWithDefaultKey(toStorjKey(encKey)) + saltenckey, err := instance.project.SaltedKeyFromPassphrase(ctx, encKey) + if err != nil { + return nil, fmt.Errorf("could not generate salted enc key: %v", err) + } + + access := uplink.NewEncryptionAccessWithDefaultKey(*saltenckey) instance.bucket, err = instance.project.OpenBucket(ctx, bucket, access) if err != nil { return nil, fmt.Errorf("could not open bucket %q: %v", bucket, err) From 79c5130066ca51b1dedc1e611721148130638131 Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Thu, 29 Aug 2019 16:35:24 +0200 Subject: [PATCH 5/6] Changes according to CR --- server/storage.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/server/storage.go b/server/storage.go index 6a80399..fbc9749 100644 --- a/server/storage.go +++ b/server/storage.go @@ -613,7 +613,7 @@ func (s *StorjStorage) Type() string { } func (s *StorjStorage) Head(token string, filename string) (contentType string, contentLength uint64, err error) { - key := fmt.Sprintf("%s/%s", token, filename) + key := storj.JoinPaths(token, filename) ctx := context.TODO() @@ -628,7 +628,7 @@ func (s *StorjStorage) Head(token string, filename string) (contentType string, } func (s *StorjStorage) Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) { - key := fmt.Sprintf("%s/%s", token, filename) + key := storj.JoinPaths(token, filename) ctx := context.TODO() obj, err := s.bucket.OpenObject(ctx, key) @@ -642,7 +642,7 @@ func (s *StorjStorage) Get(token string, filename string) (reader io.ReadCloser, } func (s *StorjStorage) Delete(token string, filename string) (err error) { - key := fmt.Sprintf("%s/%s", token, filename) + key := storj.JoinPaths(token, filename) ctx := context.TODO() @@ -652,7 +652,7 @@ func (s *StorjStorage) Delete(token string, filename string) (err error) { } func (s *StorjStorage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { - key := fmt.Sprintf("%s/%s", token, filename) + key := storj.JoinPaths(token, filename) s.logger.Printf("Uploading file %s to S3 Bucket", filename) @@ -664,9 +664,3 @@ func (s *StorjStorage) Put(token string, filename string, reader io.Reader, cont } return nil } - -func toStorjKey(key string) (newKey storj.Key) { - var encryptionKey storj.Key - copy(encryptionKey[:], []byte(key)) - return encryptionKey -} From cf00af36f2175b7491f53087e9ee1ed98ad047af Mon Sep 17 00:00:00 2001 From: stefanbenten Date: Thu, 29 Aug 2019 16:58:14 +0200 Subject: [PATCH 6/6] Changing Error Handling and increase Logging --- server/storage.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/server/storage.go b/server/storage.go index fbc9749..9d3ad75 100644 --- a/server/storage.go +++ b/server/storage.go @@ -17,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/zeebo/errs" "golang.org/x/net/context" "golang.org/x/oauth2" "golang.org/x/oauth2/google" @@ -564,6 +565,8 @@ func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) { json.NewEncoder(f).Encode(token) } +var uplinkFailure = errs.Class("uplink failure") + type StorjStorage struct { Storage uplink *uplink.Uplink @@ -580,29 +583,31 @@ func NewStorjStorage(endpoint, apiKey, bucket, encKey string, logger *log.Logger instance.uplink, err = uplink.NewUplink(ctx, nil) if err != nil { - return nil, fmt.Errorf("could not create new Uplink Instance: %v", err) + return nil, uplinkFailure.New("could not create new Uplink Instance: %v", err) } key, err := uplink.ParseAPIKey(apiKey) if err != nil { - return nil, fmt.Errorf("could not parse api key: %v", err) + return nil, uplinkFailure.New("could not parse api key: %v", err) } instance.project, err = instance.uplink.OpenProject(ctx, endpoint, key) if err != nil { - return nil, fmt.Errorf("could not open project: %v", err) + return nil, uplinkFailure.New("could not open project: %v", err) } saltenckey, err := instance.project.SaltedKeyFromPassphrase(ctx, encKey) if err != nil { - return nil, fmt.Errorf("could not generate salted enc key: %v", err) + return nil, uplinkFailure.New("could not generate salted enc key: %v", err) } access := uplink.NewEncryptionAccessWithDefaultKey(*saltenckey) + instance.bucket, err = instance.project.OpenBucket(ctx, bucket, access) if err != nil { - return nil, fmt.Errorf("could not open bucket %q: %v", bucket, err) + return nil, uplinkFailure.New("could not open bucket %q: %v", bucket, err) } + instance.logger = logger return &instance, nil @@ -629,11 +634,14 @@ func (s *StorjStorage) Head(token string, filename string) (contentType string, func (s *StorjStorage) Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) { key := storj.JoinPaths(token, filename) + + s.logger.Printf("Getting file %s from Storj Bucket", filename) + ctx := context.TODO() obj, err := s.bucket.OpenObject(ctx, key) if err != nil { - return nil, "", 0, fmt.Errorf("unable to open object %v", err) + return nil, "", 0, uplinkFailure.New("unable to open object %v", err) } contentType = obj.Meta.ContentType contentLength = uint64(obj.Meta.Size) @@ -644,6 +652,8 @@ func (s *StorjStorage) Get(token string, filename string) (reader io.ReadCloser, func (s *StorjStorage) Delete(token string, filename string) (err error) { key := storj.JoinPaths(token, filename) + s.logger.Printf("Deleting file %s from Storj Bucket", filename) + ctx := context.TODO() err = s.bucket.DeleteObject(ctx, key) @@ -654,13 +664,13 @@ func (s *StorjStorage) Delete(token string, filename string) (err error) { func (s *StorjStorage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { key := storj.JoinPaths(token, filename) - s.logger.Printf("Uploading file %s to S3 Bucket", filename) + s.logger.Printf("Uploading file %s to Storj Bucket", filename) ctx := context.TODO() err = s.bucket.UploadObject(ctx, key, reader, &uplink.UploadOptions{ContentType: contentType}) if err != nil { - return fmt.Errorf("could not upload: %v", err) + return uplinkFailure.New("could not upload: %v", err) } return nil }