diff --git a/cmd/cmd.go b/cmd/cmd.go index 1d8fdb9..991cf02 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -149,6 +149,30 @@ var globalFlags = []cli.Flag{ Usage: "", Value: googleapi.DefaultUploadChunkSize / 1024 / 1024, }, + cli.StringFlag{ + Name: "storj-endpoint", + Usage: "Satellite Address including Port.", + Value: "", + EnvVar: "STORJ_ENDPOINT", + }, + cli.StringFlag{ + Name: "storj-apikey", + Usage: "", + Value: "", + EnvVar: "STORJ_API_KEY", + }, + cli.StringFlag{ + Name: "storj-bucket", + Usage: "", + Value: "", + EnvVar: "STORJ_BUCKET", + }, + cli.StringFlag{ + Name: "storj-enckey", + Usage: "Encryption Key for local file encryption", + Value: "", + EnvVar: "STORJ_ENC_KEY", + }, cli.IntFlag{ Name: "rate-limit", Usage: "requests per minute", @@ -362,6 +386,20 @@ func New() *Cmd { } else { options = append(options, server.UseStorage(storage)) } + case "storj": + if endpoint := c.String("storj-endpoint"); endpoint == "" { + panic("storj-endpoint not set.") + } else if apiKey := c.String("storj-apikey"); apiKey == "" { + panic("storj-apikey not set.") + } else if bucket := c.String("storj-bucket"); bucket == "" { + panic("storj-enckey not set.") + } else if encKey := c.String("storj-enckey"); encKey == "" { + panic("storj-bucket not set.") + } else if storage, err := server.NewStorjStorage(endpoint, apiKey, bucket, encKey, logger); err != nil { + panic(err) + } else { + options = append(options, server.UseStorage(storage)) + } case "local": if v := c.String("basedir"); v == "" { panic("basedir not set.") diff --git a/go.mod b/go.mod index 63151ea..4b95401 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/golang/gddo v0.0.0-20190815223733-287de01127ef github.com/gorilla/mux v1.7.3 github.com/gorilla/securecookie v1.1.1 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-colorable v0.1.2 // indirect github.com/mattn/go-isatty v0.0.9 // indirect github.com/microcosm-cc/bluemonday v1.0.2 @@ -31,6 +30,7 @@ require ( google.golang.org/api v0.9.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/russross/blackfriday.v2 v2.0.1 + storj.io/storj v0.19.0 ) replace gopkg.in/russross/blackfriday.v2 v2.0.1 => github.com/russross/blackfriday/v2 v2.0.1 diff --git a/server/storage.go b/server/storage.go index e256136..9d3ad75 100644 --- a/server/storage.go +++ b/server/storage.go @@ -17,11 +17,14 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/zeebo/errs" "golang.org/x/net/context" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" + "storj.io/storj/lib/uplink" + "storj.io/storj/pkg/storj" ) type Storage interface { @@ -561,3 +564,113 @@ func saveGDriveToken(path string, token *oauth2.Token, logger *log.Logger) { json.NewEncoder(f).Encode(token) } + +var uplinkFailure = errs.Class("uplink failure") + +type StorjStorage struct { + Storage + uplink *uplink.Uplink + project *uplink.Project + bucket *uplink.Bucket + logger *log.Logger +} + +func NewStorjStorage(endpoint, apiKey, bucket, encKey string, logger *log.Logger) (*StorjStorage, error) { + var instance StorjStorage + var err error + + ctx := context.TODO() + + instance.uplink, err = uplink.NewUplink(ctx, nil) + if err != nil { + return nil, uplinkFailure.New("could not create new Uplink Instance: %v", err) + } + + key, err := uplink.ParseAPIKey(apiKey) + if err != nil { + return nil, uplinkFailure.New("could not parse api key: %v", err) + } + + instance.project, err = instance.uplink.OpenProject(ctx, endpoint, key) + if err != nil { + return nil, uplinkFailure.New("could not open project: %v", err) + } + + saltenckey, err := instance.project.SaltedKeyFromPassphrase(ctx, encKey) + if err != nil { + return nil, uplinkFailure.New("could not generate salted enc key: %v", err) + } + + access := uplink.NewEncryptionAccessWithDefaultKey(*saltenckey) + + instance.bucket, err = instance.project.OpenBucket(ctx, bucket, access) + if err != nil { + return nil, uplinkFailure.New("could not open bucket %q: %v", bucket, err) + } + + instance.logger = logger + + return &instance, nil +} + +func (s *StorjStorage) Type() string { + return "storj" +} + +func (s *StorjStorage) Head(token string, filename string) (contentType string, contentLength uint64, err error) { + key := storj.JoinPaths(token, filename) + + ctx := context.TODO() + + obj, err := s.bucket.OpenObject(ctx, key) + if err != nil { + return "", 0, fmt.Errorf("unable to open object %v", err) + } + contentType = obj.Meta.ContentType + contentLength = uint64(obj.Meta.Size) + + return +} + +func (s *StorjStorage) Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) { + key := storj.JoinPaths(token, filename) + + s.logger.Printf("Getting file %s from Storj Bucket", filename) + + ctx := context.TODO() + + obj, err := s.bucket.OpenObject(ctx, key) + if err != nil { + return nil, "", 0, uplinkFailure.New("unable to open object %v", err) + } + contentType = obj.Meta.ContentType + contentLength = uint64(obj.Meta.Size) + reader, err = obj.DownloadRange(ctx, 0, -1) + return +} + +func (s *StorjStorage) Delete(token string, filename string) (err error) { + key := storj.JoinPaths(token, filename) + + s.logger.Printf("Deleting file %s from Storj Bucket", filename) + + ctx := context.TODO() + + err = s.bucket.DeleteObject(ctx, key) + + return +} + +func (s *StorjStorage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) { + key := storj.JoinPaths(token, filename) + + s.logger.Printf("Uploading file %s to Storj Bucket", filename) + + ctx := context.TODO() + + err = s.bucket.UploadObject(ctx, key, reader, &uplink.UploadOptions{ContentType: contentType}) + if err != nil { + return uplinkFailure.New("could not upload: %v", err) + } + return nil +}