@@ -18,6 +18,7 @@ FROM scratch AS final | |||||
LABEL maintainer="Andrea Spacca <andrea.spacca@gmail.com>" | LABEL maintainer="Andrea Spacca <andrea.spacca@gmail.com>" | ||||
COPY --from=build /go/bin/transfersh /go/bin/transfersh | COPY --from=build /go/bin/transfersh /go/bin/transfersh | ||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt | |||||
ENTRYPOINT ["/go/bin/transfersh", "--listener", ":8080"] | ENTRYPOINT ["/go/bin/transfersh", "--listener", ":8080"] | ||||
@@ -163,6 +163,7 @@ provider | which storage provider to use | (s3, grdrive or local) | | |||||
aws-access-key | aws access key | | AWS_ACCESS_KEY | aws-access-key | aws access key | | AWS_ACCESS_KEY | ||||
aws-secret-key | aws access key | | AWS_SECRET_KEY | aws-secret-key | aws access key | | AWS_SECRET_KEY | ||||
bucket | aws bucket | | BUCKET | bucket | aws bucket | | BUCKET | ||||
s3-region | region of the s3 bucket | eu-west-1 | S3_REGION | |||||
s3-no-multipart | disables s3 multipart upload | false | | | s3-no-multipart | disables s3 multipart upload | false | | | ||||
basedir | path storage for local/gdrive provider| | | basedir | path storage for local/gdrive provider| | | ||||
gdrive-client-json-filepath | path to oauth client json config for gdrive provider| | | gdrive-client-json-filepath | path to oauth client json config for gdrive provider| | | ||||
@@ -202,6 +203,21 @@ For easy deployment, we've created a Docker container. | |||||
docker run --publish 8080:8080 dutchcoders/transfer.sh:latest --provider local --basedir /tmp/ | docker run --publish 8080:8080 dutchcoders/transfer.sh:latest --provider local --basedir /tmp/ | ||||
``` | ``` | ||||
## S3 Usage | |||||
For the usage with a AWS S3 Bucket, you just need to specify the following options: | |||||
- provider | |||||
- aws-access-key | |||||
- aws-secret-key | |||||
- bucket | |||||
- s3-region | |||||
If you specify the s3-region, you don't need to set the endpoint URL since the correct endpoint will used automatically. | |||||
### Custom S3 providers | |||||
To use a custom non-AWS S3 provider, you need to specify the endpoint as definied from your cloud provider. | |||||
## Contributions | ## Contributions | ||||
Contributions are welcome. | Contributions are welcome. | ||||
@@ -99,9 +99,15 @@ var globalFlags = []cli.Flag{ | |||||
cli.StringFlag{ | cli.StringFlag{ | ||||
Name: "s3-endpoint", | Name: "s3-endpoint", | ||||
Usage: "", | Usage: "", | ||||
Value: "http://s3-eu-west-1.amazonaws.com", | |||||
Value: "", | |||||
EnvVar: "S3_ENDPOINT", | EnvVar: "S3_ENDPOINT", | ||||
}, | }, | ||||
cli.StringFlag{ | |||||
Name: "s3-region", | |||||
Usage: "", | |||||
Value: "eu-west-1", | |||||
EnvVar: "S3_REGION", | |||||
}, | |||||
cli.StringFlag{ | cli.StringFlag{ | ||||
Name: "aws-access-key", | Name: "aws-access-key", | ||||
Usage: "", | Usage: "", | ||||
@@ -332,7 +338,7 @@ func New() *Cmd { | |||||
panic("secret-key not set.") | panic("secret-key not set.") | ||||
} else if bucket := c.String("bucket"); bucket == "" { | } else if bucket := c.String("bucket"); bucket == "" { | ||||
panic("bucket not set.") | panic("bucket not set.") | ||||
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger, c.Bool("s3-no-multipart")); err != nil { | |||||
} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-region"), c.String("s3-endpoint"), logger, c.Bool("s3-no-multipart")); err != nil { | |||||
panic(err) | panic(err) | ||||
} else { | } else { | ||||
options = append(options, server.UseStorage(storage)) | options = append(options, server.UseStorage(storage)) | ||||
@@ -5,13 +5,13 @@ go 1.12 | |||||
require ( | require ( | ||||
github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14 | github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14 | ||||
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2 | github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2 | ||||
github.com/aws/aws-sdk-go v1.20.6 | |||||
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e | github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e | ||||
github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329 | github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329 | ||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20190520063132-37110d436c89 | github.com/dutchcoders/transfer.sh-web v0.0.0-20190520063132-37110d436c89 | ||||
github.com/elazarl/go-bindata-assetfs v1.0.0 | github.com/elazarl/go-bindata-assetfs v1.0.0 | ||||
github.com/fatih/color v1.7.0 | github.com/fatih/color v1.7.0 | ||||
github.com/garyburd/redigo v1.6.0 // indirect | github.com/garyburd/redigo v1.6.0 // indirect | ||||
github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 | |||||
github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 | github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 | ||||
github.com/gorilla/mux v1.7.1 | github.com/gorilla/mux v1.7.1 | ||||
github.com/gorilla/securecookie v1.1.1 // indirect | github.com/gorilla/securecookie v1.1.1 // indirect | ||||
@@ -23,8 +23,8 @@ require ( | |||||
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect | github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect | ||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect | github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect | ||||
github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 | github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 | ||||
github.com/stretchr/testify v1.3.0 // indirect | |||||
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce | github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce | ||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec // indirect | |||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 | ||||
golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 | golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 | ||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a | golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a | ||||
@@ -6,15 +6,15 @@ github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14 h1:3zOOc7WdrATDX | |||||
github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14/go.mod h1:+VFiaivV54Sa94ijzA/ZHQLoHuoUIS9hIqCK6f/76Zw= | github.com/PuerkitoBio/ghost v0.0.0-20160324114900-206e6e460e14/go.mod h1:+VFiaivV54Sa94ijzA/ZHQLoHuoUIS9hIqCK6f/76Zw= | ||||
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2 h1:sIvihcW4qpN5qGSjmrsDDAbLpEq5tuHjJJfWY0Hud5Y= | github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2 h1:sIvihcW4qpN5qGSjmrsDDAbLpEq5tuHjJJfWY0Hud5Y= | ||||
github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2/go.mod h1:3YwJE8rEisS9eraee0hygGG4G3gqX8H8Nyu+nPTUnGU= | github.com/VojtechVitek/ratelimit v0.0.0-20160722140851-dc172bc0f6d2/go.mod h1:3YwJE8rEisS9eraee0hygGG4G3gqX8H8Nyu+nPTUnGU= | ||||
github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4= | |||||
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= | |||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= | ||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= | |||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |||||
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e h1:rcHHSQqzCgvlwP0I/fQ8rQMn/MpHE5gWSLdtpxtP6KQ= | github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e h1:rcHHSQqzCgvlwP0I/fQ8rQMn/MpHE5gWSLdtpxtP6KQ= | ||||
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e/go.mod h1:Byz7q8MSzSPkouskHJhX0er2mZY/m0Vj5bMeMCkkyY4= | github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e/go.mod h1:Byz7q8MSzSPkouskHJhX0er2mZY/m0Vj5bMeMCkkyY4= | ||||
github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329 h1:ERqCkG/uSyT74P1m/j9yR+so+7ynY4fbTvLY/Mr1ZMg= | github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329 h1:ERqCkG/uSyT74P1m/j9yR+so+7ynY4fbTvLY/Mr1ZMg= | ||||
github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329/go.mod h1:G5qOfE5bQZ5scycLpB7fYWgN4y3xdfXo+pYWM8z2epY= | github.com/dutchcoders/go-virustotal v0.0.0-20140923143438-24cc8e6fa329/go.mod h1:G5qOfE5bQZ5scycLpB7fYWgN4y3xdfXo+pYWM8z2epY= | ||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20190121065949-e7d393abbb07 h1:L4PB9nsRpVJqpza1aVwroFcShe/N3wh2QcWVpca53jk= | |||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20190121065949-e7d393abbb07/go.mod h1:UjR1zlrq/R2Sef7e4q3TeJm4HcbLh4vRzlCEGJP+wLg= | |||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20190518121139-cc1ae43f8d69 h1:Eb27agCP67voPNMPDEPrXBp+IEWp4ephgX9B3jfR4Uk= | |||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20190518121139-cc1ae43f8d69/go.mod h1:UjR1zlrq/R2Sef7e4q3TeJm4HcbLh4vRzlCEGJP+wLg= | |||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20190520063132-37110d436c89 h1:FJc5t2SEtRmLXDWxHFQG4QB98Vqr7/60cIiUT2F5yAA= | github.com/dutchcoders/transfer.sh-web v0.0.0-20190520063132-37110d436c89 h1:FJc5t2SEtRmLXDWxHFQG4QB98Vqr7/60cIiUT2F5yAA= | ||||
github.com/dutchcoders/transfer.sh-web v0.0.0-20190520063132-37110d436c89/go.mod h1:UjR1zlrq/R2Sef7e4q3TeJm4HcbLh4vRzlCEGJP+wLg= | github.com/dutchcoders/transfer.sh-web v0.0.0-20190520063132-37110d436c89/go.mod h1:UjR1zlrq/R2Sef7e4q3TeJm4HcbLh4vRzlCEGJP+wLg= | ||||
github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= | github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= | ||||
@@ -23,8 +23,6 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= | |||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= | ||||
github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc= | github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc= | ||||
github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= | github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= | ||||
github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c= | |||||
github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8/go.mod h1:/Ya1YZsqLQp17bDgHdyE9/XBR1uIH1HKasTvLxcoM/A= | |||||
github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 h1:KRMr9A3qfbVM7iV/WcLY/rL5LICqwMHLhwRXKu99fXw= | github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 h1:KRMr9A3qfbVM7iV/WcLY/rL5LICqwMHLhwRXKu99fXw= | ||||
github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= | github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= | ||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= | ||||
@@ -40,6 +38,8 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC | |||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= | ||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= | ||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= | ||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= | |||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= | |||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= | ||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= | ||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= | ||||
@@ -56,16 +56,19 @@ github.com/minio/cli v1.3.0 h1:vB0iUpmyaH54+1jJJj62Aa0qFF3xO3i0J3IcKiM6bHM= | |||||
github.com/minio/cli v1.3.0/go.mod h1:hLsWNQy2wIf3FKFnMlH69f4RdEyn8nbRA2shaulTjGY= | github.com/minio/cli v1.3.0/go.mod h1:hLsWNQy2wIf3FKFnMlH69f4RdEyn8nbRA2shaulTjGY= | ||||
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= | github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= | ||||
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= | github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= | ||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | |||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | |||||
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= | ||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= | ||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= | ||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= | ||||
github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 h1:lpEzuenPuO1XNTeikEmvqYFcU37GVLl8SRNblzyvGBE= | github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 h1:lpEzuenPuO1XNTeikEmvqYFcU37GVLl8SRNblzyvGBE= | ||||
github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo= | github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo= | ||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | |||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= | |||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= | |||||
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= | github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= | ||||
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= | github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= | ||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks= | |||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw= | |||||
go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= | go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= | ||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= | ||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | ||||
@@ -1,9 +1,12 @@ | |||||
package server | package server | ||||
import ( | import ( | ||||
"bytes" | |||||
"encoding/json" | "encoding/json" | ||||
"fmt" | "fmt" | ||||
"github.com/aws/aws-sdk-go/aws" | |||||
"github.com/aws/aws-sdk-go/aws/awserr" | |||||
"github.com/aws/aws-sdk-go/aws/session" | |||||
"github.com/aws/aws-sdk-go/service/s3/s3manager" | |||||
"io" | "io" | ||||
"io/ioutil" | "io/ioutil" | ||||
"log" | "log" | ||||
@@ -11,11 +14,9 @@ import ( | |||||
"net/http" | "net/http" | ||||
"os" | "os" | ||||
"path/filepath" | "path/filepath" | ||||
"strconv" | |||||
"strings" | "strings" | ||||
"sync" | |||||
"github.com/goamz/goamz/s3" | |||||
"github.com/aws/aws-sdk-go/service/s3" | |||||
"golang.org/x/net/context" | "golang.org/x/net/context" | ||||
"golang.org/x/oauth2" | "golang.org/x/oauth2" | ||||
"golang.org/x/oauth2/google" | "golang.org/x/oauth2/google" | ||||
@@ -124,18 +125,17 @@ func (s *LocalStorage) Put(token string, filename string, reader io.Reader, cont | |||||
type S3Storage struct { | type S3Storage struct { | ||||
Storage | Storage | ||||
bucket *s3.Bucket | |||||
bucket string | |||||
session *session.Session | |||||
s3 *s3.S3 | |||||
logger *log.Logger | logger *log.Logger | ||||
noMultipart bool | noMultipart bool | ||||
} | } | ||||
func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, disableMultipart bool) (*S3Storage, error) { | |||||
bucket, err := getBucket(accessKey, secretKey, bucketName, endpoint) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
func NewS3Storage(accessKey, secretKey, bucketName, region, endpoint string, logger *log.Logger, disableMultipart bool) (*S3Storage, error) { | |||||
sess := getAwsSession(accessKey, secretKey, region, endpoint) | |||||
return &S3Storage{bucket: bucket, logger: logger, noMultipart: disableMultipart}, nil | |||||
return &S3Storage{bucket: bucketName, s3: s3.New(sess), session: sess, logger: logger, noMultipart: disableMultipart}, nil | |||||
} | } | ||||
func (s *S3Storage) Type() string { | func (s *S3Storage) Type() string { | ||||
@@ -145,16 +145,20 @@ func (s *S3Storage) Type() string { | |||||
func (s *S3Storage) Head(token string, filename string) (contentType string, contentLength uint64, err error) { | func (s *S3Storage) Head(token string, filename string) (contentType string, contentLength uint64, err error) { | ||||
key := fmt.Sprintf("%s/%s", token, filename) | key := fmt.Sprintf("%s/%s", token, filename) | ||||
headRequest := &s3.HeadObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | |||||
// content type , content length | // content type , content length | ||||
response, err := s.bucket.Head(key, map[string][]string{}) | |||||
if err != nil { | |||||
return | |||||
response, err := s.s3.HeadObject(headRequest) | |||||
if response.ContentType != nil { | |||||
contentType = *response.ContentType | |||||
} | } | ||||
contentType = response.Header.Get("Content-Type") | |||||
contentLength, err = strconv.ParseUint(response.Header.Get("Content-Length"), 10, 0) | |||||
if err != nil { | |||||
return | |||||
if response.ContentLength != nil { | |||||
contentLength = uint64(*response.ContentLength) | |||||
} | } | ||||
return | return | ||||
@@ -165,26 +169,32 @@ func (s *S3Storage) IsNotExist(err error) bool { | |||||
return false | return false | ||||
} | } | ||||
s.logger.Printf("IsNotExist: %s, %#v", err.Error(), err) | |||||
if aerr, ok := err.(awserr.Error); ok { | |||||
switch aerr.Code() { | |||||
case s3.ErrCodeNoSuchKey: | |||||
return true | |||||
} | |||||
} | |||||
b := (err.Error() == "The specified key does not exist.") | |||||
b = b || (err.Error() == "Access Denied") | |||||
return b | |||||
return false | |||||
} | } | ||||
func (s *S3Storage) Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) { | func (s *S3Storage) Get(token string, filename string) (reader io.ReadCloser, contentType string, contentLength uint64, err error) { | ||||
key := fmt.Sprintf("%s/%s", token, filename) | key := fmt.Sprintf("%s/%s", token, filename) | ||||
// content type , content length | |||||
response, err := s.bucket.GetResponse(key) | |||||
if err != nil { | |||||
return | |||||
getRequest := &s3.GetObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | } | ||||
contentType = response.Header.Get("Content-Type") | |||||
contentLength, err = strconv.ParseUint(response.Header.Get("Content-Length"), 10, 0) | |||||
if err != nil { | |||||
return | |||||
response, err := s.s3.GetObject(getRequest) | |||||
if response.ContentType != nil { | |||||
contentType = *response.ContentType | |||||
} | |||||
if response.ContentLength != nil { | |||||
contentLength = uint64(*response.ContentLength) | |||||
} | } | ||||
reader = response.Body | reader = response.Body | ||||
@@ -193,123 +203,23 @@ func (s *S3Storage) Get(token string, filename string) (reader io.ReadCloser, co | |||||
func (s *S3Storage) Delete(token string, filename string) (err error) { | func (s *S3Storage) Delete(token string, filename string) (err error) { | ||||
metadata := fmt.Sprintf("%s/%s.metadata", token, filename) | metadata := fmt.Sprintf("%s/%s.metadata", token, filename) | ||||
s.bucket.Del(metadata) | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
err = s.bucket.Del(key) | |||||
return | |||||
} | |||||
func (s *S3Storage) putMulti(key string, reader io.Reader, contentType string, contentLength uint64) (err error) { | |||||
var ( | |||||
multi *s3.Multi | |||||
parts []s3.Part | |||||
) | |||||
if multi, err = s.bucket.InitMulti(key, contentType, s3.Private); err != nil { | |||||
s.logger.Printf(err.Error()) | |||||
return | |||||
deleteRequest := &s3.DeleteObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(metadata), | |||||
} | } | ||||
// 20 mb parts | |||||
partsChan := make(chan interface{}) | |||||
// partsChan := make(chan s3.Part) | |||||
go func() { | |||||
// maximize to 20 threads | |||||
sem := make(chan int, 20) | |||||
index := 1 | |||||
var wg sync.WaitGroup | |||||
for { | |||||
// buffered in memory because goamz s3 multi needs seekable reader | |||||
var ( | |||||
buffer []byte = make([]byte, (1<<20)*10) | |||||
count int | |||||
err error | |||||
) | |||||
// Amazon expects parts of at least 5MB, except for the last one | |||||
if count, err = io.ReadAtLeast(reader, buffer, (1<<20)*5); err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { | |||||
s.logger.Printf(err.Error()) | |||||
return | |||||
} | |||||
// always send minimal 1 part | |||||
if err == io.EOF && index > 1 { | |||||
s.logger.Printf("Waiting for all parts to finish uploading.") | |||||
// wait for all parts to be finished uploading | |||||
wg.Wait() | |||||
// and close the channel | |||||
close(partsChan) | |||||
return | |||||
} | |||||
wg.Add(1) | |||||
sem <- 1 | |||||
// using goroutines because of retries when upload fails | |||||
go func(multi *s3.Multi, buffer []byte, index int) { | |||||
s.logger.Printf("Uploading part %d %d", index, len(buffer)) | |||||
defer func() { | |||||
s.logger.Printf("Finished part %d %d", index, len(buffer)) | |||||
wg.Done() | |||||
<-sem | |||||
}() | |||||
partReader := bytes.NewReader(buffer) | |||||
var part s3.Part | |||||
if part, err = multi.PutPart(index, partReader); err != nil { | |||||
s.logger.Printf("Error while uploading part %d %d %s", index, len(buffer), err.Error()) | |||||
partsChan <- err | |||||
return | |||||
} | |||||
s.logger.Printf("Finished uploading part %d %d", index, len(buffer)) | |||||
partsChan <- part | |||||
}(multi, buffer[:count], index) | |||||
index++ | |||||
} | |||||
}() | |||||
// wait for all parts to be uploaded | |||||
for part := range partsChan { | |||||
switch part.(type) { | |||||
case s3.Part: | |||||
parts = append(parts, part.(s3.Part)) | |||||
case error: | |||||
// abort multi upload | |||||
s.logger.Printf("Error during upload, aborting %s.", part.(error).Error()) | |||||
err = part.(error) | |||||
multi.Abort() | |||||
return | |||||
} | |||||
_, err = s.s3.DeleteObject(deleteRequest) | |||||
if err != nil { | |||||
return | |||||
} | } | ||||
s.logger.Printf("Completing upload %d parts", len(parts)) | |||||
if err = multi.Complete(parts); err != nil { | |||||
s.logger.Printf("Error during completing upload %d parts %s", len(parts), err.Error()) | |||||
return | |||||
key := fmt.Sprintf("%s/%s", token, filename) | |||||
deleteRequest = &s3.DeleteObjectInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
} | } | ||||
s.logger.Printf("Completed uploading %d", len(parts)) | |||||
_, err = s.s3.DeleteObject(deleteRequest) | |||||
return | return | ||||
} | } | ||||
@@ -318,17 +228,26 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content | |||||
key := fmt.Sprintf("%s/%s", token, filename) | key := fmt.Sprintf("%s/%s", token, filename) | ||||
s.logger.Printf("Uploading file %s to S3 Bucket", filename) | s.logger.Printf("Uploading file %s to S3 Bucket", filename) | ||||
var concurrency int | |||||
if !s.noMultipart { | if !s.noMultipart { | ||||
err = s.putMulti(key, reader, contentType, contentLength) | |||||
concurrency = 20 | |||||
} else { | } else { | ||||
err = s.bucket.PutReader(key, reader, int64(contentLength), contentType, s3.Private, s3.Options{}) | |||||
} | |||||
if err != nil { | |||||
return | |||||
} | |||||
s.logger.Printf("Completed uploading %s", filename) | |||||
concurrency = 1 | |||||
} | |||||
// Create an uploader with the session and custom options | |||||
uploader := s3manager.NewUploader(s.session, func(u *s3manager.Uploader) { | |||||
u.PartSize = (1 << 20) * 5 // The minimum/default allowed part size is 5MB | |||||
u.Concurrency = concurrency // default is 5 | |||||
u.MaxUploadParts = concurrency | |||||
u.LeavePartsOnError = false | |||||
}) | |||||
_, err = uploader.Upload(&s3manager.UploadInput{ | |||||
Bucket: aws.String(s.bucket), | |||||
Key: aws.String(key), | |||||
Body: reader, | |||||
}) | |||||
return | return | ||||
} | } | ||||
@@ -25,56 +25,24 @@ THE SOFTWARE. | |||||
package server | package server | ||||
import ( | import ( | ||||
"github.com/aws/aws-sdk-go/aws/credentials" | |||||
"math" | "math" | ||||
"net/http" | "net/http" | ||||
"net/mail" | "net/mail" | ||||
"strconv" | "strconv" | ||||
"strings" | "strings" | ||||
"time" | |||||
"github.com/goamz/goamz/aws" | |||||
"github.com/goamz/goamz/s3" | |||||
"github.com/aws/aws-sdk-go/aws" | |||||
"github.com/aws/aws-sdk-go/aws/session" | |||||
"github.com/golang/gddo/httputil/header" | "github.com/golang/gddo/httputil/header" | ||||
) | ) | ||||
func getBucket(accessKey, secretKey, bucket, endpoint string) (*s3.Bucket, error) { | |||||
auth, err := aws.GetAuth(accessKey, secretKey, "", time.Time{}) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
var EUWestWithoutHTTPS = aws.Region{ | |||||
Name: "eu-west-1", | |||||
EC2Endpoint: "https://ec2.eu-west-1.amazonaws.com", | |||||
S3Endpoint: endpoint, | |||||
S3BucketEndpoint: "", | |||||
S3LocationConstraint: true, | |||||
S3LowercaseBucket: true, | |||||
SDBEndpoint: "https://sdb.eu-west-1.amazonaws.com", | |||||
SESEndpoint: "https://email.eu-west-1.amazonaws.com", | |||||
SNSEndpoint: "https://sns.eu-west-1.amazonaws.com", | |||||
SQSEndpoint: "https://sqs.eu-west-1.amazonaws.com", | |||||
IAMEndpoint: "https://iam.amazonaws.com", | |||||
ELBEndpoint: "https://elasticloadbalancing.eu-west-1.amazonaws.com", | |||||
DynamoDBEndpoint: "https://dynamodb.eu-west-1.amazonaws.com", | |||||
CloudWatchServicepoint: aws.ServiceInfo{ | |||||
Endpoint: "https://monitoring.eu-west-1.amazonaws.com", | |||||
Signer: aws.V2Signature, | |||||
}, | |||||
AutoScalingEndpoint: "https://autoscaling.eu-west-1.amazonaws.com", | |||||
RDSEndpoint: aws.ServiceInfo{ | |||||
Endpoint: "https://rds.eu-west-1.amazonaws.com", | |||||
Signer: aws.V2Signature, | |||||
}, | |||||
STSEndpoint: "https://sts.amazonaws.com", | |||||
CloudFormationEndpoint: "https://cloudformation.eu-west-1.amazonaws.com", | |||||
ECSEndpoint: "https://ecs.eu-west-1.amazonaws.com", | |||||
DynamoDBStreamsEndpoint: "https://streams.dynamodb.eu-west-1.amazonaws.com", | |||||
} | |||||
conn := s3.New(auth, EUWestWithoutHTTPS) | |||||
b := conn.Bucket(bucket) | |||||
return b, nil | |||||
func getAwsSession(accessKey, secretKey, region, endpoint string) *session.Session { | |||||
return session.Must(session.NewSession(&aws.Config{ | |||||
Region: aws.String(region), | |||||
Endpoint: aws.String(endpoint), | |||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), | |||||
})) | |||||
} | } | ||||
func formatNumber(format string, s uint64) string { | func formatNumber(format string, s uint64) string { | ||||