From 989debecb5279572be67a3f2d540aa895b9f6847 Mon Sep 17 00:00:00 2001 From: Remco Date: Tue, 28 Mar 2017 16:12:31 +0200 Subject: [PATCH] Add object metadata - support for max downloads - support for expiring downloads --- server/handlers.go | 158 ++++++++++++++++++++++++++++++++++++++++++--- server/server.go | 7 +- server/storage.go | 8 +++ 3 files changed, 163 insertions(+), 10 deletions(-) diff --git a/server/handlers.go b/server/handlers.go index 54c62dc..8c55db2 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -32,6 +32,7 @@ import ( "archive/zip" "bytes" "compress/gzip" + "encoding/json" "errors" "fmt" "html" @@ -48,6 +49,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" text_template "text/template" "time" @@ -256,6 +258,19 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) { contentLength := n + metadata := MetadataForRequest(contentType, r) + + buffer := &bytes.Buffer{} + if err := json.NewEncoder(buffer).Encode(metadata); err != nil { + log.Printf("%s", err.Error()) + http.Error(w, errors.New("Could not encode metadata").Error(), 500) + return + } else if err := s.storage.Put(token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil { + log.Printf("%s", err.Error()) + http.Error(w, errors.New("Could not save metadata").Error(), 500) + return + } + log.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType) if err = s.storage.Put(token, filename, reader, contentType, uint64(contentLength)); err != nil { @@ -271,6 +286,42 @@ func (s *Server) postHandler(w http.ResponseWriter, r *http.Request) { } } +type Metadata struct { + // ContentType is the original uploading content type + ContentType string + // Secret as knowledge to delete file + // Secret string + // Downloads is the actual number of downloads + Downloads int + // MaxDownloads contains the maximum numbers of downloads + MaxDownloads int + // MaxDate contains the max age of the file + MaxDate time.Time +} + +func MetadataForRequest(contentType string, r *http.Request) Metadata { + metadata := Metadata{ + ContentType: contentType, + MaxDate: time.Now().Add(time.Hour * 24 * 365 * 10), + Downloads: 0, + MaxDownloads: 99999999, + } + + if v := r.Header.Get("Max-Downloads"); v == "" { + } else if v, err := strconv.Atoi(v); err != nil { + } else { + metadata.MaxDownloads = v + } + + if v := r.Header.Get("Max-Days"); v == "" { + } else if v, err := strconv.Atoi(v); err != nil { + } else { + metadata.MaxDate = time.Now().Add(time.Hour * 24 * time.Duration(v)) + } + + return metadata +} + func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -332,6 +383,19 @@ func (s *Server) putHandler(w http.ResponseWriter, r *http.Request) { token := Encode(10000000 + int64(rand.Intn(1000000000))) + metadata := MetadataForRequest(contentType, r) + + buffer := &bytes.Buffer{} + if err := json.NewEncoder(buffer).Encode(metadata); err != nil { + log.Printf("%s", err.Error()) + http.Error(w, errors.New("Could not encode metadata").Error(), 500) + return + } else if err := s.storage.Put(token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil { + log.Printf("%s", err.Error()) + http.Error(w, errors.New("Could not save metadata").Error(), 500) + return + } + log.Printf("Uploading %s %s %d %s", token, filename, contentLength, contentType) var err error @@ -377,6 +441,63 @@ func getURL(r *http.Request) *url.URL { return &u } +func (s *Server) Lock(token, filename string) error { + key := path.Join(token, filename) + + if _, ok := s.locks[key]; !ok { + s.locks[key] = &sync.Mutex{} + } + + s.locks[key].Lock() + + return nil +} + +func (s *Server) Unlock(token, filename string) error { + key := path.Join(token, filename) + s.locks[key].Unlock() + + return nil +} + +func (s *Server) CheckMetadata(token, filename string) error { + s.Lock(token, filename) + defer s.Unlock(token, filename) + + var metadata Metadata + + r, _, _, err := s.storage.Get(token, fmt.Sprintf("%s.metadata", filename)) + if s.storage.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + + defer r.Close() + + if err := json.NewDecoder(r).Decode(&metadata); err != nil { + return err + } else if metadata.Downloads >= metadata.MaxDownloads { + return errors.New("MaxDownloads expired.") + } else if time.Now().After(metadata.MaxDate) { + return errors.New("MaxDate expired.") + } else { + // todo(nl5887): mutex? + + // update number of downloads + metadata.Downloads++ + + buffer := &bytes.Buffer{} + if err := json.NewEncoder(buffer).Encode(metadata); err != nil { + return errors.New("Could not encode metadata") + } else if err := s.storage.Put(token, fmt.Sprintf("%s.metadata", filename), buffer, "text/json", uint64(buffer.Len())); err != nil { + return errors.New("Could not save metadata") + } + } + + return nil +} + func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -400,6 +521,11 @@ func (s *Server) zipHandler(w http.ResponseWriter, r *http.Request) { token := strings.Split(key, "/")[0] filename := sanitize(strings.Split(key, "/")[1]) + if err := s.CheckMetadata(token, filename); err != nil { + log.Printf("Error metadata: %s", err.Error()) + continue + } + reader, _, _, err := s.storage.Get(token, filename) if err != nil { @@ -471,6 +597,11 @@ func (s *Server) tarGzHandler(w http.ResponseWriter, r *http.Request) { token := strings.Split(key, "/")[0] filename := sanitize(strings.Split(key, "/")[1]) + if err := s.CheckMetadata(token, filename); err != nil { + log.Printf("Error metadata: %s", err.Error()) + continue + } + reader, _, contentLength, err := s.storage.Get(token, filename) if err != nil { if s.storage.IsNotExist(err) { @@ -523,6 +654,11 @@ func (s *Server) tarHandler(w http.ResponseWriter, r *http.Request) { token := strings.Split(key, "/")[0] filename := strings.Split(key, "/")[1] + if err := s.CheckMetadata(token, filename); err != nil { + log.Printf("Error metadata: %s", err.Error()) + continue + } + reader, _, contentLength, err := s.storage.Get(token, filename) if err != nil { if s.storage.IsNotExist(err) { @@ -563,16 +699,20 @@ func (s *Server) getHandler(w http.ResponseWriter, r *http.Request) { token := vars["token"] filename := vars["filename"] + if err := s.CheckMetadata(token, filename); err != nil { + log.Printf("Error metadata: %s", err.Error()) + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + reader, contentType, contentLength, err := s.storage.Get(token, filename) - if err != nil { - if s.storage.IsNotExist(err) { - http.Error(w, "File not found", 404) - return - } else { - log.Printf("%s", err.Error()) - http.Error(w, "Could not retrieve file.", 500) - return - } + if s.storage.IsNotExist(err) { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } else if err != nil { + log.Printf("%s", err.Error()) + http.Error(w, "Could not retrieve file.", 500) + return } defer reader.Close() diff --git a/server/server.go b/server/server.go index a7d83cf..40cd5c9 100644 --- a/server/server.go +++ b/server/server.go @@ -35,6 +35,7 @@ import ( "os" "os/signal" "strings" + "sync" "syscall" "time" @@ -177,6 +178,8 @@ type Server struct { profilerEnabled bool + locks map[string]*sync.Mutex + storage Storage forceHTTPs bool @@ -198,7 +201,9 @@ type Server struct { } func New(options ...OptionFn) (*Server, error) { - s := &Server{} + s := &Server{ + locks: map[string]*sync.Mutex{}, + } for _, optionFn := range options { optionFn(s) diff --git a/server/storage.go b/server/storage.go index c895cbb..1c92771 100644 --- a/server/storage.go +++ b/server/storage.go @@ -72,6 +72,10 @@ func (s *LocalStorage) Get(token string, filename string) (reader io.ReadCloser, } func (s *LocalStorage) IsNotExist(err error) bool { + if err == nil { + return false + } + return os.IsNotExist(err) } @@ -137,6 +141,10 @@ func (s *S3Storage) Head(token string, filename string) (contentType string, con } func (s *S3Storage) IsNotExist(err error) bool { + if err == nil { + return false + } + log.Printf("IsNotExist: %s, %#v", err.Error(), err) b := (err.Error() == "The specified key does not exist.")