|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482 |
- // Copyright 2017 Google Inc. All Rights Reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
-
- // +build integration
-
- package storage
-
- import (
- "bytes"
- "encoding/base64"
- "errors"
- "fmt"
- "io/ioutil"
- "log"
- "net/http"
- "os"
- "strings"
- "testing"
-
- "golang.org/x/net/context"
- "golang.org/x/oauth2"
- "golang.org/x/oauth2/google"
- "google.golang.org/api/googleapi"
- storage "google.golang.org/api/storage/v1"
- )
-
- type object struct {
- name, contents string
- }
-
- var (
- projectID string
- bucket string
- objects = []object{
- {"obj1", testContents},
- {"obj2", testContents},
- {"obj/with/slashes", testContents},
- {"resumable", testContents},
- {"large", strings.Repeat("a", 514)}, // larger than the first section of content that is sniffed by ContentSniffer.
- }
- aclObjects = []string{"acl1", "acl2"}
- copyObj = "copy-object"
- )
-
- const (
- envProject = "GCLOUD_TESTS_GOLANG_PROJECT_ID"
- envPrivateKey = "GCLOUD_TESTS_GOLANG_KEY"
- // NOTE that running this test on a bucket deletes ALL contents of the bucket!
- envBucket = "GCLOUD_TESTS_GOLANG_DESTRUCTIVE_TEST_BUCKET_NAME"
- testContents = "some text that will be saved to a bucket object"
- )
-
- func verifyAcls(obj *storage.Object, wantDomainRole, wantAllUsersRole string) (err error) {
- var gotDomainRole, gotAllUsersRole string
- for _, acl := range obj.Acl {
- if acl.Entity == "domain-google.com" {
- gotDomainRole = acl.Role
- }
- if acl.Entity == "allUsers" {
- gotAllUsersRole = acl.Role
- }
- }
- if gotDomainRole != wantDomainRole {
- err = fmt.Errorf("domain-google.com role = %q; want %q", gotDomainRole, wantDomainRole)
- }
- if gotAllUsersRole != wantAllUsersRole {
- err = fmt.Errorf("allUsers role = %q; want %q; %v", gotAllUsersRole, wantAllUsersRole, err)
- }
- return err
- }
-
- // TODO(gmlewis): Move this to a common location.
- func tokenSource(ctx context.Context, scopes ...string) (oauth2.TokenSource, error) {
- keyFile := os.Getenv(envPrivateKey)
- if keyFile == "" {
- return nil, errors.New(envPrivateKey + " not set")
- }
- jsonKey, err := ioutil.ReadFile(keyFile)
- if err != nil {
- return nil, fmt.Errorf("unable to read %q: %v", keyFile, err)
- }
- conf, err := google.JWTConfigFromJSON(jsonKey, scopes...)
- if err != nil {
- return nil, fmt.Errorf("google.JWTConfigFromJSON: %v", err)
- }
- return conf.TokenSource(ctx), nil
- }
-
- const defaultType = "text/plain; charset=utf-8"
-
- // writeObject writes some data and default metadata to the specified object.
- // Resumable upload is used if resumable is true.
- // The written data is returned.
- func writeObject(s *storage.Service, bucket, obj string, resumable bool, contents string) error {
- o := &storage.Object{
- Bucket: bucket,
- Name: obj,
- ContentType: defaultType,
- ContentEncoding: "utf-8",
- ContentLanguage: "en",
- Metadata: map[string]string{"foo": "bar"},
- }
- f := strings.NewReader(contents)
- insert := s.Objects.Insert(bucket, o)
- if resumable {
- insert.ResumableMedia(context.Background(), f, int64(len(contents)), defaultType)
- } else {
- insert.Media(f)
- }
- _, err := insert.Do()
- return err
- }
-
- func checkMetadata(t *testing.T, s *storage.Service, bucket, obj string) {
- o, err := s.Objects.Get(bucket, obj).Do()
- if err != nil {
- t.Error(err)
- }
- if got, want := o.Name, obj; got != want {
- t.Errorf("name of %q = %q; want %q", obj, got, want)
- }
- if got, want := o.ContentType, defaultType; got != want {
- t.Errorf("contentType of %q = %q; want %q", obj, got, want)
- }
- if got, want := o.Metadata["foo"], "bar"; got != want {
- t.Errorf("metadata entry foo of %q = %q; want %q", obj, got, want)
- }
- }
-
- func createService() *storage.Service {
- if projectID = os.Getenv(envProject); projectID == "" {
- log.Print("no project ID specified")
- return nil
- }
- if bucket = os.Getenv(envBucket); bucket == "" {
- log.Print("no bucket specified")
- return nil
- }
-
- ctx := context.Background()
- ts, err := tokenSource(ctx, storage.DevstorageFullControlScope)
- if err != nil {
- log.Printf("tokenSource: %v", err)
- return nil
- }
- client := oauth2.NewClient(ctx, ts)
- s, err := storage.New(client)
- if err != nil {
- log.Printf("unable to create service: %v", err)
- return nil
- }
- return s
- }
-
- func TestMain(m *testing.M) {
- if err := cleanup(); err != nil {
- log.Fatalf("Pre-test cleanup failed: %v", err)
- }
- exit := m.Run()
- if err := cleanup(); err != nil {
- log.Fatalf("Post-test cleanup failed: %v", err)
- }
- os.Exit(exit)
- }
-
- func TestContentType(t *testing.T) {
- s := createService()
- if s == nil {
- t.Fatal("Could not create service")
- }
-
- type testCase struct {
- objectContentType string
- useOptionContentType bool
- optionContentType string
-
- wantContentType string
- }
-
- // The Media method will use resumable upload if the supplied data is
- // larger than googleapi.DefaultUploadChunkSize We run the following
- // tests with two different file contents: one that will trigger
- // resumable upload, and one that won't.
- forceResumableData := bytes.Repeat([]byte("a"), googleapi.DefaultUploadChunkSize+1)
- smallData := bytes.Repeat([]byte("a"), 2)
-
- // In the following test, the content type, if any, in the Object struct is always "text/plain".
- // The content type configured via googleapi.ContentType, if any, is always "text/html".
- for _, tc := range []testCase{
- // With content type specified in the object struct
- // Temporarily disable this test during rollout of strict Content-Type.
- // TODO(djd): Re-enable once strict check is 100%.
- // {
- // objectContentType: "text/plain",
- // useOptionContentType: true,
- // optionContentType: "text/html",
- // wantContentType: "text/html",
- // },
- {
- objectContentType: "text/plain",
- useOptionContentType: true,
- optionContentType: "",
- wantContentType: "text/plain",
- },
- {
- objectContentType: "text/plain",
- useOptionContentType: false,
- wantContentType: "text/plain",
- },
-
- // Without content type specified in the object struct
- {
- useOptionContentType: true,
- optionContentType: "text/html",
- wantContentType: "text/html",
- },
- {
- useOptionContentType: true,
- optionContentType: "",
- wantContentType: "", // Result is an object without a content type.
- },
- {
- useOptionContentType: false,
- wantContentType: "text/plain; charset=utf-8", // sniffed.
- },
- } {
- // The behavior should be the same, regardless of whether resumable upload is used or not.
- for _, data := range [][]byte{smallData, forceResumableData} {
- o := &storage.Object{
- Bucket: bucket,
- Name: "test-content-type",
- ContentType: tc.objectContentType,
- }
- call := s.Objects.Insert(bucket, o)
- var opts []googleapi.MediaOption
- if tc.useOptionContentType {
- opts = append(opts, googleapi.ContentType(tc.optionContentType))
- }
- call.Media(bytes.NewReader(data), opts...)
-
- _, err := call.Do()
- if err != nil {
- t.Fatalf("unable to insert object %q: %v", o.Name, err)
- }
-
- readObj, err := s.Objects.Get(bucket, o.Name).Do()
- if err != nil {
- t.Error(err)
- }
- if got, want := readObj.ContentType, tc.wantContentType; got != want {
- t.Errorf("contentType of %q; got %q; want %q", o.Name, got, want)
- }
- }
- }
- }
-
- func TestFunctions(t *testing.T) {
- s := createService()
- if s == nil {
- t.Fatal("Could not create service")
- }
-
- t.Logf("Listing buckets for project %q", projectID)
- var numBuckets int
- pageToken := ""
- for {
- call := s.Buckets.List(projectID)
- if pageToken != "" {
- call.PageToken(pageToken)
- }
- resp, err := call.Do()
- if err != nil {
- t.Fatalf("unable to list buckets for project %q: %v", projectID, err)
- }
- numBuckets += len(resp.Items)
- if pageToken = resp.NextPageToken; pageToken == "" {
- break
- }
- }
- if numBuckets == 0 {
- t.Fatalf("no buckets found for project %q", projectID)
- }
-
- for _, obj := range objects {
- t.Logf("Writing %q", obj.name)
- // TODO(mcgreevy): stop relying on "resumable" name to determine whether to
- // do a resumable upload.
- err := writeObject(s, bucket, obj.name, obj.name == "resumable", obj.contents)
- if err != nil {
- t.Fatalf("unable to insert object %q: %v", obj.name, err)
- }
- }
-
- for _, obj := range objects {
- t.Logf("Reading %q", obj.name)
- resp, err := s.Objects.Get(bucket, obj.name).Download()
- if err != nil {
- t.Fatalf("unable to get object %q: %v", obj.name, err)
- }
- slurp, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- t.Fatalf("unable to read response body %q: %v", obj.name, err)
- }
- resp.Body.Close()
- if got, want := string(slurp), obj.contents; got != want {
- t.Errorf("contents of %q = %q; want %q", obj.name, got, want)
- }
- }
-
- name := "obj-not-exists"
- if _, err := s.Objects.Get(bucket, name).Download(); !isError(err, http.StatusNotFound) {
- t.Errorf("object %q should not exist, err = %v", name, err)
- } else {
- t.Log("Successfully tested StatusNotFound.")
- }
-
- for _, obj := range objects {
- t.Logf("Checking %q metadata", obj.name)
- checkMetadata(t, s, bucket, obj.name)
- }
-
- name = objects[0].name
-
- t.Logf("Rewriting %q to %q", name, copyObj)
- copy, err := s.Objects.Rewrite(bucket, name, bucket, copyObj, nil).Do()
- if err != nil {
- t.Fatalf("unable to rewrite object %q to %q: %v", name, copyObj, err)
- }
- if copy.Resource.Name != copyObj {
- t.Errorf("copy object's name = %q; want %q", copy.Resource.Name, copyObj)
- }
- if copy.Resource.Bucket != bucket {
- t.Errorf("copy object's bucket = %q; want %q", copy.Resource.Bucket, bucket)
- }
-
- // Note that arrays such as ACLs below are completely overwritten using Patch
- // semantics, so these must be updated in a read-modify-write sequence of operations.
- // See https://cloud.google.com/storage/docs/json_api/v1/how-tos/performance#patch-semantics
- // for more details.
- t.Logf("Updating attributes of %q", name)
- obj, err := s.Objects.Get(bucket, name).Projection("full").Fields("acl").Do()
- if err != nil {
- t.Fatalf("Objects.Get(%q, %q): %v", bucket, name, err)
- }
- if err := verifyAcls(obj, "", ""); err != nil {
- t.Errorf("before update ACLs: %v", err)
- }
- obj.ContentType = "text/html"
- for _, entity := range []string{"domain-google.com", "allUsers"} {
- obj.Acl = append(obj.Acl, &storage.ObjectAccessControl{Entity: entity, Role: "READER"})
- }
- updated, err := s.Objects.Patch(bucket, name, obj).Projection("full").Fields("contentType", "acl").Do()
- if err != nil {
- t.Fatalf("Objects.Patch(%q, %q, %#v) failed with %v", bucket, name, obj, err)
- }
- if want := "text/html"; updated.ContentType != want {
- t.Errorf("updated.ContentType == %q; want %q", updated.ContentType, want)
- }
- if err := verifyAcls(updated, "READER", "READER"); err != nil {
- t.Errorf("after update ACLs: %v", err)
- }
-
- t.Log("Testing checksums")
- checksumCases := []struct {
- name string
- contents string
- size uint64
- md5 string
- crc32c uint32
- }{
- {
- name: "checksum-object",
- contents: "helloworld",
- size: 10,
- md5: "fc5e038d38a57032085441e7fe7010b0",
- crc32c: 1456190592,
- },
- {
- name: "zero-object",
- contents: "",
- size: 0,
- md5: "d41d8cd98f00b204e9800998ecf8427e",
- crc32c: 0,
- },
- }
- for _, c := range checksumCases {
- f := strings.NewReader(c.contents)
- o := &storage.Object{
- Bucket: bucket,
- Name: c.name,
- ContentType: defaultType,
- ContentEncoding: "utf-8",
- ContentLanguage: "en",
- }
- obj, err := s.Objects.Insert(bucket, o).Media(f).Do()
- if err != nil {
- t.Fatalf("unable to insert object %q: %v", obj, err)
- }
- if got, want := obj.Size, c.size; got != want {
- t.Errorf("object %q size = %v; want %v", c.name, got, want)
- }
- md5, err := base64.StdEncoding.DecodeString(obj.Md5Hash)
- if err != nil {
- t.Fatalf("object %q base64 decode of MD5 %q: %v", c.name, obj.Md5Hash, err)
- }
- if got, want := fmt.Sprintf("%x", md5), c.md5; got != want {
- t.Errorf("object %q MD5 = %q; want %q", c.name, got, want)
- }
- var crc32c uint32
- d, err := base64.StdEncoding.DecodeString(obj.Crc32c)
- if err != nil {
- t.Errorf("object %q base64 decode of CRC32 %q: %v", c.name, obj.Crc32c, err)
- }
- if err == nil && len(d) == 4 {
- crc32c = uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3])
- }
- if got, want := crc32c, c.crc32c; got != want {
- t.Errorf("object %q CRC32C = %v; want %v", c.name, got, want)
- }
- }
- }
-
- // cleanup destroys ALL objects in the bucket!
- func cleanup() error {
- s := createService()
- if s == nil {
- return errors.New("Could not create service")
- }
-
- var pageToken string
- var failed bool
- for {
- call := s.Objects.List(bucket)
- if pageToken != "" {
- call.PageToken(pageToken)
- }
- resp, err := call.Do()
- if err != nil {
- return fmt.Errorf("cleanup list failed: %v", err)
- }
- for _, obj := range resp.Items {
- log.Printf("Cleanup deletion of %q", obj.Name)
- if err := s.Objects.Delete(bucket, obj.Name).Do(); err != nil {
- // Print the error out, but keep going.
- log.Printf("Cleanup deletion of %q failed: %v", obj.Name, err)
- failed = true
- }
- if _, err := s.Objects.Get(bucket, obj.Name).Download(); !isError(err, http.StatusNotFound) {
- log.Printf("object %q should not exist, err = %v", obj.Name, err)
- failed = true
- } else {
- log.Printf("Successfully deleted %q.", obj.Name)
- }
- }
- if pageToken = resp.NextPageToken; pageToken == "" {
- break
- }
- }
- if failed {
- return errors.New("Failed to delete at least one object")
- }
- return nil
- }
-
- func isError(err error, code int) bool {
- if err == nil {
- return false
- }
- ae, ok := err.(*googleapi.Error)
- return ok && ae.Code == code
- }
|