From ff29aed05dc63e0ab654fbdeb9f0018acbc270a2 Mon Sep 17 00:00:00 2001 From: wd256 Date: Wed, 15 Aug 2018 04:53:39 +1000 Subject: [PATCH] gcs: Translate S3 user-defined metadata prefix to/from GCS custom metadata prefix (#6270) --- cmd/gateway/gcs/gateway-gcs.go | 47 ++++++++----- cmd/gateway/gcs/gateway-gcs_test.go | 104 ++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 19 deletions(-) diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index eedbd8000..cd1135a2b 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -23,8 +23,8 @@ import ( "fmt" "io" "io/ioutil" - "math" + "net/http" "os" "regexp" "strings" @@ -768,22 +768,27 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo { // Refer https://cloud.google.com/storage/docs/hashes-etags. Use CRC32C for ETag metadata := make(map[string]string) for k, v := range attrs.Metadata { + k = http.CanonicalHeaderKey(k) + // Translate the GCS custom metadata prefix + if strings.HasPrefix(k, "X-Goog-Meta-") { + k = strings.Replace(k, "X-Goog-Meta-", "X-Amz-Meta-", 1) + } metadata[k] = v } if attrs.ContentType != "" { - metadata["content-type"] = attrs.ContentType + metadata["Content-Type"] = attrs.ContentType } if attrs.ContentEncoding != "" { - metadata["content-encoding"] = attrs.ContentEncoding + metadata["Content-Encoding"] = attrs.ContentEncoding } if attrs.CacheControl != "" { - metadata["cache-control"] = attrs.CacheControl + metadata["Cache-Control"] = attrs.CacheControl } if attrs.ContentDisposition != "" { - metadata["content-disposition"] = attrs.ContentDisposition + metadata["Content-Disposition"] = attrs.ContentDisposition } if attrs.ContentLanguage != "" { - metadata["content-language"] = attrs.ContentLanguage + metadata["Content-Language"] = attrs.ContentLanguage } return minio.ObjectInfo{ Name: attrs.Name, @@ -799,21 +804,25 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo { // applyMetadataToGCSAttrs applies metadata to a GCS ObjectAttrs instance func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAttrs) { - attrs.ContentType = metadata["content-type"] - attrs.ContentEncoding = metadata["content-encoding"] - attrs.CacheControl = metadata["cache-control"] - attrs.ContentDisposition = metadata["content-disposition"] - attrs.ContentLanguage = metadata["content-language"] - attrs.Metadata = make(map[string]string) for k, v := range metadata { - attrs.Metadata[k] = v - } - // Filter metadata which is stored as a unique attribute - for _, key := range []string{ - "content-type", "content-encoding", "cache-control", "content-disposition", "content-language", - } { - delete(attrs.Metadata, key) + k = http.CanonicalHeaderKey(k) + switch { + case strings.HasPrefix(k, "X-Amz-Meta-"): + // Translate the S3 user-defined metadata prefix + k = strings.Replace(k, "X-Amz-Meta-", "x-goog-meta-", 1) + attrs.Metadata[k] = v + case k == "Content-Type": + attrs.ContentType = v + case k == "Content-Encoding": + attrs.ContentEncoding = v + case k == "Cache-Control": + attrs.CacheControl = v + case k == "Content-Disposition": + attrs.ContentDisposition = v + case k == "Content-Language": + attrs.ContentLanguage = v + } } } diff --git a/cmd/gateway/gcs/gateway-gcs_test.go b/cmd/gateway/gcs/gateway-gcs_test.go index 24064ee43..68d67fcd8 100644 --- a/cmd/gateway/gcs/gateway-gcs_test.go +++ b/cmd/gateway/gcs/gateway-gcs_test.go @@ -23,7 +23,9 @@ import ( "path" "reflect" "testing" + "time" + "cloud.google.com/go/storage" "google.golang.org/api/googleapi" miniogo "github.com/minio/minio-go" @@ -393,3 +395,105 @@ func TestGCSToObjectError(t *testing.T) { } } } + +func TestS3MetaToGCSAttributes(t *testing.T) { + headers := map[string]string{ + "accept-encoding": "gzip", + "content-encoding": "gzip", + "cache-control": "age: 3600", + "content-disposition": "dummy", + "content-type": "application/javascript", + "Content-Language": "en", + "X-Amz-Meta-Hdr": "value", + "X-Amz-Meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=", + "X-Amz-Meta-X-Amz-Matdesc": "{}", + "X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==", + } + // Only X-Amz-Meta- prefixed entries will be returned in + // Metadata (without the prefix!) + expectedHeaders := map[string]string{ + "x-goog-meta-Hdr": "value", + "x-goog-meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=", + "x-goog-meta-X-Amz-Matdesc": "{}", + "x-goog-meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==", + } + + attrs := storage.ObjectAttrs{} + applyMetadataToGCSAttrs(headers, &attrs) + + if !reflect.DeepEqual(attrs.Metadata, expectedHeaders) { + t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, attrs.Metadata) + } + + if attrs.CacheControl != headers["cache-control"] { + t.Fatalf("Test failed with Cache-Control mistmatch, expected %s, got %s", headers["cache-control"], attrs.CacheControl) + } + if attrs.ContentDisposition != headers["content-disposition"] { + t.Fatalf("Test failed with Content-Disposition mistmatch, expected %s, got %s", headers["content-disposition"], attrs.ContentDisposition) + } + if attrs.ContentEncoding != headers["content-encoding"] { + t.Fatalf("Test failed with Content-Encoding mistmatch, expected %s, got %s", headers["content-encoding"], attrs.ContentEncoding) + } + if attrs.ContentLanguage != headers["Content-Language"] { + t.Fatalf("Test failed with Content-Language mistmatch, expected %s, got %s", headers["Content-Language"], attrs.ContentLanguage) + } + if attrs.ContentType != headers["content-type"] { + t.Fatalf("Test failed with Content-Type mistmatch, expected %s, got %s", headers["content-type"], attrs.ContentType) + } +} + +func TestGCSAttrsToObjectInfo(t *testing.T) { + metadata := map[string]string{ + "x-goog-meta-Hdr": "value", + "x-goog-meta-x_amz_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=", + "x-goog-meta-x-amz-matdesc": "{}", + "x-goog-meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==", + } + expectedMeta := map[string]string{ + "X-Amz-Meta-Hdr": "value", + "X-Amz-Meta-X_amz_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=", + "X-Amz-Meta-X-Amz-Matdesc": "{}", + "X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==", + "Cache-Control": "max-age: 3600", + "Content-Disposition": "dummy", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Type": "application/javascript", + } + + attrs := storage.ObjectAttrs{ + Name: "test-obj", + Bucket: "test-bucket", + Updated: time.Now(), + Size: 123, + CRC32C: 45312398, + CacheControl: "max-age: 3600", + ContentDisposition: "dummy", + ContentEncoding: "gzip", + ContentLanguage: "en", + ContentType: "application/javascript", + Metadata: metadata, + } + expectedETag := minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C)) + + objInfo := fromGCSAttrsToObjectInfo(&attrs) + if !reflect.DeepEqual(objInfo.UserDefined, expectedMeta) { + t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, objInfo.UserDefined) + } + + if objInfo.Name != attrs.Name { + t.Fatalf("Test failed with Name mistmatch, expected %s, got %s", attrs.Name, objInfo.Name) + } + if objInfo.Bucket != attrs.Bucket { + t.Fatalf("Test failed with Bucket mistmatch, expected %s, got %s", attrs.Bucket, objInfo.Bucket) + } + if objInfo.ModTime != attrs.Updated { + t.Fatalf("Test failed with ModTime mistmatch, expected %s, got %s", attrs.Updated, objInfo.ModTime) + } + if objInfo.Size != attrs.Size { + t.Fatalf("Test failed with Size mistmatch, expected %d, got %d", attrs.Size, objInfo.Size) + } + if objInfo.ETag != expectedETag { + t.Fatalf("Test failed with ETag mistmatch, expected %s, got %s", expectedETag, objInfo.ETag) + } +}