gcs: Translate S3 user-defined metadata prefix to/from GCS custom metadata prefix (#6270)

cmd/gateway/gcs/gateway-gcs.go

@@ -23,8 +23,8 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-
 	"math"
+	"net/http"
 	"os"
 	"regexp"
 	"strings"
@@ -768,22 +768,27 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 	// Refer https://cloud.google.com/storage/docs/hashes-etags. Use CRC32C for ETag
 	metadata := make(map[string]string)
 	for k, v := range attrs.Metadata {
+		k = http.CanonicalHeaderKey(k)
+		// Translate the GCS custom metadata prefix
+		if strings.HasPrefix(k, "X-Goog-Meta-") {
+			k = strings.Replace(k, "X-Goog-Meta-", "X-Amz-Meta-", 1)
+		}
 		metadata[k] = v
 	}
 	if attrs.ContentType != "" {
-		metadata["content-type"] = attrs.ContentType
+		metadata["Content-Type"] = attrs.ContentType
 	}
 	if attrs.ContentEncoding != "" {
-		metadata["content-encoding"] = attrs.ContentEncoding
+		metadata["Content-Encoding"] = attrs.ContentEncoding
 	}
 	if attrs.CacheControl != "" {
-		metadata["cache-control"] = attrs.CacheControl
+		metadata["Cache-Control"] = attrs.CacheControl
 	}
 	if attrs.ContentDisposition != "" {
-		metadata["content-disposition"] = attrs.ContentDisposition
+		metadata["Content-Disposition"] = attrs.ContentDisposition
 	}
 	if attrs.ContentLanguage != "" {
-		metadata["content-language"] = attrs.ContentLanguage
+		metadata["Content-Language"] = attrs.ContentLanguage
 	}
 	return minio.ObjectInfo{
 		Name:            attrs.Name,
@@ -799,21 +804,25 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 
 // applyMetadataToGCSAttrs applies metadata to a GCS ObjectAttrs instance
 func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAttrs) {
-	attrs.ContentType = metadata["content-type"]
-	attrs.ContentEncoding = metadata["content-encoding"]
-	attrs.CacheControl = metadata["cache-control"]
-	attrs.ContentDisposition = metadata["content-disposition"]
-	attrs.ContentLanguage = metadata["content-language"]
-
 	attrs.Metadata = make(map[string]string)
 	for k, v := range metadata {
-		attrs.Metadata[k] = v
+		k = http.CanonicalHeaderKey(k)
-	}
+		switch {
-	// Filter metadata which is stored as a unique attribute
+		case strings.HasPrefix(k, "X-Amz-Meta-"):
-	for _, key := range []string{
+			// Translate the S3 user-defined metadata prefix
-		"content-type", "content-encoding", "cache-control", "content-disposition", "content-language",
+			k = strings.Replace(k, "X-Amz-Meta-", "x-goog-meta-", 1)
-	} {
+			attrs.Metadata[k] = v
-		delete(attrs.Metadata, key)
+		case k == "Content-Type":
+			attrs.ContentType = v
+		case k == "Content-Encoding":
+			attrs.ContentEncoding = v
+		case k == "Cache-Control":
+			attrs.CacheControl = v
+		case k == "Content-Disposition":
+			attrs.ContentDisposition = v
+		case k == "Content-Language":
+			attrs.ContentLanguage = v
+		}
 	}
 }
 

cmd/gateway/gcs/gateway-gcs_test.go

@@ -23,7 +23,9 @@ import (
 	"path"
 	"reflect"
 	"testing"
+	"time"
 
+	"cloud.google.com/go/storage"
 	"google.golang.org/api/googleapi"
 
 	miniogo "github.com/minio/minio-go"
@@ -393,3 +395,105 @@ func TestGCSToObjectError(t *testing.T) {
 		}
 	}
 }
+
+func TestS3MetaToGCSAttributes(t *testing.T) {
+	headers := map[string]string{
+		"accept-encoding":          "gzip",
+		"content-encoding":         "gzip",
+		"cache-control":            "age: 3600",
+		"content-disposition":      "dummy",
+		"content-type":             "application/javascript",
+		"Content-Language":         "en",
+		"X-Amz-Meta-Hdr":           "value",
+		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
+		"X-Amz-Meta-X-Amz-Matdesc": "{}",
+		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
+	}
+	// Only X-Amz-Meta- prefixed entries will be returned in
+	// Metadata (without the prefix!)
+	expectedHeaders := map[string]string{
+		"x-goog-meta-Hdr":           "value",
+		"x-goog-meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
+		"x-goog-meta-X-Amz-Matdesc": "{}",
+		"x-goog-meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
+	}
+
+	attrs := storage.ObjectAttrs{}
+	applyMetadataToGCSAttrs(headers, &attrs)
+
+	if !reflect.DeepEqual(attrs.Metadata, expectedHeaders) {
+		t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, attrs.Metadata)
+	}
+
+	if attrs.CacheControl != headers["cache-control"] {
+		t.Fatalf("Test failed with Cache-Control mistmatch, expected %s, got %s", headers["cache-control"], attrs.CacheControl)
+	}
+	if attrs.ContentDisposition != headers["content-disposition"] {
+		t.Fatalf("Test failed with Content-Disposition mistmatch, expected %s, got %s", headers["content-disposition"], attrs.ContentDisposition)
+	}
+	if attrs.ContentEncoding != headers["content-encoding"] {
+		t.Fatalf("Test failed with Content-Encoding mistmatch, expected %s, got %s", headers["content-encoding"], attrs.ContentEncoding)
+	}
+	if attrs.ContentLanguage != headers["Content-Language"] {
+		t.Fatalf("Test failed with Content-Language mistmatch, expected %s, got %s", headers["Content-Language"], attrs.ContentLanguage)
+	}
+	if attrs.ContentType != headers["content-type"] {
+		t.Fatalf("Test failed with Content-Type mistmatch, expected %s, got %s", headers["content-type"], attrs.ContentType)
+	}
+}
+
+func TestGCSAttrsToObjectInfo(t *testing.T) {
+	metadata := map[string]string{
+		"x-goog-meta-Hdr":           "value",
+		"x-goog-meta-x_amz_key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
+		"x-goog-meta-x-amz-matdesc": "{}",
+		"x-goog-meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
+	}
+	expectedMeta := map[string]string{
+		"X-Amz-Meta-Hdr":           "value",
+		"X-Amz-Meta-X_amz_key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
+		"X-Amz-Meta-X-Amz-Matdesc": "{}",
+		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
+		"Cache-Control":            "max-age: 3600",
+		"Content-Disposition":      "dummy",
+		"Content-Encoding":         "gzip",
+		"Content-Language":         "en",
+		"Content-Type":             "application/javascript",
+	}
+
+	attrs := storage.ObjectAttrs{
+		Name:               "test-obj",
+		Bucket:             "test-bucket",
+		Updated:            time.Now(),
+		Size:               123,
+		CRC32C:             45312398,
+		CacheControl:       "max-age: 3600",
+		ContentDisposition: "dummy",
+		ContentEncoding:    "gzip",
+		ContentLanguage:    "en",
+		ContentType:        "application/javascript",
+		Metadata:           metadata,
+	}
+	expectedETag := minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C))
+
+	objInfo := fromGCSAttrsToObjectInfo(&attrs)
+	if !reflect.DeepEqual(objInfo.UserDefined, expectedMeta) {
+		t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, objInfo.UserDefined)
+	}
+
+	if objInfo.Name != attrs.Name {
+		t.Fatalf("Test failed with Name mistmatch, expected %s, got %s", attrs.Name, objInfo.Name)
+	}
+	if objInfo.Bucket != attrs.Bucket {
+		t.Fatalf("Test failed with Bucket mistmatch, expected %s, got %s", attrs.Bucket, objInfo.Bucket)
+	}
+	if objInfo.ModTime != attrs.Updated {
+		t.Fatalf("Test failed with ModTime mistmatch, expected %s, got %s", attrs.Updated, objInfo.ModTime)
+	}
+	if objInfo.Size != attrs.Size {
+		t.Fatalf("Test failed with Size mistmatch, expected %d, got %d", attrs.Size, objInfo.Size)
+	}
+	if objInfo.ETag != expectedETag {
+		t.Fatalf("Test failed with ETag mistmatch, expected %s, got %s", expectedETag, objInfo.ETag)
+	}
+}