oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

gcs: Translate S3 user-defined metadata prefix to/from GCS custom metadata prefix (#6270)

Browse Source
master
wd256 7 years ago committed by kannappanr
parent 64f2c61813
commit ff29aed05d
2 changed files with 132 additions and 19 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 47
      cmd/gateway/gcs/gateway-gcs.go
  2. 104
      cmd/gateway/gcs/gateway-gcs_test.go

47
cmd/gateway/gcs/gateway-gcs.go
Unescape View File

@ -23,8 +23,8 @@ import (
"fmt"
"io"
"io/ioutil"
"math"
"net/http"
"os"
"regexp"
"strings"
@ -768,22 +768,27 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
// Refer https://cloud.google.com/storage/docs/hashes-etags. Use CRC32C for ETag
metadata := make(map[string]string)
for k, v := range attrs.Metadata {
k = http.CanonicalHeaderKey(k)
// Translate the GCS custom metadata prefix
if strings.HasPrefix(k, "X-Goog-Meta-") {
k = strings.Replace(k, "X-Goog-Meta-", "X-Amz-Meta-", 1)
}
metadata[k] = v
}
if attrs.ContentType != "" {
metadata["content-type"] = attrs.ContentType
metadata["Content-Type"] = attrs.ContentType
}
if attrs.ContentEncoding != "" {
metadata["content-encoding"] = attrs.ContentEncoding
metadata["Content-Encoding"] = attrs.ContentEncoding
}
if attrs.CacheControl != "" {
metadata["cache-control"] = attrs.CacheControl
metadata["Cache-Control"] = attrs.CacheControl
}
if attrs.ContentDisposition != "" {
metadata["content-disposition"] = attrs.ContentDisposition
metadata["Content-Disposition"] = attrs.ContentDisposition
}
if attrs.ContentLanguage != "" {
metadata["content-language"] = attrs.ContentLanguage
metadata["Content-Language"] = attrs.ContentLanguage
}
return minio.ObjectInfo{
Name: attrs.Name,
@ -799,21 +804,25 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
// applyMetadataToGCSAttrs applies metadata to a GCS ObjectAttrs instance
func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAttrs) {
attrs.ContentType = metadata["content-type"]
attrs.ContentEncoding = metadata["content-encoding"]
attrs.CacheControl = metadata["cache-control"]
attrs.ContentDisposition = metadata["content-disposition"]
attrs.ContentLanguage = metadata["content-language"]
attrs.Metadata = make(map[string]string)
for k, v := range metadata {
attrs.Metadata[k] = v
}
// Filter metadata which is stored as a unique attribute
for _, key := range []string{
"content-type", "content-encoding", "cache-control", "content-disposition", "content-language",
} {
delete(attrs.Metadata, key)
k = http.CanonicalHeaderKey(k)
switch {
case strings.HasPrefix(k, "X-Amz-Meta-"):
// Translate the S3 user-defined metadata prefix
k = strings.Replace(k, "X-Amz-Meta-", "x-goog-meta-", 1)
attrs.Metadata[k] = v
case k == "Content-Type":
attrs.ContentType = v
case k == "Content-Encoding":
attrs.ContentEncoding = v
case k == "Cache-Control":
attrs.CacheControl = v
case k == "Content-Disposition":
attrs.ContentDisposition = v
case k == "Content-Language":
attrs.ContentLanguage = v
}
}
}

104
cmd/gateway/gcs/gateway-gcs_test.go
Unescape View File

@ -23,7 +23,9 @@ import (
"path"
"reflect"
"testing"
"time"
"cloud.google.com/go/storage"
"google.golang.org/api/googleapi"
miniogo "github.com/minio/minio-go"
@ -393,3 +395,105 @@ func TestGCSToObjectError(t *testing.T) {
}
}
}
func TestS3MetaToGCSAttributes(t *testing.T) {
headers := map[string]string{
"accept-encoding": "gzip",
"content-encoding": "gzip",
"cache-control": "age: 3600",
"content-disposition": "dummy",
"content-type": "application/javascript",
"Content-Language": "en",
"X-Amz-Meta-Hdr": "value",
"X-Amz-Meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
"X-Amz-Meta-X-Amz-Matdesc": "{}",
"X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
}
// Only X-Amz-Meta- prefixed entries will be returned in
// Metadata (without the prefix!)
expectedHeaders := map[string]string{
"x-goog-meta-Hdr": "value",
"x-goog-meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
"x-goog-meta-X-Amz-Matdesc": "{}",
"x-goog-meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
}
attrs := storage.ObjectAttrs{}
applyMetadataToGCSAttrs(headers, &attrs)
if !reflect.DeepEqual(attrs.Metadata, expectedHeaders) {
t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, attrs.Metadata)
}
if attrs.CacheControl != headers["cache-control"] {
t.Fatalf("Test failed with Cache-Control mistmatch, expected %s, got %s", headers["cache-control"], attrs.CacheControl)
}
if attrs.ContentDisposition != headers["content-disposition"] {
t.Fatalf("Test failed with Content-Disposition mistmatch, expected %s, got %s", headers["content-disposition"], attrs.ContentDisposition)
}
if attrs.ContentEncoding != headers["content-encoding"] {
t.Fatalf("Test failed with Content-Encoding mistmatch, expected %s, got %s", headers["content-encoding"], attrs.ContentEncoding)
}
if attrs.ContentLanguage != headers["Content-Language"] {
t.Fatalf("Test failed with Content-Language mistmatch, expected %s, got %s", headers["Content-Language"], attrs.ContentLanguage)
}
if attrs.ContentType != headers["content-type"] {
t.Fatalf("Test failed with Content-Type mistmatch, expected %s, got %s", headers["content-type"], attrs.ContentType)
}
}
func TestGCSAttrsToObjectInfo(t *testing.T) {
metadata := map[string]string{
"x-goog-meta-Hdr": "value",
"x-goog-meta-x_amz_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
"x-goog-meta-x-amz-matdesc": "{}",
"x-goog-meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
}
expectedMeta := map[string]string{
"X-Amz-Meta-Hdr": "value",
"X-Amz-Meta-X_amz_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
"X-Amz-Meta-X-Amz-Matdesc": "{}",
"X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
"Cache-Control": "max-age: 3600",
"Content-Disposition": "dummy",
"Content-Encoding": "gzip",
"Content-Language": "en",
"Content-Type": "application/javascript",
}
attrs := storage.ObjectAttrs{
Name: "test-obj",
Bucket: "test-bucket",
Updated: time.Now(),
Size: 123,
CRC32C: 45312398,
CacheControl: "max-age: 3600",
ContentDisposition: "dummy",
ContentEncoding: "gzip",
ContentLanguage: "en",
ContentType: "application/javascript",
Metadata: metadata,
}
expectedETag := minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C))
objInfo := fromGCSAttrsToObjectInfo(&attrs)
if !reflect.DeepEqual(objInfo.UserDefined, expectedMeta) {
t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, objInfo.UserDefined)
}
if objInfo.Name != attrs.Name {
t.Fatalf("Test failed with Name mistmatch, expected %s, got %s", attrs.Name, objInfo.Name)
}
if objInfo.Bucket != attrs.Bucket {
t.Fatalf("Test failed with Bucket mistmatch, expected %s, got %s", attrs.Bucket, objInfo.Bucket)
}
if objInfo.ModTime != attrs.Updated {
t.Fatalf("Test failed with ModTime mistmatch, expected %s, got %s", attrs.Updated, objInfo.ModTime)
}
if objInfo.Size != attrs.Size {
t.Fatalf("Test failed with Size mistmatch, expected %d, got %d", attrs.Size, objInfo.Size)
}
if objInfo.ETag != expectedETag {
t.Fatalf("Test failed with ETag mistmatch, expected %s, got %s", expectedETag, objInfo.ETag)
}
}

Write Preview
Loading…
Cancel
Save