From 810dcbf34b94fd4f2da0fa3fed90d85dce4043fb Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 17 Aug 2016 13:26:08 -0700 Subject: [PATCH] fs: ListObjects should populate ETag properly if fs.json is available. (#2480) Fixes #2470 --- fs-v1-multipart.go | 1 + fs-v1.go | 33 +++++++++++++++++++++------------ web-handlers.go | 3 +-- xl-v1-multipart.go | 3 +-- xl-v1-object.go | 4 ++-- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/fs-v1-multipart.go b/fs-v1-multipart.go index 998a957e0..6f41452c8 100644 --- a/fs-v1-multipart.go +++ b/fs-v1-multipart.go @@ -219,6 +219,7 @@ func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMark func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[string]string) (uploadID string, err error) { // Initialize `fs.json` values. fsMeta := newFSMetaV1() + // Save additional metadata only if extended headers such as "X-Amz-Meta-" are set. if hasExtendedHeader(meta) { fsMeta.Meta = meta diff --git a/fs-v1.go b/fs-v1.go index ea45329e4..a798df063 100644 --- a/fs-v1.go +++ b/fs-v1.go @@ -23,7 +23,6 @@ import ( "io" "os" "path" - "path/filepath" "sort" "strings" @@ -336,7 +335,7 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) { // Guess content-type from the extension if possible. if fsMeta.Meta["content-type"] == "" { - if objectExt := filepath.Ext(object); objectExt != "" { + if objectExt := path.Ext(object); objectExt != "" { if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok { fsMeta.Meta["content-type"] = content.ContentType } @@ -369,6 +368,10 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. Object: object, } } + // No metadata is set, allocate a new one. + if metadata == nil { + metadata = make(map[string]string) + } uniqueID := getUUID() @@ -419,10 +422,9 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. } newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil)) - // md5Hex representation. - var md5Hex string - if len(metadata) != 0 { - md5Hex = metadata["md5Sum"] + // Update the md5sum if not set with the newly calculated one. + if len(metadata["md5Sum"]) == 0 { + metadata["md5Sum"] = newMD5Hex } // Validate if payload is valid. @@ -435,6 +437,8 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. } } + // md5Hex representation. + md5Hex := metadata["md5Sum"] if md5Hex != "" { if newMD5Hex != md5Hex { // MD5 mismatch, delete the temporary object. @@ -505,7 +509,9 @@ func isBucketExist(storage StorageAPI, bucketName string) bool { return true } -func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { +// ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool +// state for future re-entrant list requests. +func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { // Convert entry to FileInfo entryToFileInfo := func(entry string) (fileInfo FileInfo, err error) { if strings.HasSuffix(entry, slashSeparator) { @@ -517,8 +523,16 @@ func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey if fileInfo, err = fs.storage.StatFile(bucket, entry); err != nil { return } + fsMeta, mErr := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, entry)) + if mErr != nil && mErr != errFileNotFound { + return FileInfo{}, mErr + } + if len(fsMeta.Meta) == 0 { + fsMeta.Meta = make(map[string]string) + } // Object name needs to be full path. fileInfo.Name = entry + fileInfo.MD5Sum = fsMeta.Meta["md5Sum"] return } @@ -640,11 +654,6 @@ func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey return result, nil } -// ListObjects - list all objects. -func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { - return fs.listObjects(bucket, prefix, marker, delimiter, maxKeys) -} - // HealObject - no-op for fs. Valid only for XL. func (fs fsObjects) HealObject(bucket, object string) error { return NotImplemented{} diff --git a/web-handlers.go b/web-handlers.go index 06297858d..cda31496a 100644 --- a/web-handlers.go +++ b/web-handlers.go @@ -21,7 +21,6 @@ import ( "net/http" "os" "path" - "path/filepath" "runtime" "strconv" "time" @@ -430,7 +429,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { return } // Add content disposition. - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base(object))) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(object))) objInfo, err := web.ObjectAPI.GetObjectInfo(bucket, object) if err != nil { diff --git a/xl-v1-multipart.go b/xl-v1-multipart.go index 8fbdc92a4..58dd02783 100644 --- a/xl-v1-multipart.go +++ b/xl-v1-multipart.go @@ -23,7 +23,6 @@ import ( "io" "io/ioutil" "path" - "path/filepath" "strings" "time" @@ -259,7 +258,7 @@ func (xl xlObjects) newMultipartUpload(bucket string, object string, meta map[st // If not set default to "application/octet-stream" if meta["content-type"] == "" { contentType := "application/octet-stream" - if objectExt := filepath.Ext(object); objectExt != "" { + if objectExt := path.Ext(object); objectExt != "" { content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))] if ok { contentType = content.ContentType diff --git a/xl-v1-object.go b/xl-v1-object.go index dbffda23c..9e0775485 100644 --- a/xl-v1-object.go +++ b/xl-v1-object.go @@ -21,7 +21,6 @@ import ( "encoding/hex" "io" "path" - "path/filepath" "strings" "sync" "time" @@ -498,6 +497,7 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. if metadata == nil { metadata = make(map[string]string) } + uniqueID := getUUID() tempErasureObj := path.Join(tmpMetaPrefix, uniqueID, "part.1") minioMetaTmpBucket := path.Join(minioMetaBucket, tmpMetaPrefix) @@ -581,7 +581,7 @@ func (xl xlObjects) PutObject(bucket string, object string, size int64, data io. // Guess content-type from the extension if possible. if metadata["content-type"] == "" { - if objectExt := filepath.Ext(object); objectExt != "" { + if objectExt := path.Ext(object); objectExt != "" { if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok { metadata["content-type"] = content.ContentType }