oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix: CopyObject behavior on expanded zones (#9729)

Browse Source 
CopyObject was not correctly figuring out the correct
destination object location and would end up creating
duplicate objects on two different zones, reproduced
by doing encryption based key rotation.
master
Harshavardhana 5 years ago committed by GitHub
parent b2db8123ec
commit 41688a936b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 79 additions and 73 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      cmd/api-headers.go
  2. 2
      cmd/api-response.go
  3. 4
      cmd/disk-cache.go
  4. 7
      cmd/generic-handlers.go
  5. 2
      cmd/object-handlers.go
  6. 11
      cmd/xl-sets.go
  7. 11
      cmd/xl-v1-object.go
  8. 31
      cmd/xl-zones.go

2
cmd/api-headers.go
Unescape View File

@ -112,7 +112,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
// Set all other user defined metadata. // Set all other user defined metadata.
for k, v := range objInfo.UserDefined { for k, v := range objInfo.UserDefined {
if strings.HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
// Do not need to send any internal metadata // Do not need to send any internal metadata
// values to client. // values to client.
continue continue

2
cmd/api-response.go
Unescape View File

@ -541,7 +541,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
if metadata { if metadata {
content.UserMetadata = make(StringMap) content.UserMetadata = make(StringMap)
for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) { for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
if strings.HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
// Do not need to send any internal metadata // Do not need to send any internal metadata
// values to client. // values to client.
continue continue

4
cmd/disk-cache.go
Unescape View File

@ -99,14 +99,14 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
bkMeta := make(map[string]string) bkMeta := make(map[string]string)
cacheMeta := make(map[string]string) cacheMeta := make(map[string]string)
for k, v := range bkObjectInfo.UserDefined { for k, v := range bkObjectInfo.UserDefined {
if strings.HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
// Do not need to send any internal metadata // Do not need to send any internal metadata
continue continue
} }
bkMeta[http.CanonicalHeaderKey(k)] = v bkMeta[http.CanonicalHeaderKey(k)] = v
} }
for k, v := range cacheObjInfo.UserDefined { for k, v := range cacheObjInfo.UserDefined {
if strings.HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
// Do not need to send any internal metadata // Do not need to send any internal metadata
continue continue
} }

7
cmd/generic-handlers.go
Unescape View File

@ -116,7 +116,10 @@ func isHTTPHeaderSizeTooLarge(header http.Header) bool {
// ReservedMetadataPrefix is the prefix of a metadata key which // ReservedMetadataPrefix is the prefix of a metadata key which
// is reserved and for internal use only. // is reserved and for internal use only.
const ReservedMetadataPrefix = "X-Minio-Internal-" const (
ReservedMetadataPrefix = "X-Minio-Internal-"
ReservedMetadataPrefixLower = "x-minio-internal-"
)
type reservedMetadataHandler struct { type reservedMetadataHandler struct {
http.Handler http.Handler
@ -141,7 +144,7 @@ func (h reservedMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
// and must not set by clients // and must not set by clients
func containsReservedMetadata(header http.Header) bool { func containsReservedMetadata(header http.Header) bool {
for key := range header { for key := range header {
if HasPrefix(key, ReservedMetadataPrefix) { if strings.HasPrefix(strings.ToLower(key), ReservedMetadataPrefixLower) {
return true return true
} }
} }

2
cmd/object-handlers.go
Unescape View File

@ -995,7 +995,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
} }
for k, v := range srcInfo.UserDefined { for k, v := range srcInfo.UserDefined {
if HasPrefix(k, ReservedMetadataPrefix) { if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
encMetadata[k] = v encMetadata[k] = v
} }
} }

11
cmd/xl-sets.go
Unescape View File

@ -773,18 +773,17 @@ func (s *xlSets) DeleteObjects(ctx context.Context, bucket string, objects []str
} }
// CopyObject - copies objects from one hashedSet to another hashedSet, on server side. // CopyObject - copies objects from one hashedSet to another hashedSet, on server side.
func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) { func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
srcSet := s.getHashedSet(srcObject) srcSet := s.getHashedSet(srcObject)
destSet := s.getHashedSet(destObject) dstSet := s.getHashedSet(dstObject)
// Check if this request is only metadata update. // Check if this request is only metadata update.
cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(destBucket, destObject)) if srcSet == dstSet && srcInfo.metadataOnly {
if cpSrcDstSame && srcInfo.metadataOnly { return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
return srcSet.CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
} }
putOpts := ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined} putOpts := ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined}
return destSet.putObject(ctx, destBucket, destObject, srcInfo.PutObjReader, putOpts) return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
} }
// FileInfoCh - file info channel // FileInfoCh - file info channel

11
cmd/xl-v1-object.go
Unescape View File

@ -65,10 +65,11 @@ func (xl xlObjects) putObjectDir(ctx context.Context, bucket, object string, wri
// if source object and destination object are same we only // if source object and destination object are same we only
// update metadata. // update metadata.
func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) { func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) // This call shouldn't be used for anything other than metadata updates.
if !srcInfo.metadataOnly {
return oi, NotImplemented{}
}
// Check if this request is only metadata update.
if cpSrcDstSame {
defer ObjectPathUpdated(path.Join(dstBucket, dstObject)) defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
// Read metadata associated with the object from all disks. // Read metadata associated with the object from all disks.
@ -119,10 +120,6 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc
} }
return xlMeta.ToObjectInfo(srcBucket, srcObject), nil return xlMeta.ToObjectInfo(srcBucket, srcObject), nil
}
putOpts := ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined}
return xl.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
} }
// GetObjectNInfo - returns object info and an object // GetObjectNInfo - returns object info and an object

31
cmd/xl-zones.go
Unescape View File

@ -533,11 +533,11 @@ func (z *xlZones) DeleteObjects(ctx context.Context, bucket string, objects []st
return derrs, nil return derrs, nil
} }
func (z *xlZones) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) { func (z *xlZones) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
// Check if this request is only metadata update. // Check if this request is only metadata update.
cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(destBucket, destObject)) cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
if !cpSrcDstSame { if !cpSrcDstSame {
lk := z.NewNSLock(ctx, destBucket, destObject) lk := z.NewNSLock(ctx, dstBucket, dstObject)
if err := lk.GetLock(globalObjectTimeout); err != nil { if err := lk.GetLock(globalObjectTimeout); err != nil {
return objInfo, err return objInfo, err
} }
@ -545,24 +545,30 @@ func (z *xlZones) CopyObject(ctx context.Context, srcBucket, srcObject, destBuck
} }
if z.SingleZone() { if z.SingleZone() {
return z.zones[0].CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts) return z.zones[0].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
} }
if cpSrcDstSame && srcInfo.metadataOnly {
for _, zone := range z.zones { zoneIndex := -1
objInfo, err = zone.CopyObject(ctx, srcBucket, srcObject, destBucket, for i, zone := range z.zones {
destObject, srcInfo, srcOpts, dstOpts) objInfo, err := zone.GetObjectInfo(ctx, dstBucket, dstObject, srcOpts)
if err != nil { if err != nil {
if isErrObjectNotFound(err) { if isErrObjectNotFound(err) {
continue continue
} }
return objInfo, err return objInfo, err
} }
return objInfo, nil zoneIndex = i
break
} }
return objInfo, ObjectNotFound{Bucket: srcBucket, Object: srcObject}
putOpts := ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined}
if zoneIndex >= 0 {
if cpSrcDstSame && srcInfo.metadataOnly {
return z.zones[zoneIndex].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
} }
return z.zones[z.getAvailableZoneIdx(ctx)].CopyObject(ctx, srcBucket, srcObject, return z.zones[zoneIndex].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
destBucket, destObject, srcInfo, srcOpts, dstOpts) }
return z.zones[z.getAvailableZoneIdx(ctx)].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
} }
func (z *xlZones) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) { func (z *xlZones) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
@ -1050,6 +1056,7 @@ func (z *xlZones) PutObjectPart(ctx context.Context, bucket, object, uploadID st
if z.SingleZone() { if z.SingleZone() {
return z.zones[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts) return z.zones[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
} }
for _, zone := range z.zones { for _, zone := range z.zones {
_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts) _, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
if err == nil { if err == nil {

Write Preview
Loading…
Cancel
Save