From 79bab6b5610ab84f5dc51509ef38d6cb908ca63b Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 24 Jul 2016 18:08:15 -0700 Subject: [PATCH] XL: Operations on `uploads.json` should cater for disk being unavailable. (#2277) --- xl-v1-bucket.go | 5 ++++ xl-v1-metadata.go | 1 + xl-v1-multipart-common.go | 56 ++++++++++++++++++++++++++------------- xl-v1-multipart.go | 32 ++-------------------- 4 files changed, 45 insertions(+), 49 deletions(-) diff --git a/xl-v1-bucket.go b/xl-v1-bucket.go index ad70f6d97..49ffdfd68 100644 --- a/xl-v1-bucket.go +++ b/xl-v1-bucket.go @@ -216,6 +216,11 @@ func (xl xlObjects) listBuckets() (bucketsInfo []BucketInfo, err error) { Created: volInfo.Created, }) } + // For buckets info empty, loop once again to check + // if we have, can happen if disks are down. + if len(bucketsInfo) == 0 { + continue + } return bucketsInfo, nil } // Ignore any disks not found. diff --git a/xl-v1-metadata.go b/xl-v1-metadata.go index d5e013975..49a16f2b4 100644 --- a/xl-v1-metadata.go +++ b/xl-v1-metadata.go @@ -206,6 +206,7 @@ var objMetadataOpIgnoredErrs = []error{ errFaultyDisk, errVolumeNotFound, errFileAccessDenied, + errFileNotFound, } // readXLMetadata - returns the object metadata `xl.json` content from diff --git a/xl-v1-multipart-common.go b/xl-v1-multipart-common.go index 2bb32b1f0..2dca561ca 100644 --- a/xl-v1-multipart-common.go +++ b/xl-v1-multipart-common.go @@ -87,6 +87,24 @@ func (xl xlObjects) updateUploadsJSON(bucket, object string, uploadsJSON uploads return nil } +// Reads uploads.json from any of the load balanced disks. +func (xl xlObjects) readUploadsJSON(bucket, object string) (uploadsJSON uploadsV1, err error) { + for _, disk := range xl.getLoadBalancedDisks() { + if disk == nil { + continue + } + uploadsJSON, err = readUploadsJSON(bucket, object, disk) + if err == nil { + return uploadsJSON, nil + } + if isErrIgnored(err, objMetadataOpIgnoredErrs) { + continue + } + break + } + return uploadsV1{}, err +} + // writeUploadJSON - create `uploads.json` or update it with new uploadID. func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated time.Time) (err error) { uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile) @@ -96,16 +114,9 @@ func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated t var errs = make([]error, len(xl.storageDisks)) var wg = &sync.WaitGroup{} - var uploadsJSON uploadsV1 - for _, disk := range xl.storageDisks { - if disk == nil { - continue - } - uploadsJSON, err = readUploadsJSON(bucket, object, disk) - break - } + // Reads `uploads.json` and returns error. + uploadsJSON, err := xl.readUploadsJSON(bucket, object) if err != nil { - // For any other errors. if err != errFileNotFound { return err } @@ -151,12 +162,8 @@ func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated t // Wait here for all the writes to finish. wg.Wait() - // Count all the errors and validate if we have write quorum. + // Do we have write quorum?. if !isDiskQuorum(errs, xl.writeQuorum) { - // Do we have readQuorum?. - if isDiskQuorum(errs, xl.readQuorum) { - return nil - } // Rename `uploads.json` left over back to tmp location. for index, disk := range xl.storageDisks { if disk == nil { @@ -175,7 +182,14 @@ func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated t wg.Wait() return errXLWriteQuorum } - return nil + + // Ignored errors list. + ignoredErrs := []error{ + errDiskNotFound, + errFaultyDisk, + errDiskAccessDenied, + } + return reduceErrs(errs, ignoredErrs) } // Returns if the prefix is a multipart upload. @@ -234,10 +248,13 @@ func (xl xlObjects) statPart(bucket, object, uploadID, partName string) (fileInf if err == nil { return fileInfo, nil } - // For any reason disk was deleted or goes offline, continue + + // For any reason disk was deleted or goes offline we continue to next disk. if isErrIgnored(err, objMetadataOpIgnoredErrs) { continue } + + // Catastrophic error, we return. break } return FileInfo{}, err @@ -283,10 +300,11 @@ func commitXLMetadata(disks []StorageAPI, srcPrefix, dstPrefix string, quorum in return errXLWriteQuorum } - return reduceErrs(mErrs, []error{ + // List of ignored errors. + ignoredErrs := []error{ errDiskNotFound, errDiskAccessDenied, errFaultyDisk, - errVolumeNotFound, - }) + } + return reduceErrs(mErrs, ignoredErrs) } diff --git a/xl-v1-multipart.go b/xl-v1-multipart.go index 325ce3543..eb6cc9d4c 100644 --- a/xl-v1-multipart.go +++ b/xl-v1-multipart.go @@ -755,21 +755,7 @@ func (xl xlObjects) CompleteMultipartUpload(bucket string, object string, upload // Validate if there are other incomplete upload-id's present for // the object, if yes do not attempt to delete 'uploads.json'. - var disk StorageAPI - var uploadsJSON uploadsV1 - for _, disk = range xl.getLoadBalancedDisks() { - if disk == nil { - continue - } - uploadsJSON, err = readUploadsJSON(bucket, object, disk) - if err == nil { - break - } - if isErrIgnored(err, objMetadataOpIgnoredErrs) { - continue - } - break - } + uploadsJSON, err := xl.readUploadsJSON(bucket, object) if err != nil { return "", toObjectErr(err, minioMetaBucket, object) } @@ -809,21 +795,7 @@ func (xl xlObjects) abortMultipartUpload(bucket, object, uploadID string) (err e defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object)) // Validate if there are other incomplete upload-id's present for // the object, if yes do not attempt to delete 'uploads.json'. - var disk StorageAPI - var uploadsJSON uploadsV1 - for _, disk = range xl.getLoadBalancedDisks() { - if disk == nil { - continue - } - uploadsJSON, err = readUploadsJSON(bucket, object, disk) - if err == nil { - break - } - if isErrIgnored(err, objMetadataOpIgnoredErrs) { - continue - } - break - } + uploadsJSON, err := xl.readUploadsJSON(bucket, object) if err != nil { return toObjectErr(err, bucket, object) }