Continue healing other objects even if objects without quorum exist (#5851)

fixes #5815
master
Krishna Srinivas 7 years ago committed by Dee Koder
parent 6831177394
commit 9aace6d36d
  1. 15
      cmd/xl-sets.go
  2. 25
      cmd/xl-v1-healing.go
  3. 4
      cmd/xl-v1-healing_test.go
  4. 3
      cmd/xl-v1-metadata.go

@ -1379,19 +1379,8 @@ func (s *xlSets) listObjectsHeal(ctx context.Context, bucket, prefix, marker, de
return loi, toObjectErr(walkResult.err, bucket, prefix) return loi, toObjectErr(walkResult.err, bucket, prefix)
} }
var objInfo ObjectInfo var objInfo ObjectInfo
var err error objInfo.Bucket = bucket
if hasSuffix(walkResult.entry, slashSeparator) { objInfo.Name = walkResult.entry
objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfoDir(ctx, bucket, walkResult.entry)
} else {
objInfo, err = s.getHashedSet(walkResult.entry).getObjectInfo(ctx, bucket, walkResult.entry)
}
if err != nil {
// Ignore errFileNotFound
if err == errFileNotFound {
continue
}
return loi, toObjectErr(err, bucket, prefix)
}
nextMarker = objInfo.Name nextMarker = objInfo.Name
objInfos = append(objInfos, objInfo) objInfos = append(objInfos, objInfo)
i++ i++

@ -284,12 +284,21 @@ func healObject(ctx context.Context, storageDisks []StorageAPI, bucket string, o
partsMetadata, errs := readAllXLMetadata(ctx, storageDisks, bucket, object) partsMetadata, errs := readAllXLMetadata(ctx, storageDisks, bucket, object)
// readQuorum suffices for xl.json since we use monotonic errCount := 0
// system time to break the tie when a split-brain situation for _, err := range errs {
// arises. if err != nil {
errCount++
}
}
if errCount == len(errs) {
// Only if we get errors from all the disks we return error. Else we need to
// continue to return filled madmin.HealResultItem struct which includes info
// on what disks the file is available etc.
if reducedErr := reduceReadQuorumErrs(ctx, errs, nil, quorum); reducedErr != nil { if reducedErr := reduceReadQuorumErrs(ctx, errs, nil, quorum); reducedErr != nil {
return result, toObjectErr(reducedErr, bucket, object) return result, toObjectErr(reducedErr, bucket, object)
} }
}
// List of disks having latest version of the object xl.json // List of disks having latest version of the object xl.json
// (by modtime). // (by modtime).
@ -536,17 +545,11 @@ func healObject(ctx context.Context, storageDisks []StorageAPI, bucket string, o
// and later the disk comes back up again, heal on the object // and later the disk comes back up again, heal on the object
// should delete it. // should delete it.
func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool) (hr madmin.HealResultItem, err error) { func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool) (hr madmin.HealResultItem, err error) {
// FIXME: Metadata is read again in the healObject() call below. // FIXME: Metadata is read again in the healObject() call below.
// Read metadata files from all the disks // Read metadata files from all the disks
partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object) partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object)
// get read quorum for this object latestXLMeta, _ := getLatestXLMeta(partsMetadata, errs)
var readQuorum int
readQuorum, _, err = objectQuorumFromMeta(xl, partsMetadata, errs)
if err != nil {
return hr, err
}
// Lock the object before healing. // Lock the object before healing.
objectLock := xl.nsMutex.NewNSLock(bucket, object) objectLock := xl.nsMutex.NewNSLock(bucket, object)
@ -556,5 +559,5 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
defer objectLock.RUnlock() defer objectLock.RUnlock()
// Heal the object. // Heal the object.
return healObject(ctx, xl.getDisks(), bucket, object, readQuorum, dryRun) return healObject(ctx, xl.getDisks(), bucket, object, latestXLMeta.Erasure.DataBlocks, dryRun)
} }

@ -144,7 +144,7 @@ func TestHealObjectXL(t *testing.T) {
// Try healing now, expect to receive errDiskNotFound. // Try healing now, expect to receive errDiskNotFound.
_, err = obj.HealObject(context.Background(), bucket, object, false) _, err = obj.HealObject(context.Background(), bucket, object, false)
// since majority of xl.jsons are not available, object quorum can't be read properly and error will be errXLReadQuorum // since majority of xl.jsons are not available, object quorum can't be read properly and error will be errXLReadQuorum
if err != errXLReadQuorum { if _, ok := err.(InsufficientReadQuorum); !ok {
t.Errorf("Expected %v but received %v", errDiskNotFound, err) t.Errorf("Expected %v but received %v", InsufficientWriteQuorum{}, err)
} }
} }

@ -387,8 +387,7 @@ func (m xlMetaV1) ObjectToPartOffset(ctx context.Context, offset int64) (partInd
} }
// pickValidXLMeta - picks one valid xlMeta content and returns from a // pickValidXLMeta - picks one valid xlMeta content and returns from a
// slice of xlmeta content. If no value is found this function panics // slice of xlmeta content.
// and dies.
func pickValidXLMeta(ctx context.Context, metaArr []xlMetaV1, modTime time.Time) (xmv xlMetaV1, e error) { func pickValidXLMeta(ctx context.Context, metaArr []xlMetaV1, modTime time.Time) (xmv xlMetaV1, e error) {
// Pick latest valid metadata. // Pick latest valid metadata.
for _, meta := range metaArr { for _, meta := range metaArr {

Loading…
Cancel
Save