oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

XL: Simplify heal-format operations

Browse Source 
This is in preparation for updated admin heal API.

* Improve case analysis of healFormatXL() - fixes a case where disks
  could have unhandled errors.

* Simplify healFormatXLFreshDisks() and healFormatXLCorruptedDisks()
  to share more code and handle fewer cases for improved simplicity
  and reduced code repetition.

* Fix test cases.
master
Aditya Manthramurthy 7 years ago committed by Harshavardhana
parent b10fa507b2
commit 32da1aa9d6
4 changed files with 223 additions and 435 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 345
      cmd/format-config-v1.go
  2. 232
      cmd/format-config-v1_test.go
  3. 36
      cmd/xl-v1-healing.go
  4. 37
      cmd/xl-v1-healing_test.go

345
cmd/format-config-v1.go
Unescape View File

@ -241,48 +241,32 @@ else // No healing at this point forward, some disks are offline or dead.
fi fi
*/ */
// error returned when some disks are found to be unformatted.
var errSomeDiskUnformatted = errors.New("some disks are found to be unformatted")
// error returned when some disks are offline. // error returned when some disks are offline.
var errSomeDiskOffline = errors.New("some disks are offline") var errSomeDiskOffline = errors.New("some disks are offline")
// errDiskOrderMismatch - returned when disk UUID is not in consistent JBOD order. // errDiskOrderMismatch - returned when disk UUID is not in consistent JBOD order.
var errDiskOrderMismatch = errors.New("disk order mismatch") var errDiskOrderMismatch = errors.New("disk order mismatch")
// Returns error slice into understandable errors. // formatErrsSummary - summarizes errors into different classes
func reduceFormatErrs(errs []error, diskCount int) (err error) { func formatErrsSummary(errs []error) (formatCount, unformattedDiskCount,
var errUnformattedDiskCount = 0 diskNotFoundCount, corruptedFormatCount, otherErrCount int) {
var errDiskNotFoundCount = 0
var errCorruptedFormatCount = 0 for _, err := range errs {
for _, dErr := range errs { switch err {
if dErr == errUnformattedDisk { case errDiskNotFound:
errUnformattedDiskCount++ diskNotFoundCount++
} else if dErr == errDiskNotFound { case errUnformattedDisk:
errDiskNotFoundCount++ unformattedDiskCount++
} else if dErr == errCorruptedFormat { case errCorruptedFormat:
errCorruptedFormatCount++ corruptedFormatCount++
} case nil:
} // implies that format is not nil
if errCorruptedFormatCount > 0 { formatCount++
return errCorruptedFormat default:
} otherErrCount++
// Unformatted disks found, we need to figure out if any disks are offline.
if errUnformattedDiskCount > 0 {
// Returns errUnformattedDisk if all disks report unFormattedDisk.
if errUnformattedDiskCount < diskCount {
if errDiskNotFoundCount > 0 {
// Only some disks are fresh but some disks are offline as well.
return errSomeDiskOffline
}
// Some disks are fresh disks an unformatted, not disks are offline.
return errSomeDiskUnformatted
} }
// All disks returned unformatted, all disks must be fresh.
return errUnformattedDisk
} }
// No unformatted disks found no need to handle disk not found case, return success here. return
return nil
} }
// loadAllFormats - load all format config from all input disks in parallel. // loadAllFormats - load all format config from all input disks in parallel.
@ -469,29 +453,50 @@ func findDiskIndex(disk string, jbod []string) int {
return -1 return -1
} }
// reorderDisks - reorder disks in JBOD order. // reorderDisks - reorder disks in JBOD order, and return reference
func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) { // format-config. If assignUUIDs is true, it assigns UUIDs to disks
var savedJBOD []string // with missing format configurations in the reference configuration.
for _, format := range formatConfigs { func reorderDisks(bootstrapDisks []StorageAPI,
if format == nil { formatConfigs []*formatConfigV1, assignUUIDs bool) (*formatConfigV1,
continue []StorageAPI, error) {
}
savedJBOD = format.XL.JBOD // Pick first non-nil format-cfg as reference
var refCfg *formatConfigV1
for _, formatConfig := range formatConfigs {
if formatConfig != nil {
refCfg = formatConfig
break break
} }
// Pick the first JBOD list to verify the order and construct new set of disk slice. }
if refCfg == nil {
return nil, nil, fmt.Errorf("could not find any valid config")
}
refJBOD := refCfg.XL.JBOD
// construct reordered disk slice
var newDisks = make([]StorageAPI, len(bootstrapDisks)) var newDisks = make([]StorageAPI, len(bootstrapDisks))
for fIndex, format := range formatConfigs { for fIndex, format := range formatConfigs {
if format == nil { if format == nil {
continue continue
} }
jIndex := findDiskIndex(format.XL.Disk, savedJBOD) jIndex := findDiskIndex(format.XL.Disk, refJBOD)
if jIndex == -1 { if jIndex == -1 {
return nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found") return nil, nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
} }
newDisks[jIndex] = bootstrapDisks[fIndex] newDisks[jIndex] = bootstrapDisks[fIndex]
} }
return newDisks, nil
if assignUUIDs {
// Based on orderedDisks generate new UUIDs in the ref. config
// for disks without format-configs.
for index, disk := range newDisks {
if disk == nil {
refCfg.XL.JBOD[index] = mustGetUUID()
}
}
}
return refCfg, newDisks, nil
} }
// loadFormat - loads format.json from disk. // loadFormat - loads format.json from disk.
@ -550,81 +555,14 @@ func isFormatFound(formats []*formatConfigV1) bool {
return true return true
} }
// Heals any missing format.json on the drives. Returns error only for unexpected errors // collectNSaveNewFormatConfigs - creates new format configs based on
// as regular errors can be ignored since there might be enough quorum to be operational. // the reference config and saves it on all disks, this is to be
// Heals only fresh disks. // called from healFormatXL* functions.
func healFormatXLFreshDisks(storageDisks []StorageAPI) error { func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1,
formatConfigs := make([]*formatConfigV1, len(storageDisks)) orderedDisks []StorageAPI) error {
var referenceConfig *formatConfigV1
// Loads `format.json` from all disks.
for index, disk := range storageDisks {
// Disk not found or ignored is a valid case.
if disk == nil {
// Return nil, one of the disk is offline.
return nil
}
formatXL, err := loadFormat(disk)
if err != nil {
if err == errUnformattedDisk {
// format.json is missing, should be healed.
continue
} else if err == errDiskNotFound { // Is a valid case we
// can proceed without healing.
return nil
}
// Return error for unsupported errors.
return err
} // Success.
formatConfigs[index] = formatXL
}
// All `format.json` has been read successfully, previously completed.
if isFormatFound(formatConfigs) {
// Return success.
return nil
}
// All disks are fresh, format.json will be written by initFormatXL()
if isFormatNotFound(formatConfigs) {
return initFormatXL(storageDisks)
}
// Validate format configs for consistency in JBOD and disks.
if err := checkFormatXL(formatConfigs); err != nil {
return err
}
if referenceConfig == nil {
// This config will be used to update the drives missing format.json.
for _, formatConfig := range formatConfigs {
if formatConfig == nil {
continue
}
referenceConfig = formatConfig
break
}
}
// Collect new JBOD.
newJBOD := referenceConfig.XL.JBOD
// Reorder the disks based on the JBOD order.
orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
if err != nil {
return err
}
// From ordered disks fill the UUID position.
for index, disk := range orderedDisks {
if disk == nil {
newJBOD[index] = mustGetUUID()
}
}
// Collect new format configs.
var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
// Collect new format configs that need to be written. // Collect new format configs that need to be written.
var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
for index := range orderedDisks { for index := range orderedDisks {
// New configs are generated since we are going // New configs are generated since we are going
// to re-populate across all disks. // to re-populate across all disks.
@ -633,13 +571,35 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI) error {
Format: referenceConfig.Format, Format: referenceConfig.Format,
XL: &xlFormat{ XL: &xlFormat{
Version: referenceConfig.XL.Version, Version: referenceConfig.XL.Version,
Disk: newJBOD[index], Disk: referenceConfig.XL.JBOD[index],
JBOD: newJBOD, JBOD: referenceConfig.XL.JBOD,
}, },
} }
newFormatConfigs[index] = config newFormatConfigs[index] = config
} }
// Initialize meta volume, if volume already exists ignores it.
if err := initMetaVolume(orderedDisks); err != nil {
return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
}
// Save new `format.json` across all disks, in JBOD order.
return saveFormatXL(orderedDisks, newFormatConfigs)
}
// Heals any missing format.json on the drives. Returns error only for
// unexpected errors as regular errors can be ignored since there
// might be enough quorum to be operational. Heals only fresh disks.
func healFormatXLFreshDisks(storageDisks []StorageAPI,
formatConfigs []*formatConfigV1) error {
// Reorder the disks based on the JBOD order.
referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
formatConfigs, true)
if err != nil {
return err
}
// Fill in the missing disk back from format configs. // Fill in the missing disk back from format configs.
// We need to make sure we have kept the previous order // We need to make sure we have kept the previous order
// and allowed fresh disks to be arranged anywhere. // and allowed fresh disks to be arranged anywhere.
@ -658,37 +618,37 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI) error {
} }
} }
// Initialize meta volume, if volume already exists ignores it. // apply new format config and save to all disks
if err := initMetaVolume(orderedDisks); err != nil { return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
} }
// Save new `format.json` across all disks, in JBOD order. // collectUnAssignedDisks - collect disks unassigned to orderedDisks
return saveFormatXL(orderedDisks, newFormatConfigs) // from storageDisks and return them.
} func collectUnAssignedDisks(storageDisks, orderedDisks []StorageAPI) (
uDisks []StorageAPI) {
// Disks from storageDiks are put in assignedDisks if found in orderedDisks and in unAssignedDisks otherwise // search for each disk from storageDisks in orderedDisks
func splitDisksByUse(storageDisks, orderedDisks []StorageAPI) (assignedDisks []StorageAPI, unAssignedDisks []StorageAPI) {
// Populate unAssignDisks
for i := range storageDisks { for i := range storageDisks {
found := false found := false
for j := range orderedDisks { for j := range orderedDisks {
if storageDisks[i] == orderedDisks[j] { if storageDisks[i] == orderedDisks[j] {
found = true found = true
assignedDisks = append(assignedDisks, storageDisks[i])
break break
} }
} }
if !found { if !found {
unAssignedDisks = append(unAssignedDisks, storageDisks[i]) // append not found disk to result
uDisks = append(uDisks, storageDisks[i])
} }
} }
return assignedDisks, unAssignedDisks return uDisks
} }
// Inspect the content of all disks to guess the right order according to the format files. // Inspect the content of all disks to guess the right order according
// The right order is represented in orderedDisks // to the format files. The right order is represented in orderedDisks
func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) { func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
for index, format := range formatConfigs { for index, format := range formatConfigs {
if format != nil { if format != nil {
continue continue
@ -701,7 +661,8 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatCon
continue continue
} }
volName := "" volName := ""
// Avoid picking minioMetaBucket because ListVols() returns a non ordered list // Avoid picking minioMetaBucket because ListVols()
// returns a non ordered list
for i := range vols { for i := range vols {
if vols[i].Name != minioMetaBucket { if vols[i].Name != minioMetaBucket {
volName = vols[i].Name volName = vols[i].Name
@ -742,84 +703,28 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatCon
} }
// Heals corrupted format json in all disks // Heals corrupted format json in all disks
func healFormatXLCorruptedDisks(storageDisks []StorageAPI) error { func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
formatConfigs := make([]*formatConfigV1, len(storageDisks)) formatConfigs []*formatConfigV1) error {
var referenceConfig *formatConfigV1
// Loads `format.json` from all disks.
for index, disk := range storageDisks {
// Disk not found or ignored is a valid case.
if disk == nil {
// Return nil, one of the disk is offline.
return nil
}
formatXL, err := loadFormat(disk)
if err != nil {
if err == errUnformattedDisk || err == errCorruptedFormat {
// format.json is missing or corrupted, should be healed.
continue
} else if err == errDiskNotFound { // Is a valid case we
// can proceed without healing.
return nil
}
// Return error for unsupported errors.
return err
} // Success.
formatConfigs[index] = formatXL
}
// All `format.json` has been read successfully, previously completed.
if isFormatFound(formatConfigs) {
// Return success.
return nil
}
// All disks are fresh, format.json will be written by initFormatXL()
if isFormatNotFound(formatConfigs) {
return initFormatXL(storageDisks)
}
// Validate format configs for consistency in JBOD and disks.
if err := checkFormatXL(formatConfigs); err != nil {
return err
}
if referenceConfig == nil {
// This config will be used to update the drives missing format.json.
for _, formatConfig := range formatConfigs {
if formatConfig == nil {
continue
}
referenceConfig = formatConfig
break
}
}
// Collect new JBOD.
newJBOD := referenceConfig.XL.JBOD
// Reorder the disks based on the JBOD order. // Reorder the disks based on the JBOD order.
orderedDisks, err := reorderDisks(storageDisks, formatConfigs) referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
formatConfigs, true)
if err != nil { if err != nil {
return err return err
} }
// From ordered disks fill the UUID position. // For disks with corrupted formats, inspect the disks
for index, disk := range orderedDisks { // contents to guess the disks order
if disk == nil { orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks,
newJBOD[index] = mustGetUUID() formatConfigs)
}
}
// For disks with corrupted formats, inspect the disks contents to guess the disks order
orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks, formatConfigs)
if err != nil { if err != nil {
return err return err
} }
// At this stage, all disks with corrupted formats but with objects inside found their way. // At this stage, all disks with corrupted formats but with
// Now take care of unformatted disks, which are the `unAssignedDisks` // objects inside found their way. Now take care of
_, unAssignedDisks := splitDisksByUse(storageDisks, orderedDisks) // unformatted disks, which are the `unAssignedDisks`
unAssignedDisks := collectUnAssignedDisks(storageDisks, orderedDisks)
// Assign unassigned disks to nil elements in orderedDisks // Assign unassigned disks to nil elements in orderedDisks
for i, disk := range orderedDisks { for i, disk := range orderedDisks {
@ -829,27 +734,8 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI) error {
} }
} }
// Collect new format configs. // apply new format config and save to all disks
var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks)) return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
// Collect new format configs that need to be written.
for index := range orderedDisks {
// New configs are generated since we are going
// to re-populate across all disks.
config := &formatConfigV1{
Version: referenceConfig.Version,
Format: referenceConfig.Format,
XL: &xlFormat{
Version: referenceConfig.XL.Version,
Disk: newJBOD[index],
JBOD: newJBOD,
},
}
newFormatConfigs[index] = config
}
// Save new `format.json` across all disks, in JBOD order.
return saveFormatXL(orderedDisks, newFormatConfigs)
} }
// loadFormatXL - loads XL `format.json` and returns back properly // loadFormatXL - loads XL `format.json` and returns back properly
@ -900,8 +786,11 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
if err = checkFormatXL(formatConfigs); err != nil { if err = checkFormatXL(formatConfigs); err != nil {
return nil, err return nil, err
} }
// Erasure code requires disks to be presented in the same order each time. // Erasure code requires disks to be presented in the same
return reorderDisks(bootstrapDisks, formatConfigs) // order each time.
_, orderedDisks, err := reorderDisks(bootstrapDisks, formatConfigs,
false)
return orderedDisks, err
} }
func checkFormatXLValue(formatXL *formatConfigV1) error { func checkFormatXLValue(formatXL *formatConfigV1) error {

232
cmd/format-config-v1_test.go
Unescape View File

@ -288,8 +288,12 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// Attempt to load all `format.json`.
formatConfigs, _ := loadAllFormats(storageDisks)
// Start healing disks // Start healing disks
err = healFormatXLFreshDisks(storageDisks) err = healFormatXLFreshDisks(storageDisks, formatConfigs)
if err != nil { if err != nil {
t.Fatal("healing corrupted disk failed: ", err) t.Fatal("healing corrupted disk failed: ", err)
} }
@ -304,42 +308,6 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
removeRoots(fsDirs) removeRoots(fsDirs)
} }
func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
nDisks := 16
fsDirs, err := getRandomDisks(nDisks)
if err != nil {
t.Fatal(err)
}
// Create an instance of xl backend.
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Error(err)
}
storageDisks, err := prepareFormatXLHealFreshDisks(obj)
if err != nil {
t.Fatal(err)
}
// Prepares all disks are offline.
prepareNOfflineDisks(storageDisks, 16, t)
// Load again XL format.json to validate it
_, err = loadFormatXL(storageDisks, 8)
if err == nil {
t.Fatal("loading format disk error")
}
storageDisks[3] = nil
err = healFormatXLFreshDisks(storageDisks)
if err != nil {
t.Fatal("didn't get nil when one disk is offline")
}
// Clean all
removeRoots(fsDirs)
}
// Simulate XL disks creation, delete some format.json and remove the content of // Simulate XL disks creation, delete some format.json and remove the content of
// a given disk to test healing a corrupted disk // a given disk to test healing a corrupted disk
func TestFormatXLHealCorruptedDisks(t *testing.T) { func TestFormatXLHealCorruptedDisks(t *testing.T) {
@ -397,8 +365,10 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9], xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
xl.storageDisks[5], xl.storageDisks[11]} xl.storageDisks[5], xl.storageDisks[11]}
formatConfigs, _ := loadAllFormats(permutedStorageDisks)
// Start healing disks // Start healing disks
err = healFormatXLCorruptedDisks(permutedStorageDisks) err = healFormatXLCorruptedDisks(permutedStorageDisks, formatConfigs)
if err != nil { if err != nil {
t.Fatal("healing corrupted disk failed: ", err) t.Fatal("healing corrupted disk failed: ", err)
} }
@ -454,7 +424,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
permutedFormatConfigs, _ := loadAllFormats(permutedStorageDisks) permutedFormatConfigs, _ := loadAllFormats(permutedStorageDisks)
orderedDisks, err := reorderDisks(permutedStorageDisks, permutedFormatConfigs) _, orderedDisks, err := reorderDisks(permutedStorageDisks, permutedFormatConfigs, false)
if err != nil { if err != nil {
t.Fatal("error reordering disks\n") t.Fatal("error reordering disks\n")
} }
@ -629,27 +599,29 @@ func TestInitFormatXLErrors(t *testing.T) {
} }
} }
// Test for reduceFormatErrs() // Test formatErrsSummary()
func TestReduceFormatErrs(t *testing.T) { func TestFormatErrsSummary(t *testing.T) {
// No error founds type errSummary struct {
if err := reduceFormatErrs([]error{nil, nil, nil, nil}, 4); err != nil { fc, unfmt, ntfnd, crrptd, othr int
t.Fatal("Err should be nil, found: ", err)
}
// One corrupted format
if err := reduceFormatErrs([]error{nil, nil, errCorruptedFormat, nil}, 4); err != errCorruptedFormat {
t.Fatal("Got a different error: ", err)
} }
// All disks unformatted
if err := reduceFormatErrs([]error{errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk}, 4); err != errUnformattedDisk { testCases := []struct {
t.Fatal("Got a different error: ", err) errs []error
expected errSummary
}{
{nil, errSummary{0, 0, 0, 0, 0}},
{[]error{errDiskNotFound, errUnformattedDisk, errCorruptedFormat, nil, errFaultyDisk},
errSummary{1, 1, 1, 1, 1}},
{[]error{errDiskNotFound, errDiskNotFound, errCorruptedFormat, nil, nil},
errSummary{2, 0, 2, 1, 0}},
} }
// Some disks unformatted for i, testCase := range testCases {
if err := reduceFormatErrs([]error{nil, nil, errUnformattedDisk, errUnformattedDisk}, 4); err != errSomeDiskUnformatted { a, b, c, d, e := formatErrsSummary(testCase.errs)
t.Fatal("Got a different error: ", err) got := errSummary{a, b, c, d, e}
if got != testCase.expected {
t.Errorf("Test %d: Got wrong results: %#v %#v", i+1,
got, testCase.expected)
} }
// Some disks offline
if err := reduceFormatErrs([]error{nil, nil, errDiskNotFound, errUnformattedDisk}, 4); err != errSomeDiskOffline {
t.Fatal("Got a different error: ", err)
} }
} }
@ -959,8 +931,10 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
xl := obj.(*xlObjects) xl := obj.(*xlObjects)
if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil { formatConfigs, _ := loadAllFormats(xl.storageDisks)
if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != nil {
t.Fatal("Got an unexpected error: ", err) t.Fatal("Got an unexpected error: ", err)
} }
@ -971,25 +945,6 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// Disks 0..15 are nil
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
xl = obj.(*xlObjects)
for i := 0; i <= 15; i++ {
xl.storageDisks[i] = nil
}
if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
t.Fatal("Got an unexpected error: ", err)
}
removeRoots(fsDirs)
fsDirs, err = getRandomDisks(nDisks)
if err != nil {
t.Fatal(err)
}
// One disk returns Faulty Disk // One disk returns Faulty Disk
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...)) obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil { if err != nil {
@ -1001,45 +956,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal("storage disk is not *retryStorage type") t.Fatal("storage disk is not *retryStorage type")
} }
xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk) xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
if err = healFormatXLCorruptedDisks(xl.storageDisks); err != errFaultyDisk { formatConfigs, _ = loadAllFormats(xl.storageDisks)
t.Fatal("Got an unexpected error: ", err) if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk {
}
removeRoots(fsDirs)
fsDirs, err = getRandomDisks(nDisks)
if err != nil {
t.Fatal(err)
}
// One disk is not found, heal corrupted disks should return nil
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
xl = obj.(*xlObjects)
xl.storageDisks[0] = nil
if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
t.Fatal("Got an unexpected error: ", err)
}
removeRoots(fsDirs)
fsDirs, err = getRandomDisks(nDisks)
if err != nil {
t.Fatal(err)
}
// Remove format.json of all disks
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
xl = obj.(*xlObjects)
for i := 0; i <= 15; i++ {
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
t.Fatal(err)
}
}
if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
t.Fatal("Got an unexpected error: ", err) t.Fatal("Got an unexpected error: ", err)
} }
removeRoots(fsDirs) removeRoots(fsDirs)
@ -1060,7 +978,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
if err = healFormatXLCorruptedDisks(xl.storageDisks); err == nil { formatConfigs, _ = loadAllFormats(xl.storageDisks)
if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err == nil {
t.Fatal("Should get a json parsing error, ") t.Fatal("Should get a json parsing error, ")
} }
removeRoots(fsDirs) removeRoots(fsDirs)
@ -1086,26 +1005,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
xl := obj.(*xlObjects) xl := obj.(*xlObjects)
if err = healFormatXLFreshDisks(xl.storageDisks); err != nil { formatConfigs, _ := loadAllFormats(xl.storageDisks)
t.Fatal("Got an unexpected error: ", err) if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil {
}
removeRoots(fsDirs)
fsDirs, err = getRandomDisks(nDisks)
if err != nil {
t.Fatal(err)
}
// Disks 0..15 are nil
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
xl = obj.(*xlObjects)
for i := 0; i <= 15; i++ {
xl.storageDisks[i] = nil
}
if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
t.Fatal("Got an unexpected error: ", err) t.Fatal("Got an unexpected error: ", err)
} }
removeRoots(fsDirs) removeRoots(fsDirs)
@ -1126,7 +1027,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
t.Fatal("storage disk is not *retryStorage type") t.Fatal("storage disk is not *retryStorage type")
} }
xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk) xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
if err = healFormatXLFreshDisks(xl.storageDisks); err != errFaultyDisk { formatConfigs, _ = loadAllFormats(xl.storageDisks)
if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk {
t.Fatal("Got an unexpected error: ", err) t.Fatal("Got an unexpected error: ", err)
} }
removeRoots(fsDirs) removeRoots(fsDirs)
@ -1143,59 +1045,9 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
} }
xl = obj.(*xlObjects) xl = obj.(*xlObjects)
xl.storageDisks[0] = nil xl.storageDisks[0] = nil
if err = healFormatXLFreshDisks(xl.storageDisks); err != nil { formatConfigs, _ = loadAllFormats(xl.storageDisks)
if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil {
t.Fatal("Got an unexpected error: ", err) t.Fatal("Got an unexpected error: ", err)
} }
removeRoots(fsDirs) removeRoots(fsDirs)
fsDirs, err = getRandomDisks(nDisks)
if err != nil {
t.Fatal(err)
}
// Remove format.json of all disks
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
xl = obj.(*xlObjects)
for i := 0; i <= 15; i++ {
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
t.Fatal(err)
}
}
if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
t.Fatal("Got an unexpected error: ", err)
}
removeRoots(fsDirs)
}
// Tests for isFormatFound()
func TestIsFormatFound(t *testing.T) {
formats := genFormatXLValid()
if found := isFormatFound(formats); !found {
t.Fatal("isFormatFound() should not return false")
}
formats[0] = nil
if found := isFormatFound(formats); found {
t.Fatal("isFormatFound() should not return true")
}
}
// Tests for isFormatNotFound()
func TestIsFormatNotFound(t *testing.T) {
formats := genFormatXLValid()
if found := isFormatNotFound(formats); found {
t.Fatal("isFormatFound() should not return true")
}
formats[0] = nil
if found := isFormatNotFound(formats); found {
t.Fatal("isFormatFound() should not return true")
}
for idx := range formats {
formats[idx] = nil
}
if found := isFormatNotFound(formats); !found {
t.Fatal("isFormatFound() should not return false")
}
} }

36
cmd/xl-v1-healing.go
Unescape View File

@ -36,21 +36,37 @@ func healFormatXL(storageDisks []StorageAPI) (err error) {
return err return err
} }
// Handles different cases properly. numDisks := len(storageDisks)
switch reduceFormatErrs(sErrs, len(storageDisks)) { _, unformattedDiskCount, diskNotFoundCount,
case errCorruptedFormat: corruptedFormatCount, otherErrCount := formatErrsSummary(sErrs)
if err = healFormatXLCorruptedDisks(storageDisks); err != nil {
switch {
case unformattedDiskCount == numDisks:
// all unformatted.
if err = initFormatXL(storageDisks); err != nil {
return err
}
case diskNotFoundCount > 0:
return fmt.Errorf("cannot proceed with heal as %s",
errSomeDiskOffline)
case otherErrCount > 0:
return fmt.Errorf("cannot proceed with heal as some disks had unhandled errors")
case corruptedFormatCount > 0:
if err = healFormatXLCorruptedDisks(storageDisks, formatConfigs); err != nil {
return fmt.Errorf("Unable to repair corrupted format, %s", err) return fmt.Errorf("Unable to repair corrupted format, %s", err)
} }
case errSomeDiskUnformatted:
case unformattedDiskCount > 0:
// All drives online but some report missing format.json. // All drives online but some report missing format.json.
if err = healFormatXLFreshDisks(storageDisks); err != nil { if err = healFormatXLFreshDisks(storageDisks, formatConfigs); err != nil {
// There was an unexpected unrecoverable error during healing. // There was an unexpected unrecoverable error
// during healing.
return fmt.Errorf("Unable to heal backend %s", err) return fmt.Errorf("Unable to heal backend %s", err)
} }
case errSomeDiskOffline:
// FIXME: in future.
return fmt.Errorf("Unable to initialize format %s and %s", errSomeDiskOffline, errSomeDiskUnformatted)
} }
return nil return nil
} }

37
cmd/xl-v1-healing_test.go
Unescape View File

@ -97,14 +97,15 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
// One disk is not found, heal corrupted disks should return nil // One disk is not found, heal corrupted disks should return
// error for offline disk
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...)) obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
xl = obj.(*xlObjects) xl = obj.(*xlObjects)
xl.storageDisks[0] = nil xl.storageDisks[0] = nil
if err = healFormatXL(xl.storageDisks); err != nil { if err = healFormatXL(xl.storageDisks); err != nil && err.Error() != "cannot proceed with heal as some disks are offline" {
t.Fatal("Got an unexpected error: ", err) t.Fatal("Got an unexpected error: ", err)
} }
removeRoots(fsDirs) removeRoots(fsDirs)
@ -193,7 +194,37 @@ func TestHealFormatXL(t *testing.T) {
t.Fatal("storage disk is not *retryStorage type") t.Fatal("storage disk is not *retryStorage type")
} }
xl.storageDisks[3] = newNaughtyDisk(posixDisk, nil, errDiskNotFound) xl.storageDisks[3] = newNaughtyDisk(posixDisk, nil, errDiskNotFound)
expectedErr := fmt.Errorf("Unable to initialize format %s and %s", errSomeDiskOffline, errSomeDiskUnformatted) expectedErr := fmt.Errorf("cannot proceed with heal as %s", errSomeDiskOffline)
if err = healFormatXL(xl.storageDisks); err != nil {
if err.Error() != expectedErr.Error() {
t.Fatal("Got an unexpected error: ", err)
}
}
removeRoots(fsDirs)
fsDirs, err = getRandomDisks(nDisks)
if err != nil {
t.Fatal(err)
}
// One disk has access denied error, heal should return
// appropriate error
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
if err != nil {
t.Fatal(err)
}
xl = obj.(*xlObjects)
for i := 0; i <= 2; i++ {
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
t.Fatal(err)
}
}
posixDisk, ok = xl.storageDisks[3].(*retryStorage)
if !ok {
t.Fatal("storage disk is not *retryStorage type")
}
xl.storageDisks[3] = newNaughtyDisk(posixDisk, nil, errDiskAccessDenied)
expectedErr = fmt.Errorf("cannot proceed with heal as some disks had unhandled errors")
if err = healFormatXL(xl.storageDisks); err != nil { if err = healFormatXL(xl.storageDisks); err != nil {
if err.Error() != expectedErr.Error() { if err.Error() != expectedErr.Error() {
t.Fatal("Got an unexpected error: ", err) t.Fatal("Got an unexpected error: ", err)

Write Preview
Loading…
Cancel
Save