format: Check properly for disks in valid formats. (#3427)

There was an error in how we validated disk formats,
if one of the disk was formatted and was formatted with
FS would cause confusion and object layer would never
initialize essentially go into an infinite loop.

Validate pre-emptively and also check for FS format
properly.
master
Harshavardhana 8 years ago committed by GitHub
parent 5c10f4adf0
commit 2d6f8153fa
  1. 40
      cmd/format-config-v1.go
  2. 38
      cmd/format-config-v1_test.go
  3. 36
      cmd/prepare-storage.go
  4. 2
      cmd/xl-v1-healing.go

@ -206,23 +206,24 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
return formatConfigs, sErrs return formatConfigs, sErrs
} }
// genericFormatCheck - validates and returns error. // genericFormatCheckFS - validates format config and returns an error if any.
// if (no quorum) return error func genericFormatCheckFS(formatConfig *formatConfigV1, sErr error) (err error) {
// if (any disk is corrupt) return error // phase2 if sErr != nil {
// if (jbod inconsistent) return error // phase2 return sErr
// if (disks not recognized) // Always error. }
func genericFormatCheck(formatConfigs []*formatConfigV1, sErrs []error) (err error) { // Successfully read, validate if FS.
if len(formatConfigs) == 1 { if !isFSFormat(formatConfig) {
// Successfully read, validate further.
if sErrs[0] == nil {
if !isFSFormat(formatConfigs[0]) {
return errFSDiskFormat return errFSDiskFormat
} }
return nil return nil
} // Returns error here.
return sErrs[0]
} }
// genericFormatCheckXL - validates and returns error.
// if (no quorum) return error
// if (any disk is corrupt) return error // phase2
// if (jbod inconsistent) return error // phase2
// if (disks not recognized) // Always error.
func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err error) {
// Calculate the errors. // Calculate the errors.
var ( var (
errCorruptFormatCount = 0 errCorruptFormatCount = 0
@ -248,12 +249,12 @@ func genericFormatCheck(formatConfigs []*formatConfigV1, sErrs []error) (err err
// Calculate read quorum. // Calculate read quorum.
readQuorum := len(formatConfigs) / 2 readQuorum := len(formatConfigs) / 2
// Validate the err count under tolerant limit. // Validate the err count under read quorum.
if errCount > len(formatConfigs)-readQuorum { if errCount > len(formatConfigs)-readQuorum {
return errXLReadQuorum return errXLReadQuorum
} }
// Check if number of corrupted format under quorum // Check if number of corrupted format under read quorum
if errCorruptFormatCount > len(formatConfigs)-readQuorum { if errCorruptFormatCount > len(formatConfigs)-readQuorum {
return errCorruptedFormat return errCorruptedFormat
} }
@ -793,8 +794,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
return reorderDisks(bootstrapDisks, formatConfigs) return reorderDisks(bootstrapDisks, formatConfigs)
} }
// checkFormatXL - verifies if format.json format is intact. func checkFormatXLValues(formatConfigs []*formatConfigV1) error {
func checkFormatXL(formatConfigs []*formatConfigV1) error {
for _, formatXL := range formatConfigs { for _, formatXL := range formatConfigs {
if formatXL == nil { if formatXL == nil {
continue continue
@ -813,6 +813,14 @@ func checkFormatXL(formatConfigs []*formatConfigV1) error {
return fmt.Errorf("Number of disks %d did not match the backend format %d", len(formatConfigs), len(formatXL.XL.JBOD)) return fmt.Errorf("Number of disks %d did not match the backend format %d", len(formatConfigs), len(formatXL.XL.JBOD))
} }
} }
return nil
}
// checkFormatXL - verifies if format.json format is intact.
func checkFormatXL(formatConfigs []*formatConfigV1) error {
if err := checkFormatXLValues(formatConfigs); err != nil {
return err
}
if err := checkJBODConsistency(formatConfigs); err != nil { if err := checkJBODConsistency(formatConfigs); err != nil {
return err return err
} }

@ -664,36 +664,58 @@ func TestReduceFormatErrs(t *testing.T) {
} }
} }
// Tests for genericFormatCheck() // Tests for genericFormatCheckFS()
func TestGenericFormatCheck(t *testing.T) { func TestGenericFormatCheckFS(t *testing.T) {
// Generate format configs for XL.
formatConfigs := genFormatXLInvalidJBOD()
// Validate disk format is fs, should fail.
if err := genericFormatCheckFS(formatConfigs[0], nil); err != errFSDiskFormat {
t.Fatalf("Unexpected error, expected %s, got %s", errFSDiskFormat, err)
}
// Validate disk is unformatted, should fail.
if err := genericFormatCheckFS(nil, errUnformattedDisk); err != errUnformattedDisk {
t.Fatalf("Unexpected error, expected %s, got %s", errUnformattedDisk, err)
}
// Validate when disk is in FS format.
format := newFSFormatV1()
if err := genericFormatCheckFS(format, nil); err != nil {
t.Fatalf("Unexpected error should pass, failed with %s", err)
}
}
// Tests for genericFormatCheckXL()
func TestGenericFormatCheckXL(t *testing.T) {
var errs []error var errs []error
formatConfigs := genFormatXLInvalidJBOD() formatConfigs := genFormatXLInvalidJBOD()
// Some disks has corrupted formats, one faulty disk // Some disks has corrupted formats, one faulty disk
errs = []error{nil, nil, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat, errs = []error{nil, nil, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat,
errCorruptedFormat, errFaultyDisk} errCorruptedFormat, errFaultyDisk}
if err := genericFormatCheck(formatConfigs, errs); err != errCorruptedFormat { if err := genericFormatCheckXL(formatConfigs, errs); err != errCorruptedFormat {
t.Fatal("Got unexpected err: ", err) t.Fatal("Got unexpected err: ", err)
} }
// Many faulty disks // Many faulty disks
errs = []error{nil, nil, errFaultyDisk, errFaultyDisk, errFaultyDisk, errFaultyDisk, errs = []error{nil, nil, errFaultyDisk, errFaultyDisk, errFaultyDisk, errFaultyDisk,
errCorruptedFormat, errFaultyDisk} errCorruptedFormat, errFaultyDisk}
if err := genericFormatCheck(formatConfigs, errs); err != errXLReadQuorum { if err := genericFormatCheckXL(formatConfigs, errs); err != errXLReadQuorum {
t.Fatal("Got unexpected err: ", err) t.Fatal("Got unexpected err: ", err)
} }
// All formats successfully loaded // All formats successfully loaded
errs = []error{nil, nil, nil, nil, nil, nil, nil, nil} errs = []error{nil, nil, nil, nil, nil, nil, nil, nil}
if err := genericFormatCheck(formatConfigs, errs); err == nil { if err := genericFormatCheckXL(formatConfigs, errs); err == nil {
t.Fatalf("Should fail here") t.Fatalf("Should fail here")
} }
errs = []error{nil} errs = []error{nil}
if err := genericFormatCheck([]*formatConfigV1{genFormatFS()}, errs); err != nil { if err := genericFormatCheckXL([]*formatConfigV1{genFormatFS()}, errs); err == nil {
t.Fatal("Got unexpected err: ", err) t.Fatalf("Should fail here")
} }
errs = []error{errFaultyDisk} errs = []error{errFaultyDisk}
if err := genericFormatCheck([]*formatConfigV1{genFormatFS()}, errs); err == nil { if err := genericFormatCheckXL([]*formatConfigV1{genFormatFS()}, errs); err == nil {
t.Fatalf("Should fail here") t.Fatalf("Should fail here")
} }
} }

@ -220,8 +220,24 @@ func retryFormattingDisks(firstDisk bool, endpoints []*url.URL, storageDisks []S
// for disks not being available. // for disks not being available.
printRetryMsg(sErrs, storageDisks) printRetryMsg(sErrs, storageDisks)
} }
// Check if this is a XL or distributed XL, anything > 1 is considered XL backend. if len(formatConfigs) == 1 {
if len(formatConfigs) > 1 { err := genericFormatCheckFS(formatConfigs[0], sErrs[0])
if err != nil {
if err == errUnformattedDisk {
return initFormatFS(storageDisks[0])
}
return err
}
return nil
} // Check if this is a XL or distributed XL, anything > 1 is considered XL backend.
// Pre-emptively check if one of the formatted disks
// is invalid. This function returns success for the
// most part unless one of the formats is not consistent
// with expected XL format. For example if a user is trying
// to pool FS backend.
if err := checkFormatXLValues(formatConfigs); err != nil {
return err
}
switch prepForInitXL(firstDisk, sErrs, len(storageDisks)) { switch prepForInitXL(firstDisk, sErrs, len(storageDisks)) {
case Abort: case Abort:
return errCorruptedFormat return errCorruptedFormat
@ -231,15 +247,15 @@ func retryFormattingDisks(firstDisk bool, endpoints []*url.URL, storageDisks []S
return initFormatXL(storageDisks) return initFormatXL(storageDisks)
case InitObjectLayer: case InitObjectLayer:
console.Eraseline() console.Eraseline()
// Validate formats load before proceeding forward. // Validate formats loaded before proceeding forward.
err := genericFormatCheck(formatConfigs, sErrs) err := genericFormatCheckXL(formatConfigs, sErrs)
if err == nil { if err == nil {
printRegularMsg(endpoints, storageDisks, printOnceFn()) printRegularMsg(endpoints, storageDisks, printOnceFn())
} }
return err return err
case WaitForHeal: case WaitForHeal:
// Validate formats load before proceeding forward. // Validate formats loaded before proceeding forward.
err := genericFormatCheck(formatConfigs, sErrs) err := genericFormatCheckXL(formatConfigs, sErrs)
if err == nil { if err == nil {
printHealMsg(endpoints, storageDisks, printOnceFn()) printHealMsg(endpoints, storageDisks, printOnceFn())
} }
@ -257,14 +273,6 @@ func retryFormattingDisks(firstDisk bool, endpoints []*url.URL, storageDisks []S
case WaitForFormatting: case WaitForFormatting:
console.Printf("Initializing data volume for first time. Waiting for first server to come online (elapsed %s)\n", getElapsedTime()) console.Printf("Initializing data volume for first time. Waiting for first server to come online (elapsed %s)\n", getElapsedTime())
} }
continue
} // else We have FS backend now. Check fs format as well now.
if isFormatFound(formatConfigs) {
console.Eraseline()
// Validate formats load before proceeding forward.
return genericFormatCheck(formatConfigs, sErrs)
} // else initialize the format for FS.
return initFormatFS(storageDisks[0])
case <-globalServiceDoneCh: case <-globalServiceDoneCh:
return errors.New("Initializing data volumes gracefully stopped") return errors.New("Initializing data volumes gracefully stopped")
} }

@ -31,7 +31,7 @@ func healFormatXL(storageDisks []StorageAPI) (err error) {
// Generic format check. // Generic format check.
// - if (no quorum) return error // - if (no quorum) return error
// - if (disks not recognized) // Always error. // - if (disks not recognized) // Always error.
if err = genericFormatCheck(formatConfigs, sErrs); err != nil { if err = genericFormatCheckXL(formatConfigs, sErrs); err != nil {
return err return err
} }

Loading…
Cancel
Save