diff --git a/cmd/format-fs.go b/cmd/format-fs.go index 374b889f4..e5c195ae8 100644 --- a/cmd/format-fs.go +++ b/cmd/format-fs.go @@ -24,6 +24,7 @@ import ( "path" "time" + "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/lock" ) @@ -55,11 +56,18 @@ type formatFSVersionDetect struct { } `json:"fs"` } +// Generic structure to manage both v1 and v2 structures +type formatFS struct { + formatMetaV1 + FS interface{} `json:"fs"` +} + // Returns the latest "fs" format V1 func newFormatFSV1() (format *formatFSV1) { f := &formatFSV1{} f.Version = formatMetaVersionV1 f.Format = formatBackendFS + f.ID = mustGetUUID() f.FS.Version = formatFSVersionV1 return f } @@ -69,6 +77,7 @@ func newFormatFSV2() (format *formatFSV2) { f := &formatFSV2{} f.Version = formatMetaVersionV1 f.Format = formatBackendFS + f.ID = mustGetUUID() f.FS.Version = formatFSVersionV2 return f } @@ -115,7 +124,16 @@ func formatFSMigrateV1ToV2(ctx context.Context, wlk *lock.LockedFile, fsPath str return err } - return jsonSave(wlk.File, newFormatFSV2()) + formatV1 := formatFSV1{} + if err = jsonLoad(wlk, &formatV1); err != nil { + return err + } + + formatV2 := formatFSV2{} + formatV2.formatMetaV1 = formatV1.formatMetaV1 + formatV2.FS.Version = formatFSVersionV2 + + return jsonSave(wlk.File, formatV2) } // Migrate the "fs" backend. @@ -179,6 +197,12 @@ func createFormatFS(ctx context.Context, fsFormatPath string) error { // migrate the backend when we are actively working on the backend. func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, err error) { fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile) + + // Add a deployment ID, if it does not exist. + if err := formatFSFixDeploymentID(fsFormatPath); err != nil { + return nil, err + } + // Any read on format.json should be done with read-lock. // Any write on format.json should be done with write-lock. for { @@ -235,7 +259,8 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er rlk.Close() // Hold write lock during migration so that we do not disturb any // minio processes running in parallel. - wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0) + var wlk *lock.LockedFile + wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0) if err == lock.ErrAlreadyLocked { // Lock already present, sleep and attempt again. time.Sleep(100 * time.Millisecond) @@ -253,7 +278,94 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er // Successfully migrated, now try to hold a read-lock on format.json continue } - + var id string + if id, err = formatFSGetDeploymentID(rlk); err != nil { + rlk.Close() + return nil, err + } + logger.SetDeploymentID(id) return rlk, nil } } + +func formatFSGetDeploymentID(rlk *lock.RLockedFile) (id string, err error) { + format := &formatFS{} + if err := jsonLoad(rlk, format); err != nil { + return "", err + } + return format.ID, nil +} + +// Generate a deployment ID if one does not exist already. +func formatFSFixDeploymentID(fsFormatPath string) error { + rlk, err := lock.RLockedOpenFile(fsFormatPath) + if err == nil { + // format.json can be empty in a rare condition when another + // minio process just created the file but could not hold lock + // and write to it. + var fi os.FileInfo + fi, err = rlk.Stat() + if err != nil { + rlk.Close() + return err + } + if fi.Size() == 0 { + rlk.Close() + return nil + } + } + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + formatBackend, err := formatMetaGetFormatBackendFS(rlk) + if err != nil { + rlk.Close() + return err + } + if formatBackend != formatBackendFS { + rlk.Close() + return fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend) + } + + format := &formatFS{} + err = jsonLoad(rlk, format) + rlk.Close() + if err != nil { + return err + } + + // Check if it needs to be updated + if format.ID != "" { + return nil + } + + for { + wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0) + if err == lock.ErrAlreadyLocked { + // Lock already present, sleep and attempt again. + time.Sleep(100 * time.Millisecond) + continue + } + if err != nil { + return err + } + defer wlk.Close() + + err = jsonLoad(wlk, format) + if err != nil { + return err + } + + // Check if it needs to be updated + if format.ID != "" { + return nil + } + format.ID = mustGetUUID() + return jsonSave(wlk, format) + } + +} diff --git a/cmd/format-meta.go b/cmd/format-meta.go index 9946bed52..cf479634b 100644 --- a/cmd/format-meta.go +++ b/cmd/format-meta.go @@ -49,4 +49,6 @@ type formatMetaV1 struct { Version string `json:"version"` // Format indicates the backend format type, supports two values 'xl' and 'fs'. Format string `json:"format"` + // ID is the identifier for the minio deployment + ID string `json:"id"` } diff --git a/cmd/format-xl.go b/cmd/format-xl.go index d3f44e1ba..41f9008c6 100644 --- a/cmd/format-xl.go +++ b/cmd/format-xl.go @@ -22,11 +22,13 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "sync" "encoding/hex" humanize "github.com/dustin/go-humanize" + "github.com/minio/minio/cmd/logger" sha256 "github.com/minio/sha256-simd" ) @@ -127,6 +129,7 @@ func newFormatXLV3(numSets int, setLen int) *formatXLV3 { format := &formatXLV3{} format.Version = formatMetaVersionV1 format.Format = formatBackendXL + format.ID = mustGetUUID() format.XL.Version = formatXLVersionV3 format.XL.DistributionAlgo = formatXLVersionV2DistributionAlgo format.XL.Sets = make([][]string, numSets) @@ -443,6 +446,112 @@ func checkFormatXLValues(formats []*formatXLV3) error { return nil } +// Get Deployment ID for the XL sets from format.json. +// This need not be in quorum. Even if one of the format.json +// file has this value, we assume it is valid. +// If more than one format.json's have different id, it is considered a corrupt +// backend format. +func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (string, error) { + var deploymentID string + for _, format := range formats { + if format == nil || format.ID == "" { + continue + } + if reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) { + // Found an ID in one of the format.json file + // Set deploymentID for the first time. + if deploymentID == "" { + deploymentID = format.ID + } else if deploymentID != format.ID { + // DeploymentID found earlier doesn't match with the + // current format.json's ID. + return "", errCorruptedFormat + } + } + } + return deploymentID, nil +} + +// formatXLFixDeploymentID - Add deployment id if it is not present. +func formatXLFixDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) { + // Acquire lock on format.json + mutex := newNSLock(globalIsDistXL) + formatLock := mutex.NewNSLock(minioMetaBucket, formatConfigFile) + if err = formatLock.GetLock(globalHealingTimeout); err != nil { + return err + } + defer formatLock.Unlock() + + // Attempt to load all `format.json` from all disks. + var sErrs []error + formats, sErrs := loadFormatXLAll(storageDisks) + for i, sErr := range sErrs { + if _, ok := formatCriticalErrors[sErr]; ok { + return fmt.Errorf("Disk %s: %s", globalEndpoints[i], sErr) + } + } + + for index := range formats { + // If the XL sets do not match, set those formats to nil, + // We do not have to update the ID on those format.json file. + if formats[index] != nil && !reflect.DeepEqual(formats[index].XL.Sets, refFormat.XL.Sets) { + formats[index] = nil + } + } + refFormat.ID, err = formatXLGetDeploymentID(refFormat, formats) + if err != nil { + return err + } + + // If ID is set, then some other node got the lock + // before this node could and generated an ID + // for the deployment. No need to generate one. + if refFormat.ID != "" { + return nil + } + + // ID is generated for the first time, + // We set the ID in all the formats and update. + refFormat.ID = mustGetUUID() + for _, format := range formats { + if format != nil { + format.ID = refFormat.ID + } + } + // Deployment ID needs to be set on all the disks. + // Save `format.json` across all disks. + return saveFormatXLAll(ctx, storageDisks, formats) + +} + +// Update only the valid local disks which have not been updated before. +func formatXLFixLocalDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) error { + // If this server was down when the deploymentID was updated + // then we make sure that we update the local disks with the deploymentID. + for index, storageDisk := range storageDisks { + if globalEndpoints[index].IsLocal && storageDisk != nil && storageDisk.IsOnline() { + format, err := loadFormatXL(storageDisk) + if err != nil { + // Disk can be offline etc. + // ignore the errors seen here. + continue + } + if format.ID != "" { + continue + } + if !reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) { + continue + } + format.ID = refFormat.ID + if err := saveFormatXL(storageDisk, format); err != nil { + logger.LogIf(ctx, err) + return fmt.Errorf("Unable to save format.json, %s", err) + } + } + } + return nil +} + // Get backend XL format in quorum `format.json`. func getFormatXLInQuorum(formats []*formatXLV3) (*formatXLV3, error) { formatHashes := make([]string, len(formats)) diff --git a/cmd/format-xl_test.go b/cmd/format-xl_test.go index ac19eb7b2..d8f1aa557 100644 --- a/cmd/format-xl_test.go +++ b/cmd/format-xl_test.go @@ -472,6 +472,61 @@ func TestGetFormatXLInQuorumCheck(t *testing.T) { } } +// Tests formatXLGetDeploymentID() +func TestGetXLID(t *testing.T) { + setCount := 2 + disksPerSet := 8 + + format := newFormatXLV3(setCount, disksPerSet) + formats := make([]*formatXLV3, 16) + + for i := 0; i < setCount; i++ { + for j := 0; j < disksPerSet; j++ { + newFormat := *format + newFormat.XL.This = format.XL.Sets[i][j] + formats[i*disksPerSet+j] = &newFormat + } + } + + // Return a format from list of formats in quorum. + quorumFormat, err := getFormatXLInQuorum(formats) + if err != nil { + t.Fatal(err) + } + + // Check if the reference format and input formats are same. + var id string + if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil { + t.Fatal(err) + } + + if id == "" { + t.Fatal("ID cannot be empty.") + } + + formats[0] = nil + if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil { + t.Fatal(err) + } + if id == "" { + t.Fatal("ID cannot be empty.") + } + + formats[1].XL.Sets[0][0] = "bad-uuid" + if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil { + t.Fatal(err) + } + + if id == "" { + t.Fatal("ID cannot be empty.") + } + + formats[2].ID = "bad-id" + if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat { + t.Fatal("Unexpected Success") + } +} + // Initialize new format sets. func TestNewFormatSets(t *testing.T) { setCount := 2 diff --git a/cmd/logger/logger.go b/cmd/logger/logger.go index 3b1603222..53b218b7c 100644 --- a/cmd/logger/logger.go +++ b/cmd/logger/logger.go @@ -122,14 +122,15 @@ type api struct { } type logEntry struct { - Level string `json:"level"` - Time string `json:"time"` - API *api `json:"api,omitempty"` - RemoteHost string `json:"remotehost,omitempty"` - RequestID string `json:"requestID,omitempty"` - UserAgent string `json:"userAgent,omitempty"` - Message string `json:"message,omitempty"` - Trace *traceEntry `json:"error,omitempty"` + DeploymentID string `json:"deploymentid,omitempty"` + Level string `json:"level"` + Time string `json:"time"` + API *api `json:"api,omitempty"` + RemoteHost string `json:"remotehost,omitempty"` + RequestID string `json:"requestID,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + Message string `json:"message,omitempty"` + Trace *traceEntry `json:"error,omitempty"` } // quiet: Hide startup messages if enabled @@ -138,8 +139,15 @@ var ( quiet, jsonFlag bool // Custom function to format error errorFmtFunc func(string, error, bool) string + + deploymentID string ) +// SetDeploymentID - Used to set the deployment ID, in XL and FS mode +func SetDeploymentID(id string) { + deploymentID = id +} + // EnableQuiet - turns quiet option on. func EnableQuiet() { quiet = true diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 16440c09f..eb6b8fa81 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -186,6 +186,23 @@ func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, dr return nil, err } + // Get the deploymentID if set. + format.ID, err = formatXLGetDeploymentID(format, formatConfigs) + if err != nil { + return nil, err + } + + if format.ID == "" { + if err = formatXLFixDeploymentID(context.Background(), storageDisks, format); err != nil { + return nil, err + } + } + + logger.SetDeploymentID(format.ID) + + if err = formatXLFixLocalDeploymentID(context.Background(), storageDisks, format); err != nil { + return nil, err + } return format, nil }