From 76f4f20609ae1c92a5e6bb75fc4a5e8ddac9e9ff Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 5 May 2017 08:49:09 -0700 Subject: [PATCH] fs: Migrate object metadata to objects directory. (#4195) Fixes #3352 --- cmd/format-config-v1.go | 41 ++++ cmd/fs-v1-helpers.go | 43 +++- cmd/fs-v1-helpers_test.go | 43 +++- cmd/fs-v1-metadata.go | 177 +++++++++++++--- cmd/fs-v1-metadata_test.go | 4 +- cmd/fs-v1-multipart.go | 2 +- cmd/fs-v1.go | 155 +++++++++++--- cmd/fs-v1_test.go | 378 +++++++++++++++++++++++++++++++++- cmd/object-api-common.go | 3 + cmd/posix-list-dir-others.go | 2 + cmd/posix.go | 2 +- cmd/service.go | 2 +- docs/shared-backend/DESIGN.md | 4 +- 13 files changed, 769 insertions(+), 87 deletions(-) diff --git a/cmd/format-config-v1.go b/cmd/format-config-v1.go index c6c5d687e..4ea2bb24e 100644 --- a/cmd/format-config-v1.go +++ b/cmd/format-config-v1.go @@ -20,8 +20,12 @@ import ( "encoding/json" "errors" "fmt" + "io" + "io/ioutil" "reflect" "sync" + + "github.com/minio/minio/pkg/lock" ) // fsFormat - structure holding 'fs' format. @@ -47,6 +51,43 @@ type formatConfigV1 struct { XL *xlFormat `json:"xl,omitempty"` // XL field holds xl format. } +func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) { + // Serialize to prepare to write to disk. + var fbytes []byte + fbytes, err = json.Marshal(f) + if err != nil { + return 0, traceError(err) + } + if err = lk.Truncate(0); err != nil { + return 0, traceError(err) + } + _, err = lk.Write(fbytes) + if err != nil { + return 0, traceError(err) + } + return int64(len(fbytes)), nil +} + +func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) { + var fbytes []byte + fi, err := lk.Stat() + if err != nil { + return 0, traceError(err) + } + fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size())) + if err != nil { + return 0, traceError(err) + } + if len(fbytes) == 0 { + return 0, traceError(io.EOF) + } + // Decode `format.json`. + if err = json.Unmarshal(fbytes, f); err != nil { + return 0, traceError(err) + } + return int64(len(fbytes)), nil +} + /* All disks online diff --git a/cmd/fs-v1-helpers.go b/cmd/fs-v1-helpers.go index dd71679fc..13dc04616 100644 --- a/cmd/fs-v1-helpers.go +++ b/cmd/fs-v1-helpers.go @@ -230,7 +230,7 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) { // Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer. func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) { - if filePath == "" || reader == nil || buf == nil { + if filePath == "" || reader == nil { return 0, traceError(errInvalidArgument) } @@ -263,11 +263,18 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6 } } - bytesWritten, err := io.CopyBuffer(writer, reader, buf) - if err != nil { - return 0, traceError(err) + var bytesWritten int64 + if buf != nil { + bytesWritten, err = io.CopyBuffer(writer, reader, buf) + if err != nil { + return 0, traceError(err) + } + } else { + bytesWritten, err = io.Copy(writer, reader) + if err != nil { + return 0, traceError(err) + } } - return bytesWritten, nil } @@ -276,6 +283,12 @@ func fsRemoveUploadIDPath(basePath, uploadIDPath string) error { if basePath == "" || uploadIDPath == "" { return traceError(errInvalidArgument) } + if err := checkPathLength(basePath); err != nil { + return traceError(err) + } + if err := checkPathLength(uploadIDPath); err != nil { + return traceError(err) + } // List all the entries in uploadID. entries, err := readDir(uploadIDPath) @@ -319,6 +332,26 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) { // Renames source path to destination path, creates all the // missing parents if they don't exist. func fsRenameFile(sourcePath, destPath string) error { + if err := checkPathLength(sourcePath); err != nil { + return traceError(err) + } + if err := checkPathLength(destPath); err != nil { + return traceError(err) + } + // Verify if source path exists. + if _, err := os.Stat(preparePath(sourcePath)); err != nil { + if os.IsNotExist(err) { + return traceError(errFileNotFound) + } else if os.IsPermission(err) { + return traceError(errFileAccessDenied) + } else if isSysErrPathNotFound(err) { + return traceError(errFileNotFound) + } else if isSysErrNotDir(err) { + // File path cannot be verified since one of the parents is a file. + return traceError(errFileAccessDenied) + } + return traceError(err) + } if err := mkdirAll(pathutil.Dir(destPath), 0777); err != nil { return traceError(err) } diff --git a/cmd/fs-v1-helpers_test.go b/cmd/fs-v1-helpers_test.go index b32b6b169..49003f686 100644 --- a/cmd/fs-v1-helpers_test.go +++ b/cmd/fs-v1-helpers_test.go @@ -26,6 +26,31 @@ import ( "github.com/minio/minio/pkg/lock" ) +func TestFSRenameFile(t *testing.T) { + // create posix test setup + _, path, err := newPosixTestSetup() + if err != nil { + t.Fatalf("Unable to create posix test setup, %s", err) + } + defer removeAll(path) + + if err = fsMkdir(pathJoin(path, "testvolume1")); err != nil { + t.Fatal(err) + } + if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil { + t.Fatal(err) + } + if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNotFound { + t.Fatal(err) + } + if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNameTooLong { + t.Fatal("Unexpected error", err) + } + if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong { + t.Fatal("Unexpected error", err) + } +} + func TestFSStats(t *testing.T) { // create posix test setup _, path, err := newPosixTestSetup() @@ -48,9 +73,8 @@ func TestFSStats(t *testing.T) { t.Fatalf("Unable to create volume, %s", err) } - var buf = make([]byte, 4096) var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil { + if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. @@ -60,7 +84,7 @@ func TestFSStats(t *testing.T) { t.Fatal("Unexpected error", err) } - if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, buf, reader.Size()); err != nil { + if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. @@ -174,9 +198,8 @@ func TestFSCreateAndOpen(t *testing.T) { t.Fatal("Unexpected error", err) } - var buf = make([]byte, 4096) var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil { + if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. @@ -204,7 +227,7 @@ func TestFSCreateAndOpen(t *testing.T) { } for i, testCase := range testCases { - _, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, buf, reader.Size()) + _, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0) if errorCause(err) != testCase.expectedErr { t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) } @@ -297,15 +320,14 @@ func TestFSRemoves(t *testing.T) { t.Fatalf("Unable to create directory, %s", err) } - var buf = make([]byte, 4096) var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil { + if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. reader.Seek(0, 0) - if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, buf, reader.Size()); err != nil { + if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } // Seek back. @@ -417,9 +439,8 @@ func TestFSRemoveMeta(t *testing.T) { filePath := pathJoin(fsPath, "success-vol", "success-file") - var buf = make([]byte, 4096) var reader = bytes.NewReader([]byte("Hello, world")) - if _, err = fsCreateFile(filePath, reader, buf, reader.Size()); err != nil { + if _, err = fsCreateFile(filePath, reader, nil, 0); err != nil { t.Fatalf("Unable to create file, %s", err) } diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index 260cf13b5..4df6a8516 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -18,6 +18,8 @@ package cmd import ( "encoding/json" + "errors" + "fmt" "io" "io/ioutil" "os" @@ -25,6 +27,7 @@ import ( "sort" "strings" + "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/pkg/lock" "github.com/minio/minio/pkg/mimedb" "github.com/tidwall/gjson" @@ -225,9 +228,20 @@ const ( // FS backend meta format. fsMetaFormat = "fs" + // FS backend format version. + fsFormatVersion = fsFormatV2 + // Add more constants here. ) +// FS format version strings. +const ( + fsFormatV1 = "1" // Previous format. + fsFormatV2 = "2" // Current format. + // Proceed to add "3" when we + // change the backend format in future. +) + // newFSMetaV1 - initializes new fsMetaV1. func newFSMetaV1() (fsMeta fsMetaV1) { fsMeta = fsMetaV1{} @@ -237,58 +251,167 @@ func newFSMetaV1() (fsMeta fsMetaV1) { return fsMeta } -// newFSFormatV1 - initializes new formatConfigV1 with FS format info. -func newFSFormatV1() (format *formatConfigV1) { +// newFSFormatV2 - initializes new formatConfigV1 with FS format version 2. +func newFSFormatV2() (format *formatConfigV1) { return &formatConfigV1{ Version: "1", Format: "fs", FS: &fsFormat{ - Version: "1", + Version: fsFormatV2, }, } } -// loads format.json from minioMetaBucket if it exists. -func loadFormatFS(fsPath string) (*formatConfigV1, error) { - rlk, err := lock.RLockedOpenFile(pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile)) - if err != nil { - if os.IsNotExist(err) { - return nil, errUnformattedDisk +// Checks if input format is version 1 and 2. +func isFSValidFormat(formatCfg *formatConfigV1) bool { + // Supported format versions. + var supportedFormatVersions = []string{ + fsFormatV1, + fsFormatV2, + // New supported versions here. + } + + // Check for supported format versions. + for _, version := range supportedFormatVersions { + if formatCfg.FS.Version == version { + return true } - return nil, err } - defer rlk.Close() + return false +} - formatBytes, err := ioutil.ReadAll(rlk) - if err != nil { - return nil, err +// errFSFormatOld- old fs format. +var errFSFormatOld = errors.New("old FS format found") + +// Checks if the loaded `format.json` is valid and +// is expected to be of the requested version. +func checkFormatFS(format *formatConfigV1, formatVersion string) error { + if format == nil { + return errUnexpected + } + + // Validate if we have the same format. + if format.Format != "fs" { + return fmt.Errorf("Unable to recognize backend format, Disk is not in FS format. %s", format.Format) } - format := &formatConfigV1{} - if err = json.Unmarshal(formatBytes, format); err != nil { - return nil, err + // Check if format is currently supported. + if !isFSValidFormat(format) { + return errCorruptedFormat } - return format, nil + // Check for format version is current. + if format.FS.Version != formatVersion { + return errFSFormatOld + } + + return nil } -// writes FS format (format.json) into minioMetaBucket. -func saveFormatFS(formatPath string, fsFormat *formatConfigV1) error { - metadataBytes, err := json.Marshal(fsFormat) - if err != nil { +// This is just kept as reference, there is no sanity +// check for FS format in version "1". +func checkFormatSanityFSV1(fsPath string) error { + return nil +} + +// Check for sanity of FS format in version "2". +func checkFormatSanityFSV2(fsPath string) error { + buckets, err := readDir(pathJoin(fsPath, minioMetaBucket, bucketConfigPrefix)) + if err != nil && err != errFileNotFound { return err } + // Attempt to validate all the buckets have a sanitized backend. + for _, bucket := range buckets { + entries, rerr := readDir(pathJoin(fsPath, minioMetaBucket, bucketConfigPrefix, bucket)) + if rerr != nil { + return rerr + } + + var expectedConfigs = append(bucketMetadataConfigs, objectMetaPrefix+"/") + entriesSet := set.CreateStringSet(entries...) + expectedConfigsSet := set.CreateStringSet(expectedConfigs...) + + // Entries found shouldn't be more than total + // expected config directories, files. + if len(entriesSet) > len(expectedConfigsSet) { + return errCorruptedFormat + } + + // Look for the difference between entries and the + // expected config set, resulting entries if they + // intersect with original entries set we know + // that the backend has unexpected files. + if !entriesSet.Difference(expectedConfigsSet).IsEmpty() { + return errCorruptedFormat + } + } + return nil +} + +// Check for sanity of FS format for a given version. +func checkFormatSanityFS(fsPath string, fsFormatVersion string) (err error) { + switch fsFormatVersion { + case fsFormatV2: + err = checkFormatSanityFSV2(fsPath) + default: + err = errCorruptedFormat + } + return err +} + +// Initializes a new `format.json` if not present, validates `format.json` +// if already present and migrates to newer version if necessary. Returns +// the final format version. +func initFormatFS(fsPath, fsUUID string) (err error) { + fsFormatPath := pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile) + // fsFormatJSONFile - format.json file stored in minioMetaBucket(.minio.sys) directory. - lk, err := lock.LockedOpenFile(preparePath(formatPath), os.O_CREATE|os.O_WRONLY, 0600) + lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) if err != nil { - return err + return traceError(err) } defer lk.Close() - _, err = lk.Write(metadataBytes) - // Success. - return err + var format = &formatConfigV1{} + _, err = format.ReadFrom(lk) + // For all unexpected errors, we return. + if err != nil && errorCause(err) != io.EOF { + return traceError(fmt.Errorf("Unable to load 'format.json', %s", err)) + } + + // If we couldn't read anything, The disk is unformatted. + if errorCause(err) == io.EOF { + err = errUnformattedDisk + format = newFSFormatV2() + } else { + // Validate loaded `format.json`. + err = checkFormatFS(format, fsFormatVersion) + if err != nil && err != errFSFormatOld { + return traceError(fmt.Errorf("Unable to validate 'format.json', %s", err)) + } + } + + // Disk is in old format migrate object metadata. + if err == errFSFormatOld { + if merr := migrateFSObject(fsPath, fsUUID); merr != nil { + return merr + } + + // Initialize format v2. + format = newFSFormatV2() + } + + // Rewrite or write format.json depending on if disk + // unformatted and if format is old. + if err == errUnformattedDisk || err == errFSFormatOld { + if _, err = format.WriteTo(lk); err != nil { + return traceError(fmt.Errorf("Unable to initialize 'format.json', %s", err)) + } + } + + // Check for sanity. + return checkFormatSanityFS(fsPath, format.FS.Version) } // Return if the part info in uploadedParts and completeParts are same. diff --git a/cmd/fs-v1-metadata_test.go b/cmd/fs-v1-metadata_test.go index 79e3f35d0..620c43b5c 100644 --- a/cmd/fs-v1-metadata_test.go +++ b/cmd/fs-v1-metadata_test.go @@ -58,7 +58,7 @@ func TestReadFSMetadata(t *testing.T) { } // Construct the full path of fs.json - fsPath := pathJoin("buckets", bucketName, objectName, "fs.json") + fsPath := pathJoin(bucketMetaPrefix, bucketName, objectMetaPrefix, objectName, "fs.json") fsPath = pathJoin(fs.fsPath, minioMetaBucket, fsPath) rlk, err := fs.rwPool.Open(fsPath) @@ -95,7 +95,7 @@ func TestWriteFSMetadata(t *testing.T) { } // Construct the full path of fs.json - fsPath := pathJoin("buckets", bucketName, objectName, "fs.json") + fsPath := pathJoin(bucketMetaPrefix, bucketName, objectMetaPrefix, objectName, "fs.json") fsPath = pathJoin(fs.fsPath, minioMetaBucket, fsPath) rlk, err := fs.rwPool.Open(fsPath) diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index 104f30548..040a7a090 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -754,7 +754,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload // Wait for any competing PutObject() operation on bucket/object, since same namespace // would be acquired for `fs.json`. - fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile) + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, objectMetaPrefix, object, fsMetaJSONFile) metaFile, err := fs.rwPool.Create(fsMetaPath) if err != nil { fs.rwPool.Close(fsMetaPathMultipart) diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 440e7d830..da49092ca 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -24,6 +24,7 @@ import ( "io" "io/ioutil" "os" + "os/signal" "path/filepath" "sort" "syscall" @@ -72,15 +73,117 @@ func initMetaVolumeFS(fsPath, fsUUID string) error { } +// Migrate FS object is a place holder code for all +// FS format migrations. +func migrateFSObject(fsPath, fsUUID string) (err error) { + // Writing message here is important for servers being upgraded. + log.Println("Please do not stop the server.") + + ch := make(chan os.Signal) + defer signal.Stop(ch) + defer close(ch) + + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + go func() { + for { + _, ok := <-ch + if !ok { + break + } + log.Println("Please wait server is being upgraded..") + } + }() + + return migrateFSFormatV1ToV2(fsPath, fsUUID) +} + +// List all buckets at meta bucket prefix in `.minio.sys/buckets/` path. +// This is implemented to avoid a bug on windows with using readDir(). +func fsReaddirMetaBuckets(fsPath string) ([]string, error) { + f, err := os.Open(preparePath(pathJoin(fsPath, minioMetaBucket, bucketConfigPrefix))) + if err != nil { + if os.IsNotExist(err) { + return nil, errFileNotFound + } else if os.IsPermission(err) { + return nil, errFileAccessDenied + } + return nil, err + } + return f.Readdirnames(-1) +} + +var bucketMetadataConfigs = []string{ + bucketNotificationConfig, + bucketListenerConfig, + bucketPolicyConfig, +} + +// Attempts to migrate old object metadata files to newer format +// +// i.e +// ------------------------------------------------------- +// .minio.sys/buckets///fs.json - V1 +// ------------------------------------------------------- +// .minio.sys/buckets//objects//fs.json - V2 +// ------------------------------------------------------- +// +func migrateFSFormatV1ToV2(fsPath, fsUUID string) (err error) { + metaBucket := pathJoin(fsPath, minioMetaBucket, bucketConfigPrefix) + + var buckets []string + buckets, err = fsReaddirMetaBuckets(fsPath) + if err != nil && err != errFileNotFound { + return err + } + + // Migrate all buckets present. + for _, bucket := range buckets { + // Temporary bucket of form .UUID-bucket. + tmpBucket := fmt.Sprintf(".%s-%s", fsUUID, bucket) + + // Rename existing bucket as `.UUID-bucket`. + if err = fsRenameFile(pathJoin(metaBucket, bucket), pathJoin(metaBucket, tmpBucket)); err != nil { + return err + } + + // Create a new bucket name with name as `bucket`. + if err = fsMkdir(pathJoin(metaBucket, bucket)); err != nil { + return err + } + + /// Rename all bucket metadata files to newly created `bucket`. + for _, bucketMetaFile := range bucketMetadataConfigs { + if err = fsRenameFile(pathJoin(metaBucket, tmpBucket, bucketMetaFile), + pathJoin(metaBucket, bucket, bucketMetaFile)); err != nil { + if errorCause(err) != errFileNotFound { + return err + } + } + } + + // Finally rename the temporary bucket to `bucket/objects` directory. + if err = fsRenameFile(pathJoin(metaBucket, tmpBucket), + pathJoin(metaBucket, bucket, objectMetaPrefix)); err != nil { + if errorCause(err) != errFileNotFound { + return err + } + } + } + + log.Printf("Migrating bucket metadata format from \"%s\" to newer format \"%s\"... completed successfully.", fsFormatV1, fsFormatV2) + + // If all goes well we return success. + return nil +} + // newFSObjectLayer - initialize new fs object layer. func newFSObjectLayer(fsPath string) (ObjectLayer, error) { if fsPath == "" { return nil, errInvalidArgument } - var err error // Disallow relative paths, figure out absolute paths. - fsPath, err = filepath.Abs(fsPath) + fsPath, err := filepath.Abs(fsPath) if err != nil { return nil, err } @@ -108,26 +211,6 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) { return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err) } - // Load `format.json`. - format, err := loadFormatFS(fsPath) - if err != nil && err != errUnformattedDisk { - return nil, fmt.Errorf("Unable to load 'format.json', %s", err) - } - - // If the `format.json` doesn't exist create one. - if err == errUnformattedDisk { - fsFormatPath := pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile) - // Initialize format.json, if already exists overwrite it. - if serr := saveFormatFS(fsFormatPath, newFSFormatV1()); serr != nil { - return nil, fmt.Errorf("Unable to initialize 'format.json', %s", serr) - } - } - - // Validate if we have the same format. - if err == nil && format.Format != "fs" { - return nil, fmt.Errorf("Unable to recognize backend format, Disk is not in FS format. %s", format.Format) - } - // Initialize fs objects. fs := &fsObjects{ fsPath: fsPath, @@ -141,6 +224,17 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) { }, } + // Initialize `format.json`. + if err = initFormatFS(fsPath, fsUUID); err != nil { + return nil, err + } + + // Once initialized hold read lock for the entire operation + // of filesystem backend. + if _, err = fs.rwPool.Open(pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile)); err != nil { + return nil, err + } + // Initialize and load bucket policies. err = initBucketPolicies(fs) if err != nil { @@ -159,6 +253,9 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) { // Should be called when process shuts down. func (fs fsObjects) Shutdown() error { + // Close the format.json read lock. + fs.rwPool.Close(pathJoin(fs.fsPath, minioMetaBucket, fsFormatJSONFile)) + // Cleanup and delete tmp uuid. return fsRemoveAll(pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)) } @@ -238,7 +335,7 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) { return nil, traceError(err) } var bucketInfos []BucketInfo - entries, err := readDir(preparePath(fs.fsPath)) + entries, err := readDir(fs.fsPath) if err != nil { return nil, toObjectErr(traceError(errDiskNotFound)) } @@ -322,7 +419,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string // Check if this request is only metadata update. cpMetadataOnly := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) if cpMetadataOnly { - fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, srcBucket, srcObject, fsMetaJSONFile) + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, srcBucket, objectMetaPrefix, srcObject, fsMetaJSONFile) var wlk *lock.LockedFile wlk, err = fs.rwPool.Write(fsMetaPath) if err != nil { @@ -395,7 +492,7 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, } if bucket != minioMetaBucket { - fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile) + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, objectMetaPrefix, object, fsMetaJSONFile) _, err = fs.rwPool.Open(fsMetaPath) if err != nil && err != errFileNotFound { return toObjectErr(traceError(err), bucket, object) @@ -437,7 +534,7 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo. func (fs fsObjects) getObjectInfo(bucket, object string) (ObjectInfo, error) { fsMeta := fsMetaV1{} - fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile) + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, objectMetaPrefix, object, fsMetaJSONFile) // Read `fs.json` to perhaps contend with // parallel Put() operations. @@ -520,7 +617,7 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. var wlk *lock.LockedFile if bucket != minioMetaBucket { bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix) - fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fsMetaJSONFile) + fsMetaPath := pathJoin(bucketMetaDir, bucket, objectMetaPrefix, object, fsMetaJSONFile) wlk, err = fs.rwPool.Create(fsMetaPath) if err != nil { return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) @@ -647,7 +744,7 @@ func (fs fsObjects) DeleteObject(bucket, object string) error { } minioMetaBucketDir := pathJoin(fs.fsPath, minioMetaBucket) - fsMetaPath := pathJoin(minioMetaBucketDir, bucketMetaPrefix, bucket, object, fsMetaJSONFile) + fsMetaPath := pathJoin(minioMetaBucketDir, bucketMetaPrefix, bucket, objectMetaPrefix, object, fsMetaJSONFile) if bucket != minioMetaBucket { rwlk, lerr := fs.rwPool.Write(fsMetaPath) if lerr == nil { @@ -701,7 +798,7 @@ func (fs fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc { // getObjectETag is a helper function, which returns only the md5sum // of the file on the disk. func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) { - fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fsMetaJSONFile) + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, objectMetaPrefix, entry, fsMetaJSONFile) // Read `fs.json` to perhaps contend with // parallel Put() operations. diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 0cd0290a2..f0b68b111 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -22,6 +22,8 @@ import ( "os" "path/filepath" "testing" + + "github.com/minio/minio/pkg/lock" ) // TestNewFS - tests initialization of all input disks @@ -85,8 +87,8 @@ func TestFSShutdown(t *testing.T) { } } -// TestFSLoadFormatFS - test loadFormatFS with healty and faulty disks -func TestFSLoadFormatFS(t *testing.T) { +// Tests migrating FS format without .minio.sys/buckets. +func TestFSMigrateObjectWithoutObjects(t *testing.T) { // Prepare for testing disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) defer removeAll(disk) @@ -100,13 +102,364 @@ func TestFSLoadFormatFS(t *testing.T) { } fsFormatPath := pathJoin(disk, minioMetaBucket, fsFormatJSONFile) - if err := saveFormatFS(preparePath(fsFormatPath), newFSFormatV1()); err != nil { + formatCfg := &formatConfigV1{ + Version: "1", + Format: "fs", + FS: &fsFormat{ + Version: "1", + }, + } + + lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + _, err = formatCfg.WriteTo(lk) + lk.Close() + if err != nil { t.Fatal("Should not fail here", err) } - _, err := loadFormatFS(disk) + + if err = initFormatFS(disk, uuid); err != nil { + t.Fatal("Should not fail with unexpected", err) + } + + formatCfg = &formatConfigV1{} + lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDONLY, 0600) if err != nil { + t.Fatal(err) + } + _, err = formatCfg.ReadFrom(lk) + lk.Close() + if err != nil { + t.Fatal("Should not fail here", err) + } + if formatCfg.FS.Version != fsFormatV2 { + t.Fatalf("Unexpected version detected expected \"%s\", got %s", fsFormatV2, formatCfg.FS.Version) + } +} + +// Tests migrating FS format without .minio.sys/buckets. +func TestFSMigrateObjectWithErr(t *testing.T) { + // Prepare for testing + disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) + defer removeAll(disk) + + // Assign a new UUID. + uuid := mustGetUUID() + + // Initialize meta volume, if volume already exists ignores it. + if err := initMetaVolumeFS(disk, uuid); err != nil { + t.Fatal(err) + } + + fsFormatPath := pathJoin(disk, minioMetaBucket, fsFormatJSONFile) + formatCfg := &formatConfigV1{ + Version: "1", + Format: "fs", + FS: &fsFormat{ + Version: "10", + }, + } + + lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + _, err = formatCfg.WriteTo(lk) + lk.Close() + if err != nil { + t.Fatal("Should not fail here", err) + } + + if err = initFormatFS(disk, uuid); err != nil { + if errorCause(err).Error() != + "Unable to validate 'format.json', corrupted backend format" { + t.Fatal("Should not fail with unexpected", err) + } + } + + fsFormatPath = pathJoin(disk, minioMetaBucket, fsFormatJSONFile) + formatCfg = &formatConfigV1{ + Version: "1", + Format: "garbage", + FS: &fsFormat{ + Version: "1", + }, + } + + lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + _, err = formatCfg.WriteTo(lk) + lk.Close() + if err != nil { + t.Fatal("Should not fail here", err) + } + + if err = initFormatFS(disk, uuid); err != nil { + if errorCause(err).Error() != + "Unable to validate 'format.json', Unable to recognize backend format, Disk is not in FS format. garbage" { + t.Fatal("Should not fail with unexpected", err) + } + } + +} + +// Tests migrating FS format with .minio.sys/buckets filled with +// object metadata. +func TestFSMigrateObjectWithObjects(t *testing.T) { + // Prepare for testing + disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) + defer removeAll(disk) + + // Assign a new UUID. + uuid := mustGetUUID() + + // Initialize meta volume, if volume already exists ignores it. + if err := initMetaVolumeFS(disk, uuid); err != nil { + t.Fatal(err) + } + + fsFormatPath := pathJoin(disk, minioMetaBucket, fsFormatJSONFile) + formatCfg := &formatConfigV1{ + Version: "1", + Format: "fs", + FS: &fsFormat{ + Version: "1", + }, + } + lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + _, err = formatCfg.WriteTo(lk) + lk.Close() + if err != nil { + t.Fatal("Should not fail here", err) + } + + // Construct the full path of fs.json + fsPath1 := pathJoin(bucketMetaPrefix, "testvolume1", "my-object1", fsMetaJSONFile) + fsPath1 = pathJoin(disk, minioMetaBucket, fsPath1) + + fsMetaJSON := `{"version":"1.0.0","format":"fs","minio":{"release":"DEVELOPMENT.2017-03-27T02-26-33Z"},"meta":{"md5Sum":"467886be95c8ecfd71a2900e3f461b4f"}` + if _, err = fsCreateFile(fsPath1, bytes.NewReader([]byte(fsMetaJSON)), nil, 0); err != nil { + t.Fatal(err) + } + + // Construct the full path of fs.json + fsPath2 := pathJoin(bucketMetaPrefix, "testvolume2", "my-object2", fsMetaJSONFile) + fsPath2 = pathJoin(disk, minioMetaBucket, fsPath2) + + fsMetaJSON = `{"version":"1.0.0","format":"fs","minio":{"release":"DEVELOPMENT.2017-03-27T02-26-33Z"},"meta":{"md5Sum":"467886be95c8ecfd71a2900eff461b4d"}` + if _, err = fsCreateFile(fsPath2, bytes.NewReader([]byte(fsMetaJSON)), nil, 0); err != nil { + t.Fatal(err) + } + + // Construct the full path of policy.json + ppath := pathJoin(bucketMetaPrefix, "testvolume2", bucketPolicyConfig) + ppath = pathJoin(disk, minioMetaBucket, ppath) + + policyJSON := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket/*"],"Sid":""}]}` + if _, err = fsCreateFile(ppath, bytes.NewReader([]byte(policyJSON)), nil, 0); err != nil { + t.Fatal(err) + } + + if err = initFormatFS(disk, mustGetUUID()); err != nil { + t.Fatal("Should not fail here", err) + } + + fsPath2 = pathJoin(bucketMetaPrefix, "testvolume2", objectMetaPrefix, "my-object2", fsMetaJSONFile) + fsPath2 = pathJoin(disk, minioMetaBucket, fsPath2) + fi, err := fsStatFile(fsPath2) + if err != nil { + t.Fatal("Path should exist and accessible after migration", err) + } + if fi.IsDir() { + t.Fatalf("Unexpected path %s should be a file", fsPath2) + } + + formatCfg = &formatConfigV1{} + lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDONLY, 0600) + if err != nil { + t.Fatal(err) + } + _, err = formatCfg.ReadFrom(lk) + lk.Close() + if err != nil { + t.Fatal("Should not fail here", err) + } + if formatCfg.FS.Version != fsFormatV2 { + t.Fatalf("Unexpected version detected expected \"%s\", got %s", fsFormatV2, formatCfg.FS.Version) + } + + ppath = pathJoin(bucketMetaPrefix, "testvolume2", "acl.json") + ppath = pathJoin(disk, minioMetaBucket, ppath) + + if _, err = fsCreateFile(ppath, bytes.NewReader([]byte("")), nil, 0); err != nil { + t.Fatal(err) + } + + if err = initFormatFS(disk, mustGetUUID()); errorCause(err) != errCorruptedFormat { t.Fatal("Should not fail here", err) } +} + +// TestFSCheckFormatFSErr - test loadFormatFS loading older format. +func TestFSCheckFormatFSErr(t *testing.T) { + // Prepare for testing + disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) + defer removeAll(disk) + + // Assign a new UUID. + uuid := mustGetUUID() + + // Initialize meta volume, if volume already exists ignores it. + if err := initMetaVolumeFS(disk, uuid); err != nil { + t.Fatal(err) + } + + fsFormatPath := pathJoin(disk, minioMetaBucket, fsFormatJSONFile) + formatCfg := &formatConfigV1{ + Version: "1", + Format: "fs", + FS: &fsFormat{ + Version: "1", + }, + } + + lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + + _, err = formatCfg.WriteTo(lk) + lk.Close() + if err != nil { + t.Fatal(err) + } + + formatCfg = &formatConfigV1{} + lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + _, err = formatCfg.ReadFrom(lk) + lk.Close() + if err != nil { + t.Fatal(err) + } + + if err = checkFormatFS(formatCfg, fsFormatVersion); errorCause(err) != errFSFormatOld { + t.Fatal("Should not fail with unexpected", err) + } + + formatCfg = &formatConfigV1{ + Version: "1", + Format: "fs", + FS: &fsFormat{ + Version: "10", + }, + } + + lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + + _, err = formatCfg.WriteTo(lk) + lk.Close() + if err != nil { + t.Fatal(err) + } + + if err = checkFormatFS(formatCfg, fsFormatVersion); errorCause(err) != errCorruptedFormat { + t.Fatal("Should not fail with unexpected", err) + } + + formatCfg = &formatConfigV1{ + Version: "1", + Format: "garbage", + FS: &fsFormat{ + Version: "1", + }, + } + + lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + + _, err = formatCfg.WriteTo(lk) + lk.Close() + if err != nil { + t.Fatal(err) + } + + if err = checkFormatFS(formatCfg, fsFormatVersion); err != nil { + if errorCause(err).Error() != "Unable to recognize backend format, Disk is not in FS format. garbage" { + t.Fatal("Should not fail with unexpected", err) + } + } + + if err = checkFormatFS(nil, fsFormatVersion); errorCause(err) != errUnexpected { + t.Fatal("Should fail with errUnexpected, but found", err) + } + + formatCfg = &formatConfigV1{ + Version: "1", + Format: "fs", + FS: &fsFormat{ + Version: "2", + }, + } + + lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + + _, err = formatCfg.WriteTo(lk) + lk.Close() + if err != nil { + t.Fatal(err) + } + + // Should not fail. + if err = checkFormatFS(formatCfg, fsFormatVersion); err != nil { + t.Fatal(err) + } +} + +// TestFSCheckFormatFS - test loadFormatFS with healty and faulty disks +func TestFSCheckFormatFS(t *testing.T) { + // Prepare for testing + disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) + defer removeAll(disk) + + // Assign a new UUID. + uuid := mustGetUUID() + + // Initialize meta volume, if volume already exists ignores it. + if err := initMetaVolumeFS(disk, uuid); err != nil { + t.Fatal(err) + } + + fsFormatPath := pathJoin(disk, minioMetaBucket, fsFormatJSONFile) + lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + + format := newFSFormatV2() + _, err = format.WriteTo(lk) + lk.Close() + if err != nil { + t.Fatal(err) + } + // Loading corrupted format file file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) if err != nil { @@ -114,15 +467,24 @@ func TestFSLoadFormatFS(t *testing.T) { } file.Write([]byte{'b'}) file.Close() - _, err = loadFormatFS(disk) + + lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + t.Fatal(err) + } + + format = &formatConfigV1{} + _, err = format.ReadFrom(lk) + lk.Close() if err == nil { t.Fatal("Should return an error here") } + // Loading format file from disk not found. removeAll(disk) - _, err = loadFormatFS(disk) - if err != nil && err != errUnformattedDisk { - t.Fatal("Should return unformatted disk, but got", err) + _, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDONLY, 0600) + if err != nil && !os.IsNotExist(err) { + t.Fatal("Should return 'format.json' does not exist, but got", err) } } diff --git a/cmd/object-api-common.go b/cmd/object-api-common.go index c15440362..b2056f01d 100644 --- a/cmd/object-api-common.go +++ b/cmd/object-api-common.go @@ -32,6 +32,9 @@ const ( // Buckets meta prefix. bucketMetaPrefix = "buckets" + // Objects meta prefix. + objectMetaPrefix = "objects" + // Md5Sum of empty string. emptyStrMd5Sum = "d41d8cd98f00b204e9800998ecf8427e" ) diff --git a/cmd/posix-list-dir-others.go b/cmd/posix-list-dir-others.go index 75b53d8ca..2533aa064 100644 --- a/cmd/posix-list-dir-others.go +++ b/cmd/posix-list-dir-others.go @@ -32,6 +32,8 @@ func readDir(dirPath string) (entries []string, err error) { // File is really not found. if os.IsNotExist(err) { return nil, errFileNotFound + } else if os.IsPermission(err) { + return nil, errFileAccessDenied } // File path cannot be verified since one of the parents is a file. diff --git a/cmd/posix.go b/cmd/posix.go index 7e103a983..84fc25c68 100644 --- a/cmd/posix.go +++ b/cmd/posix.go @@ -291,7 +291,7 @@ func (s *posix) ListVols() (volsInfo []VolInfo, err error) { return nil, err } - volsInfo, err = listVols(preparePath(s.diskPath)) + volsInfo, err = listVols(s.diskPath) if err != nil { return nil, err } diff --git a/cmd/service.go b/cmd/service.go index d0af1c0d4..e23a094f2 100644 --- a/cmd/service.go +++ b/cmd/service.go @@ -82,7 +82,7 @@ func (m *ServerMux) handleServiceSignals() error { // Wait for SIGTERM in a go-routine. trapCh := signalTrap(os.Interrupt, syscall.SIGTERM) - go func(<-chan bool) { + go func(trapCh <-chan bool) { <-trapCh globalServiceSignalCh <- serviceStop }(trapCh) diff --git a/docs/shared-backend/DESIGN.md b/docs/shared-backend/DESIGN.md index d22d3520e..86a2aea1f 100644 --- a/docs/shared-backend/DESIGN.md +++ b/docs/shared-backend/DESIGN.md @@ -81,7 +81,7 @@ An example here shows how the contention is handled with GetObject(). GetObject() holds a read lock on `fs.json`. ```go - fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile) + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, objectMetaPrefix, object, fsMetaJSONFile) rlk, err := fs.rwPool.Open(fsMetaPath) if err != nil { return toObjectErr(traceError(err), bucket, object) @@ -98,7 +98,7 @@ GetObject() holds a read lock on `fs.json`. A concurrent PutObject is requested on the same object, PutObject() attempts a write lock on `fs.json`. ```go - fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile) + fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, objectMetaPrefix, object, fsMetaJSONFile) wlk, err := fs.rwPool.Create(fsMetaPath) if err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object)