From 0513b3ed07def3e49d2d9df0a9579f206177c770 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Mon, 29 Aug 2016 03:31:59 +0100 Subject: [PATCH] Add Heal Disk Metadata RPC API + tests (#2556) --- cmd/control-heal-main.go | 31 ++++++++---- cmd/controller-handlers.go | 12 +++++ cmd/controller-handlers_test.go | 63 ++++++++++++++++++++++++ cmd/fs-v1.go | 5 ++ cmd/object-interface.go | 1 + cmd/test-utils_test.go | 37 ++++++++++++++ cmd/xl-v1.go | 85 +++++++++++++++++++-------------- 7 files changed, 189 insertions(+), 45 deletions(-) create mode 100644 cmd/controller-handlers_test.go diff --git a/cmd/control-heal-main.go b/cmd/control-heal-main.go index 740c77220..605ba3daa 100644 --- a/cmd/control-heal-main.go +++ b/cmd/control-heal-main.go @@ -47,8 +47,17 @@ EAMPLES: `, } +func checkHealControlSyntax(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + cli.ShowCommandHelpAndExit(ctx, "heal", 1) + } +} + // "minio control heal" entry point. func healControl(ctx *cli.Context) { + + checkHealControlSyntax(ctx) + // Parse bucket and object from url.URL.Path parseBucketObject := func(path string) (bucketName string, objectName string) { splits := strings.SplitN(path, string(slashSeparator), 3) @@ -67,18 +76,9 @@ func healControl(ctx *cli.Context) { return bucketName, objectName } - if len(ctx.Args()) != 1 { - cli.ShowCommandHelpAndExit(ctx, "heal", 1) - } - parsedURL, err := url.Parse(ctx.Args()[0]) fatalIf(err, "Unable to parse URL") - bucketName, objectName := parseBucketObject(parsedURL.Path) - if bucketName == "" { - cli.ShowCommandHelpAndExit(ctx, "heal", 1) - } - authCfg := &authConfig{ accessKey: serverConfig.GetCredential().AccessKeyID, secretKey: serverConfig.GetCredential().SecretAccessKey, @@ -88,6 +88,19 @@ func healControl(ctx *cli.Context) { } client := newAuthClient(authCfg) + // Always try to fix disk metadata + fmt.Print("Checking and healing disk metadata..") + args := &GenericArgs{} + reply := &GenericReply{} + err = client.Call("Controller.HealDiskMetadataHandler", args, reply) + fatalIf(err, "Unable to heal disk metadata.") + fmt.Println(" ok") + + bucketName, objectName := parseBucketObject(parsedURL.Path) + if bucketName == "" { + return + } + // If object does not have trailing "/" then it's an object, hence heal it. if objectName != "" && !strings.HasSuffix(objectName, slashSeparator) { fmt.Printf("Healing : /%s/%s\n", bucketName, objectName) diff --git a/cmd/controller-handlers.go b/cmd/controller-handlers.go index eaf5458a8..7757353f3 100644 --- a/cmd/controller-handlers.go +++ b/cmd/controller-handlers.go @@ -103,6 +103,18 @@ func (c *controllerAPIHandlers) HealObjectHandler(args *HealObjectArgs, reply *G return objAPI.HealObject(args.Bucket, args.Object) } +// HealObject - heal the object. +func (c *controllerAPIHandlers) HealDiskMetadataHandler(args *GenericArgs, reply *GenericReply) error { + objAPI := c.ObjectAPI() + if objAPI == nil { + return errVolumeBusy + } + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + return objAPI.HealDiskMetadata() +} + // ShutdownArgs - argument for Shutdown RPC. type ShutdownArgs struct { // Authentication token generated by Login. diff --git a/cmd/controller-handlers_test.go b/cmd/controller-handlers_test.go new file mode 100644 index 000000000..321b0c45b --- /dev/null +++ b/cmd/controller-handlers_test.go @@ -0,0 +1,63 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + // "net/rpc" + "testing" +) + +// Wrapper for calling heal disk metadata rpc Handler +func TestControllerHandlerHealDiskMetadata(t *testing.T) { + ExecObjectLayerTest(t, testHealDiskMetadataControllerHandler) +} + +// testHealDiskMetadataControllerHandler - Test Heal Disk Metadata handler +func testHealDiskMetadataControllerHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + // Register the API end points with XL/FS object layer. + serverAddress, random, err := initTestControllerRPCEndPoint(obj) + if err != nil { + t.Fatal(err) + } + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Init Test config failed") + } + // remove the root folder after the test ends. + defer removeAll(rootPath) + + authCfg := &authConfig{ + accessKey: serverConfig.GetCredential().AccessKeyID, + secretKey: serverConfig.GetCredential().SecretAccessKey, + address: serverAddress, + path: "/controller" + random, + loginMethod: "Controller.LoginHandler", + } + client := newAuthClient(authCfg) + + args := &GenericArgs{} + reply := &GenericReply{} + err = client.Call("Controller.HealDiskMetadataHandler", args, reply) + if instanceType == "FS" && err == nil { + t.Errorf("Test should fail with FS") + } + if instanceType == "XL" && err != nil { + t.Errorf("Test should succeed with XL %s", err.Error()) + } +} diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 9ca856b8e..9c2f858b6 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -659,3 +659,8 @@ func (fs fsObjects) HealObject(bucket, object string) error { func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { return ListObjectsInfo{}, NotImplemented{} } + +// HealDiskMetadata -- heal disk metadata, not supported in FS +func (fs fsObjects) HealDiskMetadata() error { + return NotImplemented{} +} diff --git a/cmd/object-interface.go b/cmd/object-interface.go index 8b868c9cb..8c944865a 100644 --- a/cmd/object-interface.go +++ b/cmd/object-interface.go @@ -22,6 +22,7 @@ import "io" type ObjectLayer interface { // Storage operations. Shutdown() error + HealDiskMetadata() error StorageInfo() StorageInfo // Bucket operations. diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 0437d2bba..d34baaf9b 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -26,8 +26,10 @@ import ( "io" "io/ioutil" "math/rand" + "net" "net/http" "net/http/httptest" + "net/rpc" "net/url" "os" "regexp" @@ -931,6 +933,7 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand return muxRouter } +// Initialize Web RPC Handlers for testing func initTestWebRPCEndPoint(objLayer ObjectLayer) http.Handler { // Initialize Web. webHandlers := &webAPIHandlers{ @@ -942,3 +945,37 @@ func initTestWebRPCEndPoint(objLayer ObjectLayer) http.Handler { registerWebRouter(muxRouter, webHandlers) return muxRouter } + +// Initialize Controller RPC Handlers for testing +func initTestControllerRPCEndPoint(objLayer ObjectLayer) (string, string, error) { + controllerHandlers := &controllerAPIHandlers{ + ObjectAPI: func() ObjectLayer { return objLayer }, + } + // Start configuring net/rpc server + server := rpc.NewServer() + server.RegisterName("Controller", controllerHandlers) + + listenTCP := func() (net.Listener, string, error) { + l, e := net.Listen("tcp", ":0") // any available address + if e != nil { + return nil, "", errors.New("net.Listen tcp :0, " + e.Error()) + } + return l, l.Addr().String(), nil + } + + l, serverAddr, err := listenTCP() + if err != nil { + return "", "", nil + } + go server.Accept(l) + + // net/rpc only accepts one registered path and doesn't help to unregister it, + // so we are registering a new rpc path each time this function is called + random := strconv.Itoa(rand.Int()) + server.HandleHTTP("/controller"+random, "/controller-debug"+random) + + testserver := httptest.NewServer(nil) + serverAddr = testserver.Listener.Addr().String() + + return serverAddr, random, nil +} diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index e3746186a..0c000a6d5 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -67,36 +67,7 @@ type xlObjects struct { objCacheEnabled bool } -// newXLObjects - initialize new xl object layer. -func newXLObjects(disks, ignoredDisks []string) (ObjectLayer, error) { - if disks == nil { - return nil, errInvalidArgument - } - disksSet := set.NewStringSet() - if len(ignoredDisks) > 0 { - disksSet = set.CreateStringSet(ignoredDisks...) - } - // Bootstrap disks. - storageDisks := make([]StorageAPI, len(disks)) - for index, disk := range disks { - // Check if disk is ignored. - if disksSet.Contains(disk) { - storageDisks[index] = nil - continue - } - var err error - // Intentionally ignore disk not found errors. XL is designed - // to handle these errors internally. - storageDisks[index], err = newStorageAPI(disk) - if err != nil && err != errDiskNotFound { - switch diskType := storageDisks[index].(type) { - case networkStorage: - diskType.rpcClient.Close() - } - return nil, err - } - } - +func repairDiskMetadata(storageDisks []StorageAPI) error { // Attempt to load all `format.json`. formatConfigs, sErrs := loadAllFormats(storageDisks) @@ -104,35 +75,70 @@ func newXLObjects(disks, ignoredDisks []string) (ObjectLayer, error) { // if (no quorum) return error // if (disks not recognized) // Always error. if err := genericFormatCheck(formatConfigs, sErrs); err != nil { - return nil, err + return err } // Initialize meta volume, if volume already exists ignores it. if err := initMetaVolume(storageDisks); err != nil { - return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err) + return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err) } // Handles different cases properly. switch reduceFormatErrs(sErrs, len(storageDisks)) { case errCorruptedFormat: if err := healFormatXLCorruptedDisks(storageDisks); err != nil { - return nil, fmt.Errorf("Unable to repair corrupted format, %s", err) + return fmt.Errorf("Unable to repair corrupted format, %s", err) } case errUnformattedDisk: // All drives online but fresh, initialize format. if err := initFormatXL(storageDisks); err != nil { - return nil, fmt.Errorf("Unable to initialize format, %s", err) + return fmt.Errorf("Unable to initialize format, %s", err) } case errSomeDiskUnformatted: // All drives online but some report missing format.json. if err := healFormatXLFreshDisks(storageDisks); err != nil { // There was an unexpected unrecoverable error during healing. - return nil, fmt.Errorf("Unable to heal backend %s", err) + return fmt.Errorf("Unable to heal backend %s", err) } case errSomeDiskOffline: // FIXME: in future. - return nil, fmt.Errorf("Unable to initialize format %s and %s", errSomeDiskOffline, errSomeDiskUnformatted) + return fmt.Errorf("Unable to initialize format %s and %s", errSomeDiskOffline, errSomeDiskUnformatted) + } + return nil +} + +// newXLObjects - initialize new xl object layer. +func newXLObjects(disks, ignoredDisks []string) (ObjectLayer, error) { + if disks == nil { + return nil, errInvalidArgument } + disksSet := set.NewStringSet() + if len(ignoredDisks) > 0 { + disksSet = set.CreateStringSet(ignoredDisks...) + } + // Bootstrap disks. + storageDisks := make([]StorageAPI, len(disks)) + for index, disk := range disks { + // Check if disk is ignored. + if disksSet.Contains(disk) { + storageDisks[index] = nil + continue + } + var err error + // Intentionally ignore disk not found errors. XL is designed + // to handle these errors internally. + storageDisks[index], err = newStorageAPI(disk) + if err != nil && err != errDiskNotFound { + switch diskType := storageDisks[index].(type) { + case networkStorage: + diskType.rpcClient.Close() + } + return nil, err + } + } + + // Fix format files in case of fresh or corrupted disks + repairDiskMetadata(storageDisks) // Runs house keeping code, like t, cleaning up tmp files etc. if err := xlHouseKeeping(storageDisks); err != nil { @@ -180,6 +186,13 @@ func (xl xlObjects) Shutdown() error { return nil } +// HealDiskMetadata function for object storage interface. +func (xl xlObjects) HealDiskMetadata() error { + nsMutex.Lock(minioMetaBucket, formatConfigFile) + defer nsMutex.Unlock(minioMetaBucket, formatConfigFile) + return repairDiskMetadata(xl.storageDisks) +} + // byDiskTotal is a collection satisfying sort.Interface. type byDiskTotal []disk.Info