From 586058f0790ed7eddbc526393670f2008ed062db Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Mon, 23 Jan 2017 14:02:55 +0530 Subject: [PATCH] Implement mgmt REST APIs to heal storage format. (#3604) * Implement heal format REST API handler * Implement admin peer rpc handler to re-initialize storage * Implement HealFormat API in pkg/madmin * Update pkg/madmin API.md to incl. HealFormat * Added unit tests for ReInitDisks rpc handler and HealFormatHandler --- cmd/admin-handlers.go | 69 ++++++++ cmd/admin-handlers_test.go | 272 +++++++++++++++-------------- cmd/admin-router.go | 2 + cmd/admin-rpc-client.go | 35 ++++ cmd/admin-rpc-server.go | 43 +++++ cmd/admin-rpc-server_test.go | 80 +++++++++ cmd/globals.go | 7 + cmd/server-main.go | 9 + cmd/test-utils_test.go | 12 ++ cmd/xl-v1.go | 8 + pkg/madmin/API.md | 29 ++- pkg/madmin/examples/heal-format.go | 49 ++++++ pkg/madmin/heal-commands.go | 29 +++ 13 files changed, 511 insertions(+), 133 deletions(-) create mode 100644 pkg/madmin/examples/heal-format.go diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 794f89689..ffd64c2ec 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -19,6 +19,7 @@ package cmd import ( "encoding/json" "encoding/xml" + "fmt" "io/ioutil" "net/http" "net/url" @@ -374,6 +375,7 @@ func (adminAPI adminAPIHandlers) ListBucketsHealHandler(w http.ResponseWriter, r } // HealBucketHandler - POST /?heal&bucket=mybucket +// - x-minio-operation = bucket // - bucket is mandatory query parameter // Heal a given bucket, if present. func (adminAPI adminAPIHandlers) HealBucketHandler(w http.ResponseWriter, r *http.Request) { @@ -425,6 +427,7 @@ func isDryRun(qval url.Values) bool { } // HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject +// - x-minio-operation = object // - bucket and object are both mandatory query parameters // Heal a given object, if present. func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *http.Request) { @@ -473,3 +476,69 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt // Return 200 on success. writeSuccessResponseHeadersOnly(w) } + +// HealFormatHandler - POST /?heal +// - x-minio-operation = format +// - bucket and object are both mandatory query parameters +// Heal a given object, if present. +func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *http.Request) { + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil { + writeErrorResponse(w, ErrServerNotInitialized, r.URL) + return + } + + // Validate request signature. + adminAPIErr := checkRequestAuthType(r, "", "", "") + if adminAPIErr != ErrNone { + writeErrorResponse(w, adminAPIErr, r.URL) + return + } + + // Check if this setup is an erasure code backend, since + // heal-format is only applicable to single node XL and + // distributed XL setup. + if !globalIsXL { + writeErrorResponse(w, ErrNotImplemented, r.URL) + return + } + + // Create a new set of storage instances to heal format.json. + bootstrapDisks, err := initStorageDisks(globalEndpoints) + if err != nil { + fmt.Println(traceError(err)) + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + // Heal format.json on available storage. + err = healFormatXL(bootstrapDisks) + if err != nil { + fmt.Println(traceError(err)) + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + // Instantiate new object layer with newly formatted storage. + newObjectAPI, err := newXLObjects(bootstrapDisks) + if err != nil { + fmt.Println(traceError(err)) + writeErrorResponse(w, toAPIErrorCode(err), r.URL) + return + } + + // Set object layer with newly formatted storage to globalObjectAPI. + globalObjLayerMutex.Lock() + globalObjectAPI = newObjectAPI + globalObjLayerMutex.Unlock() + + // Shutdown storage belonging to old object layer instance. + objectAPI.Shutdown() + + // Inform peers to reinitialize storage with newly formatted storage. + reInitPeerDisks(globalAdminPeers) + + // Return 200 on success. + writeSuccessResponseHeadersOnly(w) +} diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index 138478213..ee6376f86 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -29,6 +29,79 @@ import ( router "github.com/gorilla/mux" ) +// adminXLTestBed - encapsulates subsystems that need to be setup for +// admin-handler unit tests. +type adminXLTestBed struct { + configPath string + xlDirs []string + objLayer ObjectLayer + mux *router.Router +} + +// prepareAdminXLTestBed - helper function that setups a single-node +// XL backend for admin-handler tests. +func prepareAdminXLTestBed() (*adminXLTestBed, error) { + // reset global variables to start afresh. + resetTestGlobals() + // Initialize minio server config. + rootPath, err := newTestConfig(globalMinioDefaultRegion) + if err != nil { + return nil, err + } + // Initializing objectLayer for HealFormatHandler. + objLayer, xlDirs, xlErr := initTestXLObjLayer() + if xlErr != nil { + return nil, xlErr + } + + // Set globalEndpoints for a single node XL setup. + for _, xlDir := range xlDirs { + globalEndpoints = append(globalEndpoints, &url.URL{ + Path: xlDir, + }) + } + + // Set globalIsXL to indicate that the setup uses an erasure code backend. + globalIsXL = true + + // initialize NSLock. + isDistXL := false + initNSLock(isDistXL) + + // Setup admin mgmt REST API handlers. + adminRouter := router.NewRouter() + registerAdminRouter(adminRouter) + + return &adminXLTestBed{ + configPath: rootPath, + xlDirs: xlDirs, + objLayer: objLayer, + mux: adminRouter, + }, nil +} + +// TearDown - method that resets the test bed for subsequent unit +// tests to start afresh. +func (atb *adminXLTestBed) TearDown() { + removeAll(atb.configPath) + removeRoots(atb.xlDirs) + resetTestGlobals() +} + +// initTestObjLayer - Helper function to initialize an XL-based object +// layer and set globalObjectAPI. +func initTestXLObjLayer() (ObjectLayer, []string, error) { + objLayer, xlDirs, xlErr := prepareXL() + if xlErr != nil { + return nil, nil, xlErr + } + // Make objLayer available to all internal services via globalObjectAPI. + globalObjLayerMutex.Lock() + globalObjectAPI = objLayer + globalObjLayerMutex.Unlock() + return objLayer, xlDirs, nil +} + // cmdType - Represents different service subcomands like status, stop // and restart. type cmdType int @@ -115,17 +188,11 @@ func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Requ // testServicesCmdHandler - parametrizes service subcommand tests on // cmdType value. func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing.T) { - // reset globals. - // this is to make sure that the tests are not affected by modified value. - resetTestGlobals() - // initialize NSLock. - initNSLock(false) - // Initialize configuration for access/secret credentials. - rootPath, err := newTestConfig(globalMinioDefaultRegion) + adminTestBed, err := prepareAdminXLTestBed() if err != nil { - t.Fatalf("Unable to initialize server config. %s", err) + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") } - defer removeAll(rootPath) + defer adminTestBed.TearDown() // Initialize admin peers to make admin RPC calls. Note: In a // single node setup, this degenerates to a simple function @@ -139,29 +206,12 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing globalMinioAddr = eps[0].Host initGlobalAdminPeers(eps) - if cmd == statusCmd { - // Initializing objectLayer and corresponding - // []StorageAPI since DiskInfo() method requires it. - objLayer, xlDirs, xlErr := prepareXL() - if xlErr != nil { - t.Fatalf("failed to initialize XL based object layer - %v.", xlErr) - } - defer removeRoots(xlDirs) - // Make objLayer available to all internal services via globalObjectAPI. - globalObjLayerMutex.Lock() - globalObjectAPI = objLayer - globalObjLayerMutex.Unlock() - } - // Setting up a go routine to simulate ServerMux's // handleServiceSignals for stop and restart commands. if cmd == restartCmd { go testServiceSignalReceiver(cmd, t) } credentials := serverConfig.GetCredential() - adminRouter := router.NewRouter() - registerAdminRouter(adminRouter) - var body []byte if cmd == setCreds { @@ -174,7 +224,7 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing } rec := httptest.NewRecorder() - adminRouter.ServeHTTP(rec, req) + adminTestBed.mux.ServeHTTP(rec, req) if cmd == statusCmd { expectedInfo := newObjectLayerFn().StorageInfo() @@ -232,17 +282,11 @@ func mkLockQueryVal(bucket, prefix, relTimeStr string) url.Values { // Test for locks list management REST API. func TestListLocksHandler(t *testing.T) { - // reset globals. - // this is to make sure that the tests are not affected by modified globals. - resetTestGlobals() - // initialize NSLock. - initNSLock(false) - - rootPath, err := newTestConfig(globalMinioDefaultRegion) + adminTestBed, err := prepareAdminXLTestBed() if err != nil { - t.Fatalf("Unable to initialize server config. %s", err) + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") } - defer removeAll(rootPath) + defer adminTestBed.TearDown() // Initialize admin peers to make admin RPC calls. eps, err := parseStorageEndpoints([]string{"http://localhost"}) @@ -254,10 +298,6 @@ func TestListLocksHandler(t *testing.T) { globalMinioAddr = eps[0].Host initGlobalAdminPeers(eps) - // Setup admin mgmt REST API handlers. - adminRouter := router.NewRouter() - registerAdminRouter(adminRouter) - testCases := []struct { bucket string prefix string @@ -308,7 +348,7 @@ func TestListLocksHandler(t *testing.T) { t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err) } rec := httptest.NewRecorder() - adminRouter.ServeHTTP(rec, req) + adminTestBed.mux.ServeHTTP(rec, req) if test.expectedStatus != rec.Code { t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code) } @@ -317,17 +357,11 @@ func TestListLocksHandler(t *testing.T) { // Test for locks clear management REST API. func TestClearLocksHandler(t *testing.T) { - // reset globals. - // this is to make sure that the tests are not affected by modified globals. - resetTestGlobals() - // initialize NSLock. - initNSLock(false) - - rootPath, err := newTestConfig(globalMinioDefaultRegion) + adminTestBed, err := prepareAdminXLTestBed() if err != nil { - t.Fatalf("Unable to initialize server config. %s", err) + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") } - defer removeAll(rootPath) + defer adminTestBed.TearDown() // Initialize admin peers to make admin RPC calls. eps, err := parseStorageEndpoints([]string{"http://localhost"}) @@ -336,10 +370,6 @@ func TestClearLocksHandler(t *testing.T) { } initGlobalAdminPeers(eps) - // Setup admin mgmt REST API handlers. - adminRouter := router.NewRouter() - registerAdminRouter(adminRouter) - testCases := []struct { bucket string prefix string @@ -390,7 +420,7 @@ func TestClearLocksHandler(t *testing.T) { t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err) } rec := httptest.NewRecorder() - adminRouter.ServeHTTP(rec, req) + adminTestBed.mux.ServeHTTP(rec, req) if test.expectedStatus != rec.Code { t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code) } @@ -556,36 +586,19 @@ func TestValidateHealQueryParams(t *testing.T) { // TestListObjectsHeal - Test for ListObjectsHealHandler. func TestListObjectsHealHandler(t *testing.T) { - rootPath, err := newTestConfig("us-east-1") + adminTestBed, err := prepareAdminXLTestBed() if err != nil { - t.Fatalf("Unable to initialize server config. %s", err) - } - defer removeAll(rootPath) - - // Initializing objectLayer and corresponding []StorageAPI - // since ListObjectsHeal() method requires it. - objLayer, xlDirs, xlErr := prepareXL() - if xlErr != nil { - t.Fatalf("failed to initialize XL based object layer - %v.", xlErr) + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") } - defer removeRoots(xlDirs) + defer adminTestBed.TearDown() - err = objLayer.MakeBucket("mybucket") + err = adminTestBed.objLayer.MakeBucket("mybucket") if err != nil { t.Fatalf("Failed to make bucket - %v", err) } // Delete bucket after running all test cases. - defer objLayer.DeleteBucket("mybucket") - - // Make objLayer available to all internal services via globalObjectAPI. - globalObjLayerMutex.Lock() - globalObjectAPI = objLayer - globalObjLayerMutex.Unlock() - - // Setup admin mgmt REST API handlers. - adminRouter := router.NewRouter() - registerAdminRouter(adminRouter) + defer adminTestBed.objLayer.DeleteBucket("mybucket") testCases := []struct { bucket string @@ -695,7 +708,7 @@ func TestListObjectsHealHandler(t *testing.T) { t.Fatalf("Test %d - Failed to sign list objects needing heal request - %v", i+1, err) } rec := httptest.NewRecorder() - adminRouter.ServeHTTP(rec, req) + adminTestBed.mux.ServeHTTP(rec, req) if test.statusCode != rec.Code { t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code) } @@ -704,36 +717,19 @@ func TestListObjectsHealHandler(t *testing.T) { // TestHealBucketHandler - Test for HealBucketHandler. func TestHealBucketHandler(t *testing.T) { - rootPath, err := newTestConfig("us-east-1") + adminTestBed, err := prepareAdminXLTestBed() if err != nil { - t.Fatalf("Unable to initialize server config. %s", err) - } - defer removeAll(rootPath) - - // Initializing objectLayer and corresponding []StorageAPI - // since MakeBucket() and DeleteBucket() methods requires it. - objLayer, xlDirs, xlErr := prepareXL() - if xlErr != nil { - t.Fatalf("failed to initialize XL based object layer - %v.", xlErr) + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") } - defer removeRoots(xlDirs) + defer adminTestBed.TearDown() - err = objLayer.MakeBucket("mybucket") + err = adminTestBed.objLayer.MakeBucket("mybucket") if err != nil { t.Fatalf("Failed to make bucket - %v", err) } // Delete bucket after running all test cases. - defer objLayer.DeleteBucket("mybucket") - - // Make objLayer available to all internal services via globalObjectAPI. - globalObjLayerMutex.Lock() - globalObjectAPI = objLayer - globalObjLayerMutex.Unlock() - - // Setup admin mgmt REST API handlers. - adminRouter := router.NewRouter() - registerAdminRouter(adminRouter) + defer adminTestBed.objLayer.DeleteBucket("mybucket") testCases := []struct { bucket string @@ -771,7 +767,8 @@ func TestHealBucketHandler(t *testing.T) { req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil) if err != nil { - t.Fatalf("Test %d - Failed to construct heal bucket request - %v", i+1, err) + t.Fatalf("Test %d - Failed to construct heal bucket request - %v", + i+1, err) } req.Header.Set(minioAdminOpHeader, "bucket") @@ -779,12 +776,14 @@ func TestHealBucketHandler(t *testing.T) { cred := serverConfig.GetCredential() err = signRequestV4(req, cred.AccessKey, cred.SecretKey) if err != nil { - t.Fatalf("Test %d - Failed to sign heal bucket request - %v", i+1, err) + t.Fatalf("Test %d - Failed to sign heal bucket request - %v", + i+1, err) } rec := httptest.NewRecorder() - adminRouter.ServeHTTP(rec, req) + adminTestBed.mux.ServeHTTP(rec, req) if test.statusCode != rec.Code { - t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code) + t.Errorf("Test %d - Expected HTTP status code %d but received %d", + i+1, test.statusCode, rec.Code) } } @@ -792,29 +791,22 @@ func TestHealBucketHandler(t *testing.T) { // TestHealObjectHandler - Test for HealObjectHandler. func TestHealObjectHandler(t *testing.T) { - rootPath, err := newTestConfig("us-east-1") + adminTestBed, err := prepareAdminXLTestBed() if err != nil { - t.Fatalf("Unable to initialize server config. %s", err) - } - defer removeAll(rootPath) - - // Initializing objectLayer and corresponding []StorageAPI - // since MakeBucket(), PutObject() and DeleteBucket() method requires it. - objLayer, xlDirs, xlErr := prepareXL() - if xlErr != nil { - t.Fatalf("failed to initialize XL based object layer - %v.", xlErr) + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") } - defer removeRoots(xlDirs) + defer adminTestBed.TearDown() // Create an object myobject under bucket mybucket. bucketName := "mybucket" objName := "myobject" - err = objLayer.MakeBucket(bucketName) + err = adminTestBed.objLayer.MakeBucket(bucketName) if err != nil { t.Fatalf("Failed to make bucket %s - %v", bucketName, err) } - _, err = objLayer.PutObject(bucketName, objName, int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "") + _, err = adminTestBed.objLayer.PutObject(bucketName, objName, + int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "") if err != nil { t.Fatalf("Failed to create %s - %v", objName, err) } @@ -823,16 +815,7 @@ func TestHealObjectHandler(t *testing.T) { defer func(objLayer ObjectLayer, bucketName, objName string) { objLayer.DeleteObject(bucketName, objName) objLayer.DeleteBucket(bucketName) - }(objLayer, bucketName, objName) - - // Make objLayer available to all internal services via globalObjectAPI. - globalObjLayerMutex.Lock() - globalObjectAPI = objLayer - globalObjLayerMutex.Unlock() - - // Setup admin mgmt REST API handlers. - adminRouter := router.NewRouter() - registerAdminRouter(adminRouter) + }(adminTestBed.objLayer, bucketName, objName) testCases := []struct { bucket string @@ -899,9 +882,42 @@ func TestHealObjectHandler(t *testing.T) { t.Fatalf("Test %d - Failed to sign heal object request - %v", i+1, err) } rec := httptest.NewRecorder() - adminRouter.ServeHTTP(rec, req) + adminTestBed.mux.ServeHTTP(rec, req) if test.statusCode != rec.Code { t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code) } } } + +// TestHealFormatHandler - test for HealFormatHandler. +func TestHealFormatHandler(t *testing.T) { + adminTestBed, err := prepareAdminXLTestBed() + if err != nil { + t.Fatal("Failed to initialize a single node XL backend for admin handler tests.") + } + defer adminTestBed.TearDown() + + // Prepare query params for heal-format mgmt REST API. + queryVal := url.Values{} + queryVal.Set("heal", "") + req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil) + if err != nil { + t.Fatalf("Failed to construct heal object request - %v", err) + } + + // Set x-minio-operation header to format. + req.Header.Set(minioAdminOpHeader, "format") + + // Sign the request using signature v4. + cred := serverConfig.GetCredential() + err = signRequestV4(req, cred.AccessKey, cred.SecretKey) + if err != nil { + t.Fatalf("Failed to sign heal object request - %v", err) + } + + rec := httptest.NewRecorder() + adminTestBed.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Errorf("Expected to succeed but failed with %d", rec.Code) + } +} diff --git a/cmd/admin-router.go b/cmd/admin-router.go index a21b1ffb6..3136670f6 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -57,4 +57,6 @@ func registerAdminRouter(mux *router.Router) { adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "bucket").HandlerFunc(adminAPI.HealBucketHandler) // Heal Objects. adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler) + // Heal Format. + adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler) } diff --git a/cmd/admin-rpc-client.go b/cmd/admin-rpc-client.go index c117eac9c..5f22e6177 100644 --- a/cmd/admin-rpc-client.go +++ b/cmd/admin-rpc-client.go @@ -38,6 +38,7 @@ type remoteAdminClient struct { type adminCmdRunner interface { Restart() error ListLocks(bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) + ReInitDisks() error } // Restart - Sends a message over channel to the go-routine @@ -73,6 +74,20 @@ func (rc remoteAdminClient) ListLocks(bucket, prefix string, relTime time.Durati return reply.volLocks, nil } +// ReInitDisks - There is nothing to do here, heal format REST API +// handler has already formatted and reinitialized the local disks. +func (lc localAdminClient) ReInitDisks() error { + return nil +} + +// ReInitDisks - Signals peers via RPC to reinitialize their disks and +// object layer. +func (rc remoteAdminClient) ReInitDisks() error { + args := AuthRPCArgs{} + reply := AuthRPCReply{} + return rc.Call("Admin.ReInitDisks", &args, &reply) +} + // adminPeer - represents an entity that implements Restart methods. type adminPeer struct { addr string @@ -159,6 +174,8 @@ func sendServiceCmd(cps adminPeers, cmd serviceSignal) { errs[0] = invokeServiceCmd(cps[0], cmd) } +// listPeerLocksInfo - fetch list of locks held on the given bucket, +// matching prefix older than relTime from all peer servers. func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) { // Used to aggregate volume lock information from all nodes. allLocks := make([][]VolumeLockInfo, len(peers)) @@ -206,3 +223,21 @@ func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Dur } return groupedLockInfos, nil } + +// reInitPeerDisks - reinitialize disks and object layer on peer servers to use the new format. +func reInitPeerDisks(peers adminPeers) error { + errs := make([]error, len(peers)) + + // Send ReInitDisks RPC call to all nodes. + // for local adminPeer this is a no-op. + wg := sync.WaitGroup{} + for i, peer := range peers { + wg.Add(1) + go func(idx int, peer adminPeer) { + defer wg.Done() + errs[idx] = peer.cmdRunner.ReInitDisks() + }(i, peer) + } + wg.Wait() + return nil +} diff --git a/cmd/admin-rpc-server.go b/cmd/admin-rpc-server.go index 4bd05549a..470082fde 100644 --- a/cmd/admin-rpc-server.go +++ b/cmd/admin-rpc-server.go @@ -17,6 +17,7 @@ package cmd import ( + "errors" "net/rpc" "time" @@ -25,6 +26,8 @@ import ( const adminPath = "/admin" +var errUnsupportedBackend = errors.New("not supported for non erasure-code backend") + // adminCmd - exports RPC methods for service status, stop and // restart commands. type adminCmd struct { @@ -57,11 +60,51 @@ func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error { // ListLocks - lists locks held by requests handled by this server instance. func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error { + if err := query.IsAuthenticated(); err != nil { + return err + } volLocks := listLocksInfo(query.bucket, query.prefix, query.relTime) *reply = ListLocksReply{volLocks: volLocks} return nil } +// ReInitDisk - reinitialize storage disks and object layer to use the +// new format. +func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error { + if err := args.IsAuthenticated(); err != nil { + return err + } + + if !globalIsXL { + return errUnsupportedBackend + } + + // Get the current object layer instance. + objLayer := newObjectLayerFn() + + // Initialize new disks to include the newly formatted disks. + bootstrapDisks, err := initStorageDisks(globalEndpoints) + if err != nil { + return err + } + + // Initialize new object layer with newly formatted disks. + newObjectAPI, err := newXLObjects(bootstrapDisks) + if err != nil { + return err + } + + // Replace object layer with newly formatted storage. + globalObjLayerMutex.Lock() + globalObjectAPI = newObjectAPI + globalObjLayerMutex.Unlock() + + // Shutdown storage belonging to old object layer instance. + objLayer.Shutdown() + + return nil +} + // registerAdminRPCRouter - registers RPC methods for service status, // stop and restart commands. func registerAdminRPCRouter(mux *router.Router) error { diff --git a/cmd/admin-rpc-server_test.go b/cmd/admin-rpc-server_test.go index 5577c1b2c..32d1dd288 100644 --- a/cmd/admin-rpc-server_test.go +++ b/cmd/admin-rpc-server_test.go @@ -17,6 +17,7 @@ package cmd import ( + "net/url" "testing" "time" ) @@ -61,6 +62,85 @@ func testAdminCmd(cmd cmdType, t *testing.T) { } } +// TestAdminRestart - test for Admin.Restart RPC service. func TestAdminRestart(t *testing.T) { testAdminCmd(restartCmd, t) } + +// TestReInitDisks - test for Admin.ReInitDisks RPC service. +func TestReInitDisks(t *testing.T) { + // Reset global variables to start afresh. + resetTestGlobals() + + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Unable to initialize server config. %s", err) + } + defer removeAll(rootPath) + + // Initializing objectLayer for HealFormatHandler. + _, xlDirs, xlErr := initTestXLObjLayer() + if xlErr != nil { + t.Fatalf("failed to initialize XL based object layer - %v.", xlErr) + } + defer removeRoots(xlDirs) + + // Set globalEndpoints for a single node XL setup. + for _, xlDir := range xlDirs { + globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir}) + } + + // Setup admin rpc server for an XL backend. + globalIsXL = true + adminServer := adminCmd{} + creds := serverConfig.GetCredential() + args := LoginRPCArgs{ + Username: creds.AccessKey, + Password: creds.SecretKey, + Version: Version, + RequestTime: time.Now().UTC(), + } + reply := LoginRPCReply{} + err = adminServer.Login(&args, &reply) + if err != nil { + t.Fatalf("Failed to login to admin server - %v", err) + } + + authArgs := AuthRPCArgs{ + AuthToken: reply.AuthToken, + RequestTime: time.Now().UTC(), + } + authReply := AuthRPCReply{} + + err = adminServer.ReInitDisks(&authArgs, &authReply) + if err != nil { + t.Errorf("Expected to pass, but failed with %v", err) + } + + // Negative test case with admin rpc server setup for FS. + globalIsXL = false + fsAdminServer := adminCmd{} + fsArgs := LoginRPCArgs{ + Username: creds.AccessKey, + Password: creds.SecretKey, + Version: Version, + RequestTime: time.Now().UTC(), + } + fsReply := LoginRPCReply{} + err = fsAdminServer.Login(&fsArgs, &fsReply) + if err != nil { + t.Fatalf("Failed to login to fs admin server - %v", err) + } + + authArgs = AuthRPCArgs{ + AuthToken: fsReply.AuthToken, + RequestTime: time.Now().UTC(), + } + authReply = AuthRPCReply{} + // Attempt ReInitDisks service on a FS backend. + err = fsAdminServer.ReInitDisks(&authArgs, &authReply) + if err != errUnsupportedBackend { + t.Errorf("Expected to fail with %v, but received %v", + errUnsupportedBackend, err) + } +} diff --git a/cmd/globals.go b/cmd/globals.go index c2f2a7861..e1b943b27 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -18,6 +18,7 @@ package cmd import ( "crypto/x509" + "net/url" "os" "runtime" "strings" @@ -70,6 +71,9 @@ var ( // Indicates if the running minio server is distributed setup. globalIsDistXL = false + // Indicates if the running minio server is an erasure-code backend. + globalIsXL = false + // This flag is set to 'true' by default, it is set to `false` // when MINIO_BROWSER env is set to 'off'. globalIsBrowserEnabled = !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off") @@ -112,6 +116,9 @@ var ( // Secret key passed from the environment globalEnvSecretKey = os.Getenv("MINIO_SECRET_KEY") + // url.URL endpoints of disks that belong to the object storage. + globalEndpoints = []*url.URL{} + // Add new variable global values here. ) diff --git a/cmd/server-main.go b/cmd/server-main.go index 3bd31c4d2..431d25a14 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -418,6 +418,12 @@ func serverMain(c *cli.Context) { fatalIf(initDsyncNodes(endpoints), "Unable to initialize distributed locking clients") } + // Set globalIsXL if erasure code backend is about to be + // initialized for the given endpoints. + if len(endpoints) > 1 { + globalIsXL = true + } + // Initialize name space lock. initNSLock(globalIsDistXL) @@ -453,6 +459,9 @@ func serverMain(c *cli.Context) { fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server.") }() + // Set endpoints of []*url.URL type to globalEndpoints. + globalEndpoints = endpoints + newObject, err := newObjectLayer(srvConfig) fatalIf(err, "Initializing object layer failed") diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index 21641b42c..bd9537ca6 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -488,6 +488,14 @@ func resetGlobalEventnotify() { globalEventNotifier = nil } +func resetGlobalEndpoints() { + globalEndpoints = []*url.URL{} +} + +func resetGlobalIsXL() { + globalIsXL = false +} + // Resets all the globals used modified in tests. // Resetting ensures that the changes made to globals by one test doesn't affect others. func resetTestGlobals() { @@ -501,6 +509,10 @@ func resetTestGlobals() { resetGlobalNSLock() // Reset global event notifier. resetGlobalEventnotify() + // Reset global endpoints. + resetGlobalEndpoints() + // Reset global isXL flag. + resetGlobalIsXL() } // Configure the server for the test run. diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index d9ad936e7..a94cdd05a 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -163,6 +163,14 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) { // Shutdown function for object storage interface. func (xl xlObjects) Shutdown() error { // Add any object layer shutdown activities here. + for _, disk := range xl.storageDisks { + // This closes storage rpc client connections if any. + // Otherwise this is a no-op. + if disk == nil { + continue + } + disk.Close() + } return nil } diff --git a/pkg/madmin/API.md b/pkg/madmin/API.md index 8b6d0393a..c54fd292e 100644 --- a/pkg/madmin/API.md +++ b/pkg/madmin/API.md @@ -38,8 +38,11 @@ func main() { | Service operations|LockInfo operations|Healing operations| |:---|:---|:---| -|[`ServiceStatus`](#ServiceStatus)| | | -|[`ServiceRestart`](#ServiceRestart)| | | +|[`ServiceStatus`](#ServiceStatus)| [`ListLocks`](#ListLocks)| [`ListObjectsHeal`](#ListObjectsHeal)| +|[`ServiceRestart`](#ServiceRestart)| [`ClearLocks`](#ClearLocks)| [`ListBucketsHeal`](#ListBucketsHeal)| +| | |[`HealBucket`](#HealBucket) | +| | |[`HealObject`](#HealObject)| +| | |[`HealFormat`](#HealFormat)| ## 1. Constructor @@ -185,14 +188,14 @@ __Example__ } ``` - -### ListBucketsList() error + +### ListBucketsHeal() error If successful returns information on the list of buckets that need healing. __Example__ ``` go - // List buckets that need healing + // List buckets that need healing healBucketsList, err := madmClnt.ListBucketsHeal() if err != nil { fmt.Println(err) @@ -244,3 +247,19 @@ __Example__ log.Println("successfully healed mybucket/myobject") ``` + + +### HealFormat() error +Heal storage format on available disks. This is used when disks were replaced or were found with missing format. This is supported only for erasure-coded backend. + +__Example__ + +``` go + err := madmClnt.HealFormat() + if err != nil { + log.Fatalln(err) + } + + log.Println("successfully healed storage format on available disks.") + +``` diff --git a/pkg/madmin/examples/heal-format.go b/pkg/madmin/examples/heal-format.go new file mode 100644 index 000000000..4b915747d --- /dev/null +++ b/pkg/madmin/examples/heal-format.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * Minio Cloud Storage, (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "log" + + "github.com/minio/minio/pkg/madmin" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are + // dummy values, please replace them with original values. + + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are + // dummy values, please replace them with original values. + + // API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise. + // New returns an Minio Admin client object. + madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Heal storage format on available disks. + err = madmClnt.HealFormat() + if err != nil { + log.Fatalln(err) + } + + log.Println("successfully healed storage format on available disks.") +} diff --git a/pkg/madmin/heal-commands.go b/pkg/madmin/heal-commands.go index e439b84d2..ba6d6fa12 100644 --- a/pkg/madmin/heal-commands.go +++ b/pkg/madmin/heal-commands.go @@ -403,3 +403,32 @@ func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) error { return nil } + +// HealFormat - heal storage format on available disks. +func (adm *AdminClient) HealFormat() error { + queryVal := url.Values{} + queryVal.Set("heal", "") + + // Set x-minio-operation to format. + hdrs := make(http.Header) + hdrs.Set(minioAdminOpHeader, "format") + + reqData := requestData{ + queryValues: queryVal, + customHeaders: hdrs, + } + + // Execute POST on /?heal to heal storage format. + resp, err := adm.executeMethod("POST", reqData) + + defer closeResponse(resp) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return errors.New("Got HTTP Status: " + resp.Status) + } + + return nil +}