diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go
index 794f89689..ffd64c2ec 100644
--- a/cmd/admin-handlers.go
+++ b/cmd/admin-handlers.go
@@ -19,6 +19,7 @@ package cmd
import (
"encoding/json"
"encoding/xml"
+ "fmt"
"io/ioutil"
"net/http"
"net/url"
@@ -374,6 +375,7 @@ func (adminAPI adminAPIHandlers) ListBucketsHealHandler(w http.ResponseWriter, r
}
// HealBucketHandler - POST /?heal&bucket=mybucket
+// - x-minio-operation = bucket
// - bucket is mandatory query parameter
// Heal a given bucket, if present.
func (adminAPI adminAPIHandlers) HealBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -425,6 +427,7 @@ func isDryRun(qval url.Values) bool {
}
// HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject
+// - x-minio-operation = object
// - bucket and object are both mandatory query parameters
// Heal a given object, if present.
func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *http.Request) {
@@ -473,3 +476,69 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
// Return 200 on success.
writeSuccessResponseHeadersOnly(w)
}
+
+// HealFormatHandler - POST /?heal
+// - x-minio-operation = format
+// - bucket and object are both mandatory query parameters
+// Heal a given object, if present.
+func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *http.Request) {
+ // Get current object layer instance.
+ objectAPI := newObjectLayerFn()
+ if objectAPI == nil {
+ writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+ return
+ }
+
+ // Validate request signature.
+ adminAPIErr := checkRequestAuthType(r, "", "", "")
+ if adminAPIErr != ErrNone {
+ writeErrorResponse(w, adminAPIErr, r.URL)
+ return
+ }
+
+ // Check if this setup is an erasure code backend, since
+ // heal-format is only applicable to single node XL and
+ // distributed XL setup.
+ if !globalIsXL {
+ writeErrorResponse(w, ErrNotImplemented, r.URL)
+ return
+ }
+
+ // Create a new set of storage instances to heal format.json.
+ bootstrapDisks, err := initStorageDisks(globalEndpoints)
+ if err != nil {
+ fmt.Println(traceError(err))
+ writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+ return
+ }
+
+ // Heal format.json on available storage.
+ err = healFormatXL(bootstrapDisks)
+ if err != nil {
+ fmt.Println(traceError(err))
+ writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+ return
+ }
+
+ // Instantiate new object layer with newly formatted storage.
+ newObjectAPI, err := newXLObjects(bootstrapDisks)
+ if err != nil {
+ fmt.Println(traceError(err))
+ writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+ return
+ }
+
+ // Set object layer with newly formatted storage to globalObjectAPI.
+ globalObjLayerMutex.Lock()
+ globalObjectAPI = newObjectAPI
+ globalObjLayerMutex.Unlock()
+
+ // Shutdown storage belonging to old object layer instance.
+ objectAPI.Shutdown()
+
+ // Inform peers to reinitialize storage with newly formatted storage.
+ reInitPeerDisks(globalAdminPeers)
+
+ // Return 200 on success.
+ writeSuccessResponseHeadersOnly(w)
+}
diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go
index 138478213..ee6376f86 100644
--- a/cmd/admin-handlers_test.go
+++ b/cmd/admin-handlers_test.go
@@ -29,6 +29,79 @@ import (
router "github.com/gorilla/mux"
)
+// adminXLTestBed - encapsulates subsystems that need to be setup for
+// admin-handler unit tests.
+type adminXLTestBed struct {
+ configPath string
+ xlDirs []string
+ objLayer ObjectLayer
+ mux *router.Router
+}
+
+// prepareAdminXLTestBed - helper function that setups a single-node
+// XL backend for admin-handler tests.
+func prepareAdminXLTestBed() (*adminXLTestBed, error) {
+ // reset global variables to start afresh.
+ resetTestGlobals()
+ // Initialize minio server config.
+ rootPath, err := newTestConfig(globalMinioDefaultRegion)
+ if err != nil {
+ return nil, err
+ }
+ // Initializing objectLayer for HealFormatHandler.
+ objLayer, xlDirs, xlErr := initTestXLObjLayer()
+ if xlErr != nil {
+ return nil, xlErr
+ }
+
+ // Set globalEndpoints for a single node XL setup.
+ for _, xlDir := range xlDirs {
+ globalEndpoints = append(globalEndpoints, &url.URL{
+ Path: xlDir,
+ })
+ }
+
+ // Set globalIsXL to indicate that the setup uses an erasure code backend.
+ globalIsXL = true
+
+ // initialize NSLock.
+ isDistXL := false
+ initNSLock(isDistXL)
+
+ // Setup admin mgmt REST API handlers.
+ adminRouter := router.NewRouter()
+ registerAdminRouter(adminRouter)
+
+ return &adminXLTestBed{
+ configPath: rootPath,
+ xlDirs: xlDirs,
+ objLayer: objLayer,
+ mux: adminRouter,
+ }, nil
+}
+
+// TearDown - method that resets the test bed for subsequent unit
+// tests to start afresh.
+func (atb *adminXLTestBed) TearDown() {
+ removeAll(atb.configPath)
+ removeRoots(atb.xlDirs)
+ resetTestGlobals()
+}
+
+// initTestObjLayer - Helper function to initialize an XL-based object
+// layer and set globalObjectAPI.
+func initTestXLObjLayer() (ObjectLayer, []string, error) {
+ objLayer, xlDirs, xlErr := prepareXL()
+ if xlErr != nil {
+ return nil, nil, xlErr
+ }
+ // Make objLayer available to all internal services via globalObjectAPI.
+ globalObjLayerMutex.Lock()
+ globalObjectAPI = objLayer
+ globalObjLayerMutex.Unlock()
+ return objLayer, xlDirs, nil
+}
+
// cmdType - Represents different service subcomands like status, stop
// and restart.
type cmdType int
@@ -115,17 +188,11 @@ func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Requ
// testServicesCmdHandler - parametrizes service subcommand tests on
// cmdType value.
func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing.T) {
- // reset globals.
- // this is to make sure that the tests are not affected by modified value.
- resetTestGlobals()
- // initialize NSLock.
- initNSLock(false)
- // Initialize configuration for access/secret credentials.
- rootPath, err := newTestConfig(globalMinioDefaultRegion)
+ adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
- t.Fatalf("Unable to initialize server config. %s", err)
+ t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
- defer removeAll(rootPath)
+ defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls. Note: In a
// single node setup, this degenerates to a simple function
@@ -139,29 +206,12 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
- if cmd == statusCmd {
- // Initializing objectLayer and corresponding
- // []StorageAPI since DiskInfo() method requires it.
- objLayer, xlDirs, xlErr := prepareXL()
- if xlErr != nil {
- t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
- }
- defer removeRoots(xlDirs)
- // Make objLayer available to all internal services via globalObjectAPI.
- globalObjLayerMutex.Lock()
- globalObjectAPI = objLayer
- globalObjLayerMutex.Unlock()
- }
-
// Setting up a go routine to simulate ServerMux's
// handleServiceSignals for stop and restart commands.
if cmd == restartCmd {
go testServiceSignalReceiver(cmd, t)
}
credentials := serverConfig.GetCredential()
- adminRouter := router.NewRouter()
- registerAdminRouter(adminRouter)
-
var body []byte
if cmd == setCreds {
@@ -174,7 +224,7 @@ func testServicesCmdHandler(cmd cmdType, args map[string]interface{}, t *testing
}
rec := httptest.NewRecorder()
- adminRouter.ServeHTTP(rec, req)
+ adminTestBed.mux.ServeHTTP(rec, req)
if cmd == statusCmd {
expectedInfo := newObjectLayerFn().StorageInfo()
@@ -232,17 +282,11 @@ func mkLockQueryVal(bucket, prefix, relTimeStr string) url.Values {
// Test for locks list management REST API.
func TestListLocksHandler(t *testing.T) {
- // reset globals.
- // this is to make sure that the tests are not affected by modified globals.
- resetTestGlobals()
- // initialize NSLock.
- initNSLock(false)
-
- rootPath, err := newTestConfig(globalMinioDefaultRegion)
+ adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
- t.Fatalf("Unable to initialize server config. %s", err)
+ t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
- defer removeAll(rootPath)
+ defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://localhost"})
@@ -254,10 +298,6 @@ func TestListLocksHandler(t *testing.T) {
globalMinioAddr = eps[0].Host
initGlobalAdminPeers(eps)
- // Setup admin mgmt REST API handlers.
- adminRouter := router.NewRouter()
- registerAdminRouter(adminRouter)
-
testCases := []struct {
bucket string
prefix string
@@ -308,7 +348,7 @@ func TestListLocksHandler(t *testing.T) {
t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
}
rec := httptest.NewRecorder()
- adminRouter.ServeHTTP(rec, req)
+ adminTestBed.mux.ServeHTTP(rec, req)
if test.expectedStatus != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
}
@@ -317,17 +357,11 @@ func TestListLocksHandler(t *testing.T) {
// Test for locks clear management REST API.
func TestClearLocksHandler(t *testing.T) {
- // reset globals.
- // this is to make sure that the tests are not affected by modified globals.
- resetTestGlobals()
- // initialize NSLock.
- initNSLock(false)
-
- rootPath, err := newTestConfig(globalMinioDefaultRegion)
+ adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
- t.Fatalf("Unable to initialize server config. %s", err)
+ t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
- defer removeAll(rootPath)
+ defer adminTestBed.TearDown()
// Initialize admin peers to make admin RPC calls.
eps, err := parseStorageEndpoints([]string{"http://localhost"})
@@ -336,10 +370,6 @@ func TestClearLocksHandler(t *testing.T) {
}
initGlobalAdminPeers(eps)
- // Setup admin mgmt REST API handlers.
- adminRouter := router.NewRouter()
- registerAdminRouter(adminRouter)
-
testCases := []struct {
bucket string
prefix string
@@ -390,7 +420,7 @@ func TestClearLocksHandler(t *testing.T) {
t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
}
rec := httptest.NewRecorder()
- adminRouter.ServeHTTP(rec, req)
+ adminTestBed.mux.ServeHTTP(rec, req)
if test.expectedStatus != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
}
@@ -556,36 +586,19 @@ func TestValidateHealQueryParams(t *testing.T) {
// TestListObjectsHeal - Test for ListObjectsHealHandler.
func TestListObjectsHealHandler(t *testing.T) {
- rootPath, err := newTestConfig("us-east-1")
+ adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
- t.Fatalf("Unable to initialize server config. %s", err)
- }
- defer removeAll(rootPath)
-
- // Initializing objectLayer and corresponding []StorageAPI
- // since ListObjectsHeal() method requires it.
- objLayer, xlDirs, xlErr := prepareXL()
- if xlErr != nil {
- t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
+ t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
- defer removeRoots(xlDirs)
+ defer adminTestBed.TearDown()
- err = objLayer.MakeBucket("mybucket")
+ err = adminTestBed.objLayer.MakeBucket("mybucket")
if err != nil {
t.Fatalf("Failed to make bucket - %v", err)
}
// Delete bucket after running all test cases.
- defer objLayer.DeleteBucket("mybucket")
-
- // Make objLayer available to all internal services via globalObjectAPI.
- globalObjLayerMutex.Lock()
- globalObjectAPI = objLayer
- globalObjLayerMutex.Unlock()
-
- // Setup admin mgmt REST API handlers.
- adminRouter := router.NewRouter()
- registerAdminRouter(adminRouter)
+ defer adminTestBed.objLayer.DeleteBucket("mybucket")
testCases := []struct {
bucket string
@@ -695,7 +708,7 @@ func TestListObjectsHealHandler(t *testing.T) {
t.Fatalf("Test %d - Failed to sign list objects needing heal request - %v", i+1, err)
}
rec := httptest.NewRecorder()
- adminRouter.ServeHTTP(rec, req)
+ adminTestBed.mux.ServeHTTP(rec, req)
if test.statusCode != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
}
@@ -704,36 +717,19 @@ func TestListObjectsHealHandler(t *testing.T) {
// TestHealBucketHandler - Test for HealBucketHandler.
func TestHealBucketHandler(t *testing.T) {
- rootPath, err := newTestConfig("us-east-1")
+ adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
- t.Fatalf("Unable to initialize server config. %s", err)
- }
- defer removeAll(rootPath)
-
- // Initializing objectLayer and corresponding []StorageAPI
- // since MakeBucket() and DeleteBucket() methods requires it.
- objLayer, xlDirs, xlErr := prepareXL()
- if xlErr != nil {
- t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
+ t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
- defer removeRoots(xlDirs)
+ defer adminTestBed.TearDown()
- err = objLayer.MakeBucket("mybucket")
+ err = adminTestBed.objLayer.MakeBucket("mybucket")
if err != nil {
t.Fatalf("Failed to make bucket - %v", err)
}
// Delete bucket after running all test cases.
- defer objLayer.DeleteBucket("mybucket")
-
- // Make objLayer available to all internal services via globalObjectAPI.
- globalObjLayerMutex.Lock()
- globalObjectAPI = objLayer
- globalObjLayerMutex.Unlock()
-
- // Setup admin mgmt REST API handlers.
- adminRouter := router.NewRouter()
- registerAdminRouter(adminRouter)
+ defer adminTestBed.objLayer.DeleteBucket("mybucket")
testCases := []struct {
bucket string
@@ -771,7 +767,8 @@ func TestHealBucketHandler(t *testing.T) {
req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
if err != nil {
- t.Fatalf("Test %d - Failed to construct heal bucket request - %v", i+1, err)
+ t.Fatalf("Test %d - Failed to construct heal bucket request - %v",
+ i+1, err)
}
req.Header.Set(minioAdminOpHeader, "bucket")
@@ -779,12 +776,14 @@ func TestHealBucketHandler(t *testing.T) {
cred := serverConfig.GetCredential()
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
if err != nil {
- t.Fatalf("Test %d - Failed to sign heal bucket request - %v", i+1, err)
+ t.Fatalf("Test %d - Failed to sign heal bucket request - %v",
+ i+1, err)
}
rec := httptest.NewRecorder()
- adminRouter.ServeHTTP(rec, req)
+ adminTestBed.mux.ServeHTTP(rec, req)
if test.statusCode != rec.Code {
- t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
+ t.Errorf("Test %d - Expected HTTP status code %d but received %d",
+ i+1, test.statusCode, rec.Code)
}
}
@@ -792,29 +791,22 @@ func TestHealBucketHandler(t *testing.T) {
// TestHealObjectHandler - Test for HealObjectHandler.
func TestHealObjectHandler(t *testing.T) {
- rootPath, err := newTestConfig("us-east-1")
+ adminTestBed, err := prepareAdminXLTestBed()
if err != nil {
- t.Fatalf("Unable to initialize server config. %s", err)
- }
- defer removeAll(rootPath)
-
- // Initializing objectLayer and corresponding []StorageAPI
- // since MakeBucket(), PutObject() and DeleteBucket() method requires it.
- objLayer, xlDirs, xlErr := prepareXL()
- if xlErr != nil {
- t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
+ t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
}
- defer removeRoots(xlDirs)
+ defer adminTestBed.TearDown()
// Create an object myobject under bucket mybucket.
bucketName := "mybucket"
objName := "myobject"
- err = objLayer.MakeBucket(bucketName)
+ err = adminTestBed.objLayer.MakeBucket(bucketName)
if err != nil {
t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
}
- _, err = objLayer.PutObject(bucketName, objName, int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
+ _, err = adminTestBed.objLayer.PutObject(bucketName, objName,
+ int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
if err != nil {
t.Fatalf("Failed to create %s - %v", objName, err)
}
@@ -823,16 +815,7 @@ func TestHealObjectHandler(t *testing.T) {
defer func(objLayer ObjectLayer, bucketName, objName string) {
objLayer.DeleteObject(bucketName, objName)
objLayer.DeleteBucket(bucketName)
- }(objLayer, bucketName, objName)
-
- // Make objLayer available to all internal services via globalObjectAPI.
- globalObjLayerMutex.Lock()
- globalObjectAPI = objLayer
- globalObjLayerMutex.Unlock()
-
- // Setup admin mgmt REST API handlers.
- adminRouter := router.NewRouter()
- registerAdminRouter(adminRouter)
+ }(adminTestBed.objLayer, bucketName, objName)
testCases := []struct {
bucket string
@@ -899,9 +882,42 @@ func TestHealObjectHandler(t *testing.T) {
t.Fatalf("Test %d - Failed to sign heal object request - %v", i+1, err)
}
rec := httptest.NewRecorder()
- adminRouter.ServeHTTP(rec, req)
+ adminTestBed.mux.ServeHTTP(rec, req)
if test.statusCode != rec.Code {
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
}
}
}
+
+// TestHealFormatHandler - test for HealFormatHandler.
+func TestHealFormatHandler(t *testing.T) {
+ adminTestBed, err := prepareAdminXLTestBed()
+ if err != nil {
+ t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+ }
+ defer adminTestBed.TearDown()
+
+ // Prepare query params for heal-format mgmt REST API.
+ queryVal := url.Values{}
+ queryVal.Set("heal", "")
+ req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
+ if err != nil {
+ t.Fatalf("Failed to construct heal object request - %v", err)
+ }
+
+ // Set x-minio-operation header to format.
+ req.Header.Set(minioAdminOpHeader, "format")
+
+ // Sign the request using signature v4.
+ cred := serverConfig.GetCredential()
+ err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
+ if err != nil {
+ t.Fatalf("Failed to sign heal object request - %v", err)
+ }
+
+ rec := httptest.NewRecorder()
+ adminTestBed.mux.ServeHTTP(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Errorf("Expected to succeed but failed with %d", rec.Code)
+ }
+}
diff --git a/cmd/admin-router.go b/cmd/admin-router.go
index a21b1ffb6..3136670f6 100644
--- a/cmd/admin-router.go
+++ b/cmd/admin-router.go
@@ -57,4 +57,6 @@ func registerAdminRouter(mux *router.Router) {
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "bucket").HandlerFunc(adminAPI.HealBucketHandler)
// Heal Objects.
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
+ // Heal Format.
+ adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
}
diff --git a/cmd/admin-rpc-client.go b/cmd/admin-rpc-client.go
index c117eac9c..5f22e6177 100644
--- a/cmd/admin-rpc-client.go
+++ b/cmd/admin-rpc-client.go
@@ -38,6 +38,7 @@ type remoteAdminClient struct {
type adminCmdRunner interface {
Restart() error
ListLocks(bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error)
+ ReInitDisks() error
}
// Restart - Sends a message over channel to the go-routine
@@ -73,6 +74,20 @@ func (rc remoteAdminClient) ListLocks(bucket, prefix string, relTime time.Durati
return reply.volLocks, nil
}
+// ReInitDisks - There is nothing to do here, heal format REST API
+// handler has already formatted and reinitialized the local disks.
+func (lc localAdminClient) ReInitDisks() error {
+ return nil
+}
+
+// ReInitDisks - Signals peers via RPC to reinitialize their disks and
+// object layer.
+func (rc remoteAdminClient) ReInitDisks() error {
+ args := AuthRPCArgs{}
+ reply := AuthRPCReply{}
+ return rc.Call("Admin.ReInitDisks", &args, &reply)
+}
+
// adminPeer - represents an entity that implements Restart methods.
type adminPeer struct {
addr string
@@ -159,6 +174,8 @@ func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
errs[0] = invokeServiceCmd(cps[0], cmd)
}
+// listPeerLocksInfo - fetch list of locks held on the given bucket,
+// matching prefix older than relTime from all peer servers.
func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Duration) ([]VolumeLockInfo, error) {
// Used to aggregate volume lock information from all nodes.
allLocks := make([][]VolumeLockInfo, len(peers))
@@ -206,3 +223,21 @@ func listPeerLocksInfo(peers adminPeers, bucket, prefix string, relTime time.Dur
}
return groupedLockInfos, nil
}
+
+// reInitPeerDisks - reinitialize disks and object layer on peer servers to use the new format.
+func reInitPeerDisks(peers adminPeers) error {
+ errs := make([]error, len(peers))
+
+ // Send ReInitDisks RPC call to all nodes.
+ // for local adminPeer this is a no-op.
+ wg := sync.WaitGroup{}
+ for i, peer := range peers {
+ wg.Add(1)
+ go func(idx int, peer adminPeer) {
+ defer wg.Done()
+ errs[idx] = peer.cmdRunner.ReInitDisks()
+ }(i, peer)
+ }
+ wg.Wait()
+ return nil
+}
diff --git a/cmd/admin-rpc-server.go b/cmd/admin-rpc-server.go
index 4bd05549a..470082fde 100644
--- a/cmd/admin-rpc-server.go
+++ b/cmd/admin-rpc-server.go
@@ -17,6 +17,7 @@
package cmd
import (
+ "errors"
"net/rpc"
"time"
@@ -25,6 +26,8 @@ import (
const adminPath = "/admin"
+var errUnsupportedBackend = errors.New("not supported for non erasure-code backend")
+
// adminCmd - exports RPC methods for service status, stop and
// restart commands.
type adminCmd struct {
@@ -57,11 +60,51 @@ func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
// ListLocks - lists locks held by requests handled by this server instance.
func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error {
+ if err := query.IsAuthenticated(); err != nil {
+ return err
+ }
volLocks := listLocksInfo(query.bucket, query.prefix, query.relTime)
*reply = ListLocksReply{volLocks: volLocks}
return nil
}
+// ReInitDisk - reinitialize storage disks and object layer to use the
+// new format.
+func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
+ if err := args.IsAuthenticated(); err != nil {
+ return err
+ }
+
+ if !globalIsXL {
+ return errUnsupportedBackend
+ }
+
+ // Get the current object layer instance.
+ objLayer := newObjectLayerFn()
+
+ // Initialize new disks to include the newly formatted disks.
+ bootstrapDisks, err := initStorageDisks(globalEndpoints)
+ if err != nil {
+ return err
+ }
+
+ // Initialize new object layer with newly formatted disks.
+ newObjectAPI, err := newXLObjects(bootstrapDisks)
+ if err != nil {
+ return err
+ }
+
+ // Replace object layer with newly formatted storage.
+ globalObjLayerMutex.Lock()
+ globalObjectAPI = newObjectAPI
+ globalObjLayerMutex.Unlock()
+
+ // Shutdown storage belonging to old object layer instance.
+ objLayer.Shutdown()
+
+ return nil
+}
+
// registerAdminRPCRouter - registers RPC methods for service status,
// stop and restart commands.
func registerAdminRPCRouter(mux *router.Router) error {
diff --git a/cmd/admin-rpc-server_test.go b/cmd/admin-rpc-server_test.go
index 5577c1b2c..32d1dd288 100644
--- a/cmd/admin-rpc-server_test.go
+++ b/cmd/admin-rpc-server_test.go
@@ -17,6 +17,7 @@
package cmd
import (
+ "net/url"
"testing"
"time"
)
@@ -61,6 +62,85 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
}
}
+// TestAdminRestart - test for Admin.Restart RPC service.
func TestAdminRestart(t *testing.T) {
testAdminCmd(restartCmd, t)
}
+
+// TestReInitDisks - test for Admin.ReInitDisks RPC service.
+func TestReInitDisks(t *testing.T) {
+ // Reset global variables to start afresh.
+ resetTestGlobals()
+
+ rootPath, err := newTestConfig("us-east-1")
+ if err != nil {
+ t.Fatalf("Unable to initialize server config. %s", err)
+ }
+ defer removeAll(rootPath)
+
+ // Initializing objectLayer for HealFormatHandler.
+ _, xlDirs, xlErr := initTestXLObjLayer()
+ if xlErr != nil {
+ t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
+ }
+ defer removeRoots(xlDirs)
+
+ // Set globalEndpoints for a single node XL setup.
+ for _, xlDir := range xlDirs {
+ globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
+ }
+
+ // Setup admin rpc server for an XL backend.
+ globalIsXL = true
+ adminServer := adminCmd{}
+ creds := serverConfig.GetCredential()
+ args := LoginRPCArgs{
+ Username: creds.AccessKey,
+ Password: creds.SecretKey,
+ Version: Version,
+ RequestTime: time.Now().UTC(),
+ }
+ reply := LoginRPCReply{}
+ err = adminServer.Login(&args, &reply)
+ if err != nil {
+ t.Fatalf("Failed to login to admin server - %v", err)
+ }
+
+ authArgs := AuthRPCArgs{
+ AuthToken: reply.AuthToken,
+ RequestTime: time.Now().UTC(),
+ }
+ authReply := AuthRPCReply{}
+
+ err = adminServer.ReInitDisks(&authArgs, &authReply)
+ if err != nil {
+ t.Errorf("Expected to pass, but failed with %v", err)
+ }
+
+ // Negative test case with admin rpc server setup for FS.
+ globalIsXL = false
+ fsAdminServer := adminCmd{}
+ fsArgs := LoginRPCArgs{
+ Username: creds.AccessKey,
+ Password: creds.SecretKey,
+ Version: Version,
+ RequestTime: time.Now().UTC(),
+ }
+ fsReply := LoginRPCReply{}
+ err = fsAdminServer.Login(&fsArgs, &fsReply)
+ if err != nil {
+ t.Fatalf("Failed to login to fs admin server - %v", err)
+ }
+
+ authArgs = AuthRPCArgs{
+ AuthToken: fsReply.AuthToken,
+ RequestTime: time.Now().UTC(),
+ }
+ authReply = AuthRPCReply{}
+ // Attempt ReInitDisks service on a FS backend.
+ err = fsAdminServer.ReInitDisks(&authArgs, &authReply)
+ if err != errUnsupportedBackend {
+ t.Errorf("Expected to fail with %v, but received %v",
+ errUnsupportedBackend, err)
+ }
+}
diff --git a/cmd/globals.go b/cmd/globals.go
index c2f2a7861..e1b943b27 100644
--- a/cmd/globals.go
+++ b/cmd/globals.go
@@ -18,6 +18,7 @@ package cmd
import (
"crypto/x509"
+ "net/url"
"os"
"runtime"
"strings"
@@ -70,6 +71,9 @@ var (
// Indicates if the running minio server is distributed setup.
globalIsDistXL = false
+ // Indicates if the running minio server is an erasure-code backend.
+ globalIsXL = false
+
// This flag is set to 'true' by default, it is set to `false`
// when MINIO_BROWSER env is set to 'off'.
globalIsBrowserEnabled = !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off")
@@ -112,6 +116,9 @@ var (
// Secret key passed from the environment
globalEnvSecretKey = os.Getenv("MINIO_SECRET_KEY")
+ // url.URL endpoints of disks that belong to the object storage.
+ globalEndpoints = []*url.URL{}
+
// Add new variable global values here.
)
diff --git a/cmd/server-main.go b/cmd/server-main.go
index 3bd31c4d2..431d25a14 100644
--- a/cmd/server-main.go
+++ b/cmd/server-main.go
@@ -418,6 +418,12 @@ func serverMain(c *cli.Context) {
fatalIf(initDsyncNodes(endpoints), "Unable to initialize distributed locking clients")
}
+ // Set globalIsXL if erasure code backend is about to be
+ // initialized for the given endpoints.
+ if len(endpoints) > 1 {
+ globalIsXL = true
+ }
+
// Initialize name space lock.
initNSLock(globalIsDistXL)
@@ -453,6 +459,9 @@ func serverMain(c *cli.Context) {
fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server.")
}()
+ // Set endpoints of []*url.URL type to globalEndpoints.
+ globalEndpoints = endpoints
+
newObject, err := newObjectLayer(srvConfig)
fatalIf(err, "Initializing object layer failed")
diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go
index 21641b42c..bd9537ca6 100644
--- a/cmd/test-utils_test.go
+++ b/cmd/test-utils_test.go
@@ -488,6 +488,14 @@ func resetGlobalEventnotify() {
globalEventNotifier = nil
}
+func resetGlobalEndpoints() {
+ globalEndpoints = []*url.URL{}
+}
+
+func resetGlobalIsXL() {
+ globalIsXL = false
+}
+
// Resets all the globals used modified in tests.
// Resetting ensures that the changes made to globals by one test doesn't affect others.
func resetTestGlobals() {
@@ -501,6 +509,10 @@ func resetTestGlobals() {
resetGlobalNSLock()
// Reset global event notifier.
resetGlobalEventnotify()
+ // Reset global endpoints.
+ resetGlobalEndpoints()
+ // Reset global isXL flag.
+ resetGlobalIsXL()
}
// Configure the server for the test run.
diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go
index d9ad936e7..a94cdd05a 100644
--- a/cmd/xl-v1.go
+++ b/cmd/xl-v1.go
@@ -163,6 +163,14 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
// Shutdown function for object storage interface.
func (xl xlObjects) Shutdown() error {
// Add any object layer shutdown activities here.
+ for _, disk := range xl.storageDisks {
+ // This closes storage rpc client connections if any.
+ // Otherwise this is a no-op.
+ if disk == nil {
+ continue
+ }
+ disk.Close()
+ }
return nil
}
diff --git a/pkg/madmin/API.md b/pkg/madmin/API.md
index 8b6d0393a..c54fd292e 100644
--- a/pkg/madmin/API.md
+++ b/pkg/madmin/API.md
@@ -38,8 +38,11 @@ func main() {
| Service operations|LockInfo operations|Healing operations|
|:---|:---|:---|
-|[`ServiceStatus`](#ServiceStatus)| | |
-|[`ServiceRestart`](#ServiceRestart)| | |
+|[`ServiceStatus`](#ServiceStatus)| [`ListLocks`](#ListLocks)| [`ListObjectsHeal`](#ListObjectsHeal)|
+|[`ServiceRestart`](#ServiceRestart)| [`ClearLocks`](#ClearLocks)| [`ListBucketsHeal`](#ListBucketsHeal)|
+| | |[`HealBucket`](#HealBucket) |
+| | |[`HealObject`](#HealObject)|
+| | |[`HealFormat`](#HealFormat)|
## 1. Constructor
@@ -185,14 +188,14 @@ __Example__
}
```
-
-### ListBucketsList() error
+
+### ListBucketsHeal() error
If successful returns information on the list of buckets that need healing.
__Example__
``` go
- // List buckets that need healing
+ // List buckets that need healing
healBucketsList, err := madmClnt.ListBucketsHeal()
if err != nil {
fmt.Println(err)
@@ -244,3 +247,19 @@ __Example__
log.Println("successfully healed mybucket/myobject")
```
+
+
+### HealFormat() error
+Heal storage format on available disks. This is used when disks were replaced or were found with missing format. This is supported only for erasure-coded backend.
+
+__Example__
+
+``` go
+ err := madmClnt.HealFormat()
+ if err != nil {
+ log.Fatalln(err)
+ }
+
+ log.Println("successfully healed storage format on available disks.")
+
+```
diff --git a/pkg/madmin/examples/heal-format.go b/pkg/madmin/examples/heal-format.go
new file mode 100644
index 000000000..4b915747d
--- /dev/null
+++ b/pkg/madmin/examples/heal-format.go
@@ -0,0 +1,49 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+ "log"
+
+ "github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+ // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+ // dummy values, please replace them with original values.
+
+ // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+ // dummy values, please replace them with original values.
+
+ // API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+ // New returns an Minio Admin client object.
+ madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+ if err != nil {
+ log.Fatalln(err)
+ }
+
+ // Heal storage format on available disks.
+ err = madmClnt.HealFormat()
+ if err != nil {
+ log.Fatalln(err)
+ }
+
+ log.Println("successfully healed storage format on available disks.")
+}
diff --git a/pkg/madmin/heal-commands.go b/pkg/madmin/heal-commands.go
index e439b84d2..ba6d6fa12 100644
--- a/pkg/madmin/heal-commands.go
+++ b/pkg/madmin/heal-commands.go
@@ -403,3 +403,32 @@ func (adm *AdminClient) HealObject(bucket, object string, dryrun bool) error {
return nil
}
+
+// HealFormat - heal storage format on available disks.
+func (adm *AdminClient) HealFormat() error {
+ queryVal := url.Values{}
+ queryVal.Set("heal", "")
+
+ // Set x-minio-operation to format.
+ hdrs := make(http.Header)
+ hdrs.Set(minioAdminOpHeader, "format")
+
+ reqData := requestData{
+ queryValues: queryVal,
+ customHeaders: hdrs,
+ }
+
+ // Execute POST on /?heal to heal storage format.
+ resp, err := adm.executeMethod("POST", reqData)
+
+ defer closeResponse(resp)
+ if err != nil {
+ return err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return errors.New("Got HTTP Status: " + resp.Status)
+ }
+
+ return nil
+}