Add service API handler stubs for status, stop and restart (#3417)

cmd/admin-handlers.go

@@ -0,0 +1,64 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+const (
+	minioAdminOpHeader = "X-Minio-Operation"
+)
+
+func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, r, adminAPIErr, r.URL.Path)
+		return
+	}
+	storageInfo := newObjectLayerFn().StorageInfo()
+	jsonBytes, err := json.Marshal(storageInfo)
+	if err != nil {
+		writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path)
+		errorIf(err, "Failed to marshal storage info into json.")
+	}
+	w.WriteHeader(http.StatusOK)
+	writeSuccessResponse(w, jsonBytes)
+}
+
+func (adminAPI adminAPIHandlers) ServiceStopHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, r, adminAPIErr, r.URL.Path)
+		return
+	}
+	// Reply to the client before stopping minio server.
+	w.WriteHeader(http.StatusOK)
+	sendServiceCmd(globalAdminPeers, serviceStop)
+}
+
+func (adminAPI adminAPIHandlers) ServiceRestartHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, r, adminAPIErr, r.URL.Path)
+		return
+	}
+	// Reply to the client before restarting minio server.
+	w.WriteHeader(http.StatusOK)
+	sendServiceCmd(globalAdminPeers, serviceRestart)
+}

cmd/admin-handlers_test.go

@@ -0,0 +1,164 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	router "github.com/gorilla/mux"
+)
+
+type cmdType int
+
+const (
+	statusCmd cmdType = iota
+	stopCmd
+	restartCmd
+)
+
+func (c cmdType) String() string {
+	switch c {
+	case statusCmd:
+		return "status"
+	case stopCmd:
+		return "stop"
+	case restartCmd:
+		return "restart"
+	}
+	return ""
+}
+
+func (c cmdType) apiMethod() string {
+	switch c {
+	case statusCmd:
+		return "GET"
+	case stopCmd:
+		return "POST"
+	case restartCmd:
+		return "POST"
+	}
+	return "GET"
+}
+
+func (c cmdType) toServiceSignal() serviceSignal {
+	switch c {
+	case statusCmd:
+		return serviceStatus
+	case stopCmd:
+		return serviceStop
+	case restartCmd:
+		return serviceRestart
+	}
+	return serviceStatus
+}
+
+func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
+	expectedCmd := cmd.toServiceSignal()
+	serviceCmd := <-globalServiceSignalCh
+	if serviceCmd != expectedCmd {
+		t.Errorf("Expected service command %v but received %v", expectedCmd, serviceCmd)
+	}
+}
+
+func getAdminCmdRequest(cmd cmdType, cred credential) (*http.Request, error) {
+	req, err := newTestRequest(cmd.apiMethod(), "/?service", 0, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set(minioAdminOpHeader, cmd.String())
+	err = signRequestV4(req, cred.AccessKeyID, cred.SecretAccessKey)
+	if err != nil {
+		return nil, err
+	}
+	return req, nil
+}
+
+func testServicesCmdHandler(cmd cmdType, t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Unable to initialize server config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// Initialize admin peers to make admin RPC calls.
+	eps, err := parseStorageEndpoints([]string{"http://localhost"})
+	if err != nil {
+		t.Fatalf("Failed to parse storage end point - %v", err)
+	}
+	initGlobalAdminPeers(eps)
+
+	if cmd == statusCmd {
+		// Initializing objectLayer and corresponding
+		// []StorageAPI since DiskInfo() method requires it.
+		objLayer, fsDir, fsErr := prepareFS()
+		if fsErr != nil {
+			t.Fatalf("failed to initialize XL based object layer - %v.", fsErr)
+		}
+		defer removeRoots([]string{fsDir})
+		globalObjLayerMutex.Lock()
+		globalObjectAPI = objLayer
+		globalObjLayerMutex.Unlock()
+	}
+
+	// Setting up a go routine to simulate ServerMux's
+	// handleServiceSignals for stop and restart commands.
+	switch cmd {
+	case stopCmd, restartCmd:
+		go testServiceSignalReceiver(cmd, t)
+	}
+	credentials := serverConfig.GetCredential()
+	adminRouter := router.NewRouter()
+	registerAdminRouter(adminRouter)
+
+	rec := httptest.NewRecorder()
+	req, err := getAdminCmdRequest(cmd, credentials)
+	if err != nil {
+		t.Fatalf("Failed to build service status request %v", err)
+	}
+	adminRouter.ServeHTTP(rec, req)
+
+	if cmd == statusCmd {
+		expectedInfo := newObjectLayerFn().StorageInfo()
+		receivedInfo := StorageInfo{}
+		if jsonErr := json.Unmarshal(rec.Body.Bytes(), &receivedInfo); jsonErr != nil {
+			t.Errorf("Failed to unmarshal StorageInfo - %v", jsonErr)
+		}
+		if expectedInfo != receivedInfo {
+			t.Errorf("Expected storage info and received storage info differ, %v %v", expectedInfo, receivedInfo)
+		}
+	}
+
+	if rec.Code != http.StatusOK {
+		t.Errorf("Expected to receive %d status code but received %d",
+			http.StatusOK, rec.Code)
+	}
+}
+
+func TestServiceStatusHandler(t *testing.T) {
+	testServicesCmdHandler(statusCmd, t)
+}
+
+func TestServiceStopHandler(t *testing.T) {
+	testServicesCmdHandler(stopCmd, t)
+}
+
+func TestServiceRestartHandler(t *testing.T) {
+	testServicesCmdHandler(restartCmd, t)
+}

cmd/admin-router.go

@@ -0,0 +1,40 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import router "github.com/gorilla/mux"
+
+// adminAPIHandlers provides HTTP handlers for Minio admin API.
+type adminAPIHandlers struct {
+}
+
+// registerAdminRouter - Add handler functions for each service REST API routes.
+func registerAdminRouter(mux *router.Router) {
+
+	adminAPI := adminAPIHandlers{}
+	// Admin router
+	adminRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+
+	/// Admin operations
+
+	// Service status
+	adminRouter.Methods("GET").Queries("service", "").Headers(minioAdminOpHeader, "status").HandlerFunc(adminAPI.ServiceStatusHandler)
+	// Service stop
+	adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "stop").HandlerFunc(adminAPI.ServiceStopHandler)
+	// Service restart
+	adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "restart").HandlerFunc(adminAPI.ServiceRestartHandler)
+}

cmd/admin-rpc-client.go

@@ -0,0 +1,163 @@
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/rpc"
+	"net/url"
+	"path"
+	"sync"
+)
+
+// localAdminClient - represents admin operation to be executed locally.
+type localAdminClient struct {
+}
+
+// remoteAdminClient - represents admin operation to be executed
+// remotely, via RPC.
+type remoteAdminClient struct {
+	*AuthRPCClient
+}
+
+// stopRestarter - abstracts stop and restart operations for both
+// local and remote execution.
+type stopRestarter interface {
+	Stop() error
+	Restart() error
+}
+
+// Stop - Sends a message over channel to the go-routine responsible
+// for stopping the process.
+func (lc localAdminClient) Stop() error {
+	globalServiceSignalCh <- serviceStop
+	return nil
+}
+
+// Restart - Sends a message over channel to the go-routine
+// responsible for restarting the process.
+func (lc localAdminClient) Restart() error {
+	globalServiceSignalCh <- serviceRestart
+	return nil
+}
+
+// Stop - Sends stop command to remote server via RPC.
+func (rc remoteAdminClient) Stop() error {
+	args := GenericArgs{}
+	reply := GenericReply{}
+	err := rc.Call("Service.Shutdown", &args, &reply)
+	if err != nil && err == rpc.ErrShutdown {
+		rc.Close()
+	}
+	return err
+}
+
+// Restart - Sends restart command to remote server via RPC.
+func (rc remoteAdminClient) Restart() error {
+	args := GenericArgs{}
+	reply := GenericReply{}
+	err := rc.Call("Service.Restart", &args, &reply)
+	if err != nil && err == rpc.ErrShutdown {
+		rc.Close()
+	}
+	return err
+}
+
+// adminPeer - represents an entity that implements Stop and Restart methods.
+type adminPeer struct {
+	addr    string
+	svcClnt stopRestarter
+}
+
+// type alias for a collection of adminPeer.
+type adminPeers []adminPeer
+
+// makeAdminPeers - helper function to construct a collection of adminPeer.
+func makeAdminPeers(eps []*url.URL) adminPeers {
+	var servicePeers []adminPeer
+
+	// map to store peers that are already added to ret
+	seenAddr := make(map[string]bool)
+
+	// add local (self) as peer in the array
+	servicePeers = append(servicePeers, adminPeer{
+		globalMinioAddr,
+		localAdminClient{},
+	})
+	seenAddr[globalMinioAddr] = true
+
+	// iterate over endpoints to find new remote peers and add
+	// them to ret.
+	for _, ep := range eps {
+		if ep.Host == "" {
+			continue
+		}
+
+		// Check if the remote host has been added already
+		if !seenAddr[ep.Host] {
+			cfg := authConfig{
+				accessKey:   serverConfig.GetCredential().AccessKeyID,
+				secretKey:   serverConfig.GetCredential().SecretAccessKey,
+				address:     ep.Host,
+				secureConn:  isSSL(),
+				path:        path.Join(reservedBucket, servicePath),
+				loginMethod: "Service.LoginHandler",
+			}
+
+			servicePeers = append(servicePeers, adminPeer{
+				addr:    ep.Host,
+				svcClnt: &remoteAdminClient{newAuthClient(&cfg)},
+			})
+			seenAddr[ep.Host] = true
+		}
+	}
+
+	return servicePeers
+}
+
+// Initialize global adminPeer collection.
+func initGlobalAdminPeers(eps []*url.URL) {
+	globalAdminPeers = makeAdminPeers(eps)
+}
+
+// invokeServiceCmd - Invoke Stop/Restart command.
+func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
+	switch cmd {
+	case serviceStop:
+		err = cp.svcClnt.Stop()
+	case serviceRestart:
+		err = cp.svcClnt.Restart()
+	}
+	return err
+}
+
+// sendServiceCmd - Invoke Stop/Restart command on remote peers
+// adminPeer followed by on the local peer.
+func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
+	// Send service command like stop or restart to all remote nodes and finally run on local node.
+	errs := make([]error, len(cps))
+	var wg sync.WaitGroup
+	remotePeers := cps[1:]
+	for i := range remotePeers {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			errs[idx] = invokeServiceCmd(remotePeers[idx], cmd)
+		}(i)
+	}
+	wg.Wait()
+	errs[0] = invokeServiceCmd(cps[0], cmd)
+}

cmd/admin-rpc-server.go

@@ -0,0 +1,63 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/rpc"
+
+	router "github.com/gorilla/mux"
+)
+
+const servicePath = "/admin/service"
+
+// serviceCmd - exports RPC methods for service status, stop and
+// restart commands.
+type serviceCmd struct {
+	loginServer
+}
+
+// Shutdown - Shutdown this instance of minio server.
+func (s *serviceCmd) Shutdown(args *GenericArgs, reply *GenericReply) error {
+	if !isRPCTokenValid(args.Token) {
+		return errInvalidToken
+	}
+	globalServiceSignalCh <- serviceStop
+	return nil
+}
+
+// Restart - Restart this instance of minio server.
+func (s *serviceCmd) Restart(args *GenericArgs, reply *GenericReply) error {
+	if !isRPCTokenValid(args.Token) {
+		return errInvalidToken
+	}
+	globalServiceSignalCh <- serviceRestart
+	return nil
+}
+
+// registerAdminRPCRouter - registers RPC methods for service status,
+// stop and restart commands.
+func registerAdminRPCRouter(mux *router.Router) error {
+	adminRPCHandler := &serviceCmd{}
+	adminRPCServer := rpc.NewServer()
+	err := adminRPCServer.RegisterName("Service", adminRPCHandler)
+	if err != nil {
+		return traceError(err)
+	}
+	adminRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter()
+	adminRouter.Path(servicePath).Handler(adminRPCServer)
+	return nil
+}

cmd/admin-rpc-server_test.go

@@ -0,0 +1,86 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"testing"
+	"time"
+)
+
+func testAdminCmd(cmd cmdType, t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Failed to create test config - %v", err)
+	}
+	defer removeAll(rootPath)
+
+	adminServer := serviceCmd{}
+	creds := serverConfig.GetCredential()
+	reply := RPCLoginReply{}
+	args := RPCLoginArgs{Username: creds.AccessKeyID, Password: creds.SecretAccessKey}
+	err = adminServer.LoginHandler(&args, &reply)
+	if err != nil {
+		t.Fatalf("Failed to login to admin server - %v", err)
+	}
+
+	go func() {
+		// mocking signal receiver
+		<-globalServiceSignalCh
+	}()
+
+	validToken := reply.Token
+	timeNow := time.Now().UTC()
+	testCases := []struct {
+		ga          GenericArgs
+		expectedErr error
+	}{
+		// Valid case
+		{
+			ga:          GenericArgs{Token: validToken, Timestamp: timeNow},
+			expectedErr: nil,
+		},
+		// Invalid token
+		{
+			ga:          GenericArgs{Token: "invalidToken", Timestamp: timeNow},
+			expectedErr: errInvalidToken,
+		},
+	}
+
+	genReply := GenericReply{}
+	for i, test := range testCases {
+		switch cmd {
+		case stopCmd:
+			err = adminServer.Shutdown(&test.ga, &genReply)
+			if err != test.expectedErr {
+				t.Errorf("Test %d: Expected error %v but received %v", i+1, test.expectedErr, err)
+			}
+		case restartCmd:
+			err = adminServer.Restart(&test.ga, &genReply)
+			if err != test.expectedErr {
+				t.Errorf("Test %d: Expected error %v but received %v", i+1, test.expectedErr, err)
+			}
+		}
+	}
+}
+
+func TestAdminShutdown(t *testing.T) {
+	testAdminCmd(stopCmd, t)
+}
+
+func TestAdminRestart(t *testing.T) {
+	testAdminCmd(restartCmd, t)
+}

cmd/globals.go

@@ -85,6 +85,7 @@ var (
 	// CA root certificates, a nil value means system certs pool will be used
 	globalRootCAs *x509.CertPool
 
+	globalAdminPeers = adminPeers{}
 	// Add new variable global values here.
 )
 

cmd/routers.go

@@ -110,6 +110,12 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error)
 		registerDistXLRouters(mux, srvCmdConfig)
 	}
 
+	// Add Admin RPC router
+	err := registerAdminRPCRouter(mux)
+	if err != nil {
+		return nil, err
+	}
+
 	// Register web router when its enabled.
 	if globalIsBrowserEnabled {
 		if err := registerWebRouter(mux); err != nil {
@@ -117,6 +123,9 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error)
 		}
 	}
 
+	// Add Admin router.
+	registerAdminRouter(mux)
+
 	// Add API router.
 	registerAPIRouter(mux)
 

cmd/server-main.go

@@ -464,6 +464,9 @@ func serverMain(c *cli.Context) {
 	// Initialize S3 Peers inter-node communication
 	initGlobalS3Peers(endpoints)
 
+	// Initialize Admin Peers inter-node communication
+	initGlobalAdminPeers(endpoints)
+
 	// Start server, automatically configures TLS if certs are available.
 	go func(tls bool) {
 		var lerr error

cmd/service.go

@@ -41,7 +41,7 @@ var globalServiceDoneCh chan struct{}
 // Initialize service mutex once.
 func init() {
 	globalServiceDoneCh = make(chan struct{}, 1)
-	globalServiceSignalCh = make(chan serviceSignal, 1)
+	globalServiceSignalCh = make(chan serviceSignal)
 }
 
 // restartProcess starts a new process passing it the active fd's. It
@@ -80,36 +80,37 @@ func (m *ServerMux) handleServiceSignals() error {
 		globalServiceDoneCh <- struct{}{}
 	}
 
-	// Start listening on service signal. Monitor signals.
+	// Wait for SIGTERM in a go-routine.
 	trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)
+	go func(<-chan bool) {
+		<-trapCh
+		globalServiceSignalCh <- serviceStop
+	}(trapCh)
+
+	// Start listening on service signal. Monitor signals.
 	for {
-		select {
+		signal := <-globalServiceSignalCh
-		case <-trapCh:
+		switch signal {
-			// Initiate graceful stop.
+		case serviceStatus:
-			globalServiceSignalCh <- serviceStop
+			/// We don't do anything for this.
-		case signal := <-globalServiceSignalCh:
+		case serviceRestart:
-			switch signal {
+			if err := m.Close(); err != nil {
-			case serviceStatus:
+				errorIf(err, "Unable to close server gracefully")
-				/// We don't do anything for this.
+			}
-			case serviceRestart:
+			if err := restartProcess(); err != nil {
-				if err := m.Close(); err != nil {
+				errorIf(err, "Unable to restart the server.")
-					errorIf(err, "Unable to close server gracefully")
+			}
-				}
+			runExitFn(nil)
-				if err := restartProcess(); err != nil {
+		case serviceStop:
-					errorIf(err, "Unable to restart the server.")
+			if err := m.Close(); err != nil {
-				}
+				errorIf(err, "Unable to close server gracefully")
+			}
+			objAPI := newObjectLayerFn()
+			if objAPI == nil {
+				// Server not initialized yet, exit happily.
 				runExitFn(nil)
-			case serviceStop:
+			} else {
-				if err := m.Close(); err != nil {
+				runExitFn(objAPI.Shutdown())
-					errorIf(err, "Unable to close server gracefully")
-				}
-				objAPI := newObjectLayerFn()
-				if objAPI == nil {
-					// Server not initialized yet, exit happily.
-					runExitFn(nil)
-				} else {
-					runExitFn(objAPI.Shutdown())
-				}
 			}
 		}
 	}

docs/admin-api/service.md

@@ -0,0 +1,33 @@
+# Service REST API
+
+## Authentication
+- AWS signatureV4
+- We use "minio" as region. Here region is set only for signature calculation.
+
+## List of management APIs
+- Service
+  - Stop
+  - Restart
+  - Status
+
+- Locks
+  - List
+  - Clear
+
+- Healing
+
+### Service Management APIs
+* Stop
+  - POST /?service
+  - x-minio-operation: stop
+  - Response: On success 200
+
+* Restart
+  - POST /?service
+  - x-minio-operation: restart
+  - Response: On success 200
+
+* Status
+  - GET /?service
+  - x-minio-operation: status
+  - Response: On success 200, return json formatted StorageInfo object.