From b2f920a8687706d495282af0b12e665a96412c62 Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Fri, 16 Dec 2016 11:56:15 +0530 Subject: [PATCH] Add service API handler stubs for status, stop and restart (#3417) --- cmd/admin-handlers.go | 64 ++++++++++++++ cmd/admin-handlers_test.go | 164 +++++++++++++++++++++++++++++++++++ cmd/admin-router.go | 40 +++++++++ cmd/admin-rpc-client.go | 163 ++++++++++++++++++++++++++++++++++ cmd/admin-rpc-server.go | 63 ++++++++++++++ cmd/admin-rpc-server_test.go | 86 ++++++++++++++++++ cmd/globals.go | 1 + cmd/routers.go | 9 ++ cmd/server-main.go | 3 + cmd/service.go | 57 ++++++------ docs/admin-api/service.md | 33 +++++++ 11 files changed, 655 insertions(+), 28 deletions(-) create mode 100644 cmd/admin-handlers.go create mode 100644 cmd/admin-handlers_test.go create mode 100644 cmd/admin-router.go create mode 100644 cmd/admin-rpc-client.go create mode 100644 cmd/admin-rpc-server.go create mode 100644 cmd/admin-rpc-server_test.go create mode 100644 docs/admin-api/service.md diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go new file mode 100644 index 000000000..a4605fac9 --- /dev/null +++ b/cmd/admin-handlers.go @@ -0,0 +1,64 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "encoding/json" + "net/http" +) + +const ( + minioAdminOpHeader = "X-Minio-Operation" +) + +func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) { + adminAPIErr := checkRequestAuthType(r, "", "", "") + if adminAPIErr != ErrNone { + writeErrorResponse(w, r, adminAPIErr, r.URL.Path) + return + } + storageInfo := newObjectLayerFn().StorageInfo() + jsonBytes, err := json.Marshal(storageInfo) + if err != nil { + writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path) + errorIf(err, "Failed to marshal storage info into json.") + } + w.WriteHeader(http.StatusOK) + writeSuccessResponse(w, jsonBytes) +} + +func (adminAPI adminAPIHandlers) ServiceStopHandler(w http.ResponseWriter, r *http.Request) { + adminAPIErr := checkRequestAuthType(r, "", "", "") + if adminAPIErr != ErrNone { + writeErrorResponse(w, r, adminAPIErr, r.URL.Path) + return + } + // Reply to the client before stopping minio server. + w.WriteHeader(http.StatusOK) + sendServiceCmd(globalAdminPeers, serviceStop) +} + +func (adminAPI adminAPIHandlers) ServiceRestartHandler(w http.ResponseWriter, r *http.Request) { + adminAPIErr := checkRequestAuthType(r, "", "", "") + if adminAPIErr != ErrNone { + writeErrorResponse(w, r, adminAPIErr, r.URL.Path) + return + } + // Reply to the client before restarting minio server. + w.WriteHeader(http.StatusOK) + sendServiceCmd(globalAdminPeers, serviceRestart) +} diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go new file mode 100644 index 000000000..0e012626f --- /dev/null +++ b/cmd/admin-handlers_test.go @@ -0,0 +1,164 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + router "github.com/gorilla/mux" +) + +type cmdType int + +const ( + statusCmd cmdType = iota + stopCmd + restartCmd +) + +func (c cmdType) String() string { + switch c { + case statusCmd: + return "status" + case stopCmd: + return "stop" + case restartCmd: + return "restart" + } + return "" +} + +func (c cmdType) apiMethod() string { + switch c { + case statusCmd: + return "GET" + case stopCmd: + return "POST" + case restartCmd: + return "POST" + } + return "GET" +} + +func (c cmdType) toServiceSignal() serviceSignal { + switch c { + case statusCmd: + return serviceStatus + case stopCmd: + return serviceStop + case restartCmd: + return serviceRestart + } + return serviceStatus +} + +func testServiceSignalReceiver(cmd cmdType, t *testing.T) { + expectedCmd := cmd.toServiceSignal() + serviceCmd := <-globalServiceSignalCh + if serviceCmd != expectedCmd { + t.Errorf("Expected service command %v but received %v", expectedCmd, serviceCmd) + } +} + +func getAdminCmdRequest(cmd cmdType, cred credential) (*http.Request, error) { + req, err := newTestRequest(cmd.apiMethod(), "/?service", 0, nil) + if err != nil { + return nil, err + } + req.Header.Set(minioAdminOpHeader, cmd.String()) + err = signRequestV4(req, cred.AccessKeyID, cred.SecretAccessKey) + if err != nil { + return nil, err + } + return req, nil +} + +func testServicesCmdHandler(cmd cmdType, t *testing.T) { + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Unable to initialize server config. %s", err) + } + defer removeAll(rootPath) + + // Initialize admin peers to make admin RPC calls. + eps, err := parseStorageEndpoints([]string{"http://localhost"}) + if err != nil { + t.Fatalf("Failed to parse storage end point - %v", err) + } + initGlobalAdminPeers(eps) + + if cmd == statusCmd { + // Initializing objectLayer and corresponding + // []StorageAPI since DiskInfo() method requires it. + objLayer, fsDir, fsErr := prepareFS() + if fsErr != nil { + t.Fatalf("failed to initialize XL based object layer - %v.", fsErr) + } + defer removeRoots([]string{fsDir}) + globalObjLayerMutex.Lock() + globalObjectAPI = objLayer + globalObjLayerMutex.Unlock() + } + + // Setting up a go routine to simulate ServerMux's + // handleServiceSignals for stop and restart commands. + switch cmd { + case stopCmd, restartCmd: + go testServiceSignalReceiver(cmd, t) + } + credentials := serverConfig.GetCredential() + adminRouter := router.NewRouter() + registerAdminRouter(adminRouter) + + rec := httptest.NewRecorder() + req, err := getAdminCmdRequest(cmd, credentials) + if err != nil { + t.Fatalf("Failed to build service status request %v", err) + } + adminRouter.ServeHTTP(rec, req) + + if cmd == statusCmd { + expectedInfo := newObjectLayerFn().StorageInfo() + receivedInfo := StorageInfo{} + if jsonErr := json.Unmarshal(rec.Body.Bytes(), &receivedInfo); jsonErr != nil { + t.Errorf("Failed to unmarshal StorageInfo - %v", jsonErr) + } + if expectedInfo != receivedInfo { + t.Errorf("Expected storage info and received storage info differ, %v %v", expectedInfo, receivedInfo) + } + } + + if rec.Code != http.StatusOK { + t.Errorf("Expected to receive %d status code but received %d", + http.StatusOK, rec.Code) + } +} + +func TestServiceStatusHandler(t *testing.T) { + testServicesCmdHandler(statusCmd, t) +} + +func TestServiceStopHandler(t *testing.T) { + testServicesCmdHandler(stopCmd, t) +} + +func TestServiceRestartHandler(t *testing.T) { + testServicesCmdHandler(restartCmd, t) +} diff --git a/cmd/admin-router.go b/cmd/admin-router.go new file mode 100644 index 000000000..60b5c3aeb --- /dev/null +++ b/cmd/admin-router.go @@ -0,0 +1,40 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import router "github.com/gorilla/mux" + +// adminAPIHandlers provides HTTP handlers for Minio admin API. +type adminAPIHandlers struct { +} + +// registerAdminRouter - Add handler functions for each service REST API routes. +func registerAdminRouter(mux *router.Router) { + + adminAPI := adminAPIHandlers{} + // Admin router + adminRouter := mux.NewRoute().PathPrefix("/").Subrouter() + + /// Admin operations + + // Service status + adminRouter.Methods("GET").Queries("service", "").Headers(minioAdminOpHeader, "status").HandlerFunc(adminAPI.ServiceStatusHandler) + // Service stop + adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "stop").HandlerFunc(adminAPI.ServiceStopHandler) + // Service restart + adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "restart").HandlerFunc(adminAPI.ServiceRestartHandler) +} diff --git a/cmd/admin-rpc-client.go b/cmd/admin-rpc-client.go new file mode 100644 index 000000000..324ecda7b --- /dev/null +++ b/cmd/admin-rpc-client.go @@ -0,0 +1,163 @@ +/* + * Minio Cloud Storage, (C) 2014-2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "net/rpc" + "net/url" + "path" + "sync" +) + +// localAdminClient - represents admin operation to be executed locally. +type localAdminClient struct { +} + +// remoteAdminClient - represents admin operation to be executed +// remotely, via RPC. +type remoteAdminClient struct { + *AuthRPCClient +} + +// stopRestarter - abstracts stop and restart operations for both +// local and remote execution. +type stopRestarter interface { + Stop() error + Restart() error +} + +// Stop - Sends a message over channel to the go-routine responsible +// for stopping the process. +func (lc localAdminClient) Stop() error { + globalServiceSignalCh <- serviceStop + return nil +} + +// Restart - Sends a message over channel to the go-routine +// responsible for restarting the process. +func (lc localAdminClient) Restart() error { + globalServiceSignalCh <- serviceRestart + return nil +} + +// Stop - Sends stop command to remote server via RPC. +func (rc remoteAdminClient) Stop() error { + args := GenericArgs{} + reply := GenericReply{} + err := rc.Call("Service.Shutdown", &args, &reply) + if err != nil && err == rpc.ErrShutdown { + rc.Close() + } + return err +} + +// Restart - Sends restart command to remote server via RPC. +func (rc remoteAdminClient) Restart() error { + args := GenericArgs{} + reply := GenericReply{} + err := rc.Call("Service.Restart", &args, &reply) + if err != nil && err == rpc.ErrShutdown { + rc.Close() + } + return err +} + +// adminPeer - represents an entity that implements Stop and Restart methods. +type adminPeer struct { + addr string + svcClnt stopRestarter +} + +// type alias for a collection of adminPeer. +type adminPeers []adminPeer + +// makeAdminPeers - helper function to construct a collection of adminPeer. +func makeAdminPeers(eps []*url.URL) adminPeers { + var servicePeers []adminPeer + + // map to store peers that are already added to ret + seenAddr := make(map[string]bool) + + // add local (self) as peer in the array + servicePeers = append(servicePeers, adminPeer{ + globalMinioAddr, + localAdminClient{}, + }) + seenAddr[globalMinioAddr] = true + + // iterate over endpoints to find new remote peers and add + // them to ret. + for _, ep := range eps { + if ep.Host == "" { + continue + } + + // Check if the remote host has been added already + if !seenAddr[ep.Host] { + cfg := authConfig{ + accessKey: serverConfig.GetCredential().AccessKeyID, + secretKey: serverConfig.GetCredential().SecretAccessKey, + address: ep.Host, + secureConn: isSSL(), + path: path.Join(reservedBucket, servicePath), + loginMethod: "Service.LoginHandler", + } + + servicePeers = append(servicePeers, adminPeer{ + addr: ep.Host, + svcClnt: &remoteAdminClient{newAuthClient(&cfg)}, + }) + seenAddr[ep.Host] = true + } + } + + return servicePeers +} + +// Initialize global adminPeer collection. +func initGlobalAdminPeers(eps []*url.URL) { + globalAdminPeers = makeAdminPeers(eps) +} + +// invokeServiceCmd - Invoke Stop/Restart command. +func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) { + switch cmd { + case serviceStop: + err = cp.svcClnt.Stop() + case serviceRestart: + err = cp.svcClnt.Restart() + } + return err +} + +// sendServiceCmd - Invoke Stop/Restart command on remote peers +// adminPeer followed by on the local peer. +func sendServiceCmd(cps adminPeers, cmd serviceSignal) { + // Send service command like stop or restart to all remote nodes and finally run on local node. + errs := make([]error, len(cps)) + var wg sync.WaitGroup + remotePeers := cps[1:] + for i := range remotePeers { + wg.Add(1) + go func(idx int) { + defer wg.Done() + errs[idx] = invokeServiceCmd(remotePeers[idx], cmd) + }(i) + } + wg.Wait() + errs[0] = invokeServiceCmd(cps[0], cmd) +} diff --git a/cmd/admin-rpc-server.go b/cmd/admin-rpc-server.go new file mode 100644 index 000000000..3948901d1 --- /dev/null +++ b/cmd/admin-rpc-server.go @@ -0,0 +1,63 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "net/rpc" + + router "github.com/gorilla/mux" +) + +const servicePath = "/admin/service" + +// serviceCmd - exports RPC methods for service status, stop and +// restart commands. +type serviceCmd struct { + loginServer +} + +// Shutdown - Shutdown this instance of minio server. +func (s *serviceCmd) Shutdown(args *GenericArgs, reply *GenericReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + globalServiceSignalCh <- serviceStop + return nil +} + +// Restart - Restart this instance of minio server. +func (s *serviceCmd) Restart(args *GenericArgs, reply *GenericReply) error { + if !isRPCTokenValid(args.Token) { + return errInvalidToken + } + globalServiceSignalCh <- serviceRestart + return nil +} + +// registerAdminRPCRouter - registers RPC methods for service status, +// stop and restart commands. +func registerAdminRPCRouter(mux *router.Router) error { + adminRPCHandler := &serviceCmd{} + adminRPCServer := rpc.NewServer() + err := adminRPCServer.RegisterName("Service", adminRPCHandler) + if err != nil { + return traceError(err) + } + adminRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter() + adminRouter.Path(servicePath).Handler(adminRPCServer) + return nil +} diff --git a/cmd/admin-rpc-server_test.go b/cmd/admin-rpc-server_test.go new file mode 100644 index 000000000..99832642a --- /dev/null +++ b/cmd/admin-rpc-server_test.go @@ -0,0 +1,86 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "testing" + "time" +) + +func testAdminCmd(cmd cmdType, t *testing.T) { + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatalf("Failed to create test config - %v", err) + } + defer removeAll(rootPath) + + adminServer := serviceCmd{} + creds := serverConfig.GetCredential() + reply := RPCLoginReply{} + args := RPCLoginArgs{Username: creds.AccessKeyID, Password: creds.SecretAccessKey} + err = adminServer.LoginHandler(&args, &reply) + if err != nil { + t.Fatalf("Failed to login to admin server - %v", err) + } + + go func() { + // mocking signal receiver + <-globalServiceSignalCh + }() + + validToken := reply.Token + timeNow := time.Now().UTC() + testCases := []struct { + ga GenericArgs + expectedErr error + }{ + // Valid case + { + ga: GenericArgs{Token: validToken, Timestamp: timeNow}, + expectedErr: nil, + }, + // Invalid token + { + ga: GenericArgs{Token: "invalidToken", Timestamp: timeNow}, + expectedErr: errInvalidToken, + }, + } + + genReply := GenericReply{} + for i, test := range testCases { + switch cmd { + case stopCmd: + err = adminServer.Shutdown(&test.ga, &genReply) + if err != test.expectedErr { + t.Errorf("Test %d: Expected error %v but received %v", i+1, test.expectedErr, err) + } + case restartCmd: + err = adminServer.Restart(&test.ga, &genReply) + if err != test.expectedErr { + t.Errorf("Test %d: Expected error %v but received %v", i+1, test.expectedErr, err) + } + } + } +} + +func TestAdminShutdown(t *testing.T) { + testAdminCmd(stopCmd, t) +} + +func TestAdminRestart(t *testing.T) { + testAdminCmd(restartCmd, t) +} diff --git a/cmd/globals.go b/cmd/globals.go index 96ae4a9e7..8e039597f 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -85,6 +85,7 @@ var ( // CA root certificates, a nil value means system certs pool will be used globalRootCAs *x509.CertPool + globalAdminPeers = adminPeers{} // Add new variable global values here. ) diff --git a/cmd/routers.go b/cmd/routers.go index 6f03c480f..837180bee 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -110,6 +110,12 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) registerDistXLRouters(mux, srvCmdConfig) } + // Add Admin RPC router + err := registerAdminRPCRouter(mux) + if err != nil { + return nil, err + } + // Register web router when its enabled. if globalIsBrowserEnabled { if err := registerWebRouter(mux); err != nil { @@ -117,6 +123,9 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) } } + // Add Admin router. + registerAdminRouter(mux) + // Add API router. registerAPIRouter(mux) diff --git a/cmd/server-main.go b/cmd/server-main.go index fe41c5562..a752a15c1 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -464,6 +464,9 @@ func serverMain(c *cli.Context) { // Initialize S3 Peers inter-node communication initGlobalS3Peers(endpoints) + // Initialize Admin Peers inter-node communication + initGlobalAdminPeers(endpoints) + // Start server, automatically configures TLS if certs are available. go func(tls bool) { var lerr error diff --git a/cmd/service.go b/cmd/service.go index ea7aafc9c..d0af1c0d4 100644 --- a/cmd/service.go +++ b/cmd/service.go @@ -41,7 +41,7 @@ var globalServiceDoneCh chan struct{} // Initialize service mutex once. func init() { globalServiceDoneCh = make(chan struct{}, 1) - globalServiceSignalCh = make(chan serviceSignal, 1) + globalServiceSignalCh = make(chan serviceSignal) } // restartProcess starts a new process passing it the active fd's. It @@ -80,36 +80,37 @@ func (m *ServerMux) handleServiceSignals() error { globalServiceDoneCh <- struct{}{} } - // Start listening on service signal. Monitor signals. + // Wait for SIGTERM in a go-routine. trapCh := signalTrap(os.Interrupt, syscall.SIGTERM) + go func(<-chan bool) { + <-trapCh + globalServiceSignalCh <- serviceStop + }(trapCh) + + // Start listening on service signal. Monitor signals. for { - select { - case <-trapCh: - // Initiate graceful stop. - globalServiceSignalCh <- serviceStop - case signal := <-globalServiceSignalCh: - switch signal { - case serviceStatus: - /// We don't do anything for this. - case serviceRestart: - if err := m.Close(); err != nil { - errorIf(err, "Unable to close server gracefully") - } - if err := restartProcess(); err != nil { - errorIf(err, "Unable to restart the server.") - } + signal := <-globalServiceSignalCh + switch signal { + case serviceStatus: + /// We don't do anything for this. + case serviceRestart: + if err := m.Close(); err != nil { + errorIf(err, "Unable to close server gracefully") + } + if err := restartProcess(); err != nil { + errorIf(err, "Unable to restart the server.") + } + runExitFn(nil) + case serviceStop: + if err := m.Close(); err != nil { + errorIf(err, "Unable to close server gracefully") + } + objAPI := newObjectLayerFn() + if objAPI == nil { + // Server not initialized yet, exit happily. runExitFn(nil) - case serviceStop: - if err := m.Close(); err != nil { - errorIf(err, "Unable to close server gracefully") - } - objAPI := newObjectLayerFn() - if objAPI == nil { - // Server not initialized yet, exit happily. - runExitFn(nil) - } else { - runExitFn(objAPI.Shutdown()) - } + } else { + runExitFn(objAPI.Shutdown()) } } } diff --git a/docs/admin-api/service.md b/docs/admin-api/service.md new file mode 100644 index 000000000..bc1e580ab --- /dev/null +++ b/docs/admin-api/service.md @@ -0,0 +1,33 @@ +# Service REST API + +## Authentication +- AWS signatureV4 +- We use "minio" as region. Here region is set only for signature calculation. + +## List of management APIs +- Service + - Stop + - Restart + - Status + +- Locks + - List + - Clear + +- Healing + +### Service Management APIs +* Stop + - POST /?service + - x-minio-operation: stop + - Response: On success 200 + +* Restart + - POST /?service + - x-minio-operation: restart + - Response: On success 200 + +* Status + - GET /?service + - x-minio-operation: status + - Response: On success 200, return json formatted StorageInfo object.