From fcb56d864c784f4e4ed9691224e3db36c60d9121 Mon Sep 17 00:00:00 2001 From: Nitish Tiwari Date: Mon, 31 Dec 2018 23:16:44 +0530 Subject: [PATCH] Add ServerDrivesPerfInfo() admin API (#6969) This is part of implementation for mc admin health command. The ServerDrivesPerfInfo() admin API returns read and write speed information for all the drives (local and remote) in a given Minio server deployment. Part of minio/mc#2606 --- cmd/admin-handlers.go | 52 ++++++++++++ cmd/admin-router.go | 5 ++ cmd/endpoint.go | 29 +++++++ cmd/notification.go | 25 ++++++ cmd/peer-rpc-client.go | 9 ++ cmd/peer-rpc-server.go | 11 +++ pkg/disk/disk.go | 106 +++++++++++++++++++++++- pkg/disk/helpers.go | 52 ++++++++++++ pkg/madmin/API.md | 19 ++++- pkg/madmin/examples/drives-perf-info.go | 44 ++++++++++ pkg/madmin/info-commands.go | 46 ++++++++++ 11 files changed, 396 insertions(+), 2 deletions(-) create mode 100644 pkg/disk/helpers.go create mode 100644 pkg/madmin/examples/drives-perf-info.go diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index be882823d..19780fede 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -35,6 +35,7 @@ import ( "github.com/gorilla/mux" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/auth" + "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/iam/policy" "github.com/minio/minio/pkg/madmin" @@ -284,6 +285,57 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque writeSuccessResponseJSON(w, jsonBytes) } +// ServerDrivesPerfInfo holds informantion about address, performance +// of all drives on one server. It also reports any errors if encountered +// while trying to reach this server. +type ServerDrivesPerfInfo struct { + Addr string `json:"addr"` + Error string `json:"error,omitempty"` + Perf []disk.Performance `json:"perf"` +} + +// PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType} +// ---------- +// Get all performance information based on input type +// Supported types = drive +func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PerfInfo") + + // Authenticate request + // Setting the region as empty so as the mc server info command is irrespective to the region. + adminAPIErr := checkAdminRequestAuthType(ctx, r, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(w, adminAPIErr, r.URL) + return + } + + vars := mux.Vars(r) + perfType := vars["perfType"] + + if perfType == "drive" { + // Get drive performance details from local server's drive(s) + dp := localEndpointsPerf(globalEndpoints) + + // Notify all other Minio peers to report drive performance numbers + dps := globalNotificationSys.DrivePerfInfo() + dps = append(dps, dp) + + // Marshal API response + jsonBytes, err := json.Marshal(dps) + if err != nil { + writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL) + return + } + + // Reply with performance information (across nodes in a + // distributed setup) as json. + writeSuccessResponseJSON(w, jsonBytes) + } else { + writeErrorResponseJSON(w, ErrNotImplemented, r.URL) + } + return +} + // StartProfilingResult contains the status of the starting // profiling action in a given server type StartProfilingResult struct { diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 7f6562780..068630073 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -60,6 +60,11 @@ func registerAdminRouter(router *mux.Router, enableIAM bool) { adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler)) adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler)) adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler)) + + /// Health operations + + // Performance command - return performance details based on input type + adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}") } // Profiling operations diff --git a/cmd/endpoint.go b/cmd/endpoint.go index a25003478..7bfaa6615 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -29,6 +29,7 @@ import ( "github.com/minio/minio-go/pkg/set" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/mountinfo" ) @@ -197,6 +198,34 @@ func (endpoints EndpointList) GetString(i int) string { return endpoints[i].String() } +// localEndpointsPerf - returns ServerDrivesPerfInfo for only the +// local endpoints from given list of endpoints +func localEndpointsPerf(endpoints EndpointList) ServerDrivesPerfInfo { + var dps []disk.Performance + var addr string + for _, endpoint := range endpoints { + // Only proceed for local endpoints + if endpoint.IsLocal { + addr = GetLocalPeer(endpoints) + if _, err := os.Stat(endpoint.Path); err != nil { + // Since this drive is not available, add relevant details and proceed + dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()}) + continue + } + tempObj := mustGetUUID() + fsPath := pathJoin(endpoint.Path, minioMetaTmpBucket, tempObj) + dp := disk.GetPerformance(fsPath) + dp.Path = endpoint.Path + dps = append(dps, dp) + } + } + + return ServerDrivesPerfInfo{ + Addr: addr, + Perf: dps, + } +} + // NewEndpointList - returns new endpoint list based on input args. func NewEndpointList(args ...string) (endpoints EndpointList, err error) { var endpointType EndpointType diff --git a/cmd/notification.go b/cmd/notification.go index ab7cd50bc..24938a329 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -512,6 +512,31 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr { return sys.send(args.BucketName, args.ToEvent(), targetIDs...) } +// DrivePerfInfo - Drive speed (read and write) information +func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo { + reply := make([]ServerDrivesPerfInfo, len(sys.peerRPCClientMap)) + var wg sync.WaitGroup + var i int + for addr, client := range sys.peerRPCClientMap { + wg.Add(1) + go func(addr xnet.Host, client *PeerRPCClient, idx int) { + defer wg.Done() + di, err := client.DrivePerfInfo() + if err != nil { + reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr.String()) + ctx := logger.SetReqInfo(context.Background(), reqInfo) + logger.LogIf(ctx, err) + di.Addr = addr.String() + di.Error = err.Error() + } + reply[idx] = di + }(addr, client, i) + i++ + } + wg.Wait() + return reply +} + // NewNotificationSys - creates new notification system object. func NewNotificationSys(config *serverConfig, endpoints EndpointList) *NotificationSys { targetList := getNotificationTargets(config) diff --git a/cmd/peer-rpc-client.go b/cmd/peer-rpc-client.go index 8e73b24f2..ea8d2909c 100644 --- a/cmd/peer-rpc-client.go +++ b/cmd/peer-rpc-client.go @@ -141,6 +141,15 @@ func (rpcClient *PeerRPCClient) LoadCredentials() error { return rpcClient.Call(peerServiceName+".LoadCredentials", &args, &reply) } +// DrivePerfInfo - returns drive performance info for remote server. +func (rpcClient *PeerRPCClient) DrivePerfInfo() (ServerDrivesPerfInfo, error) { + args := AuthArgs{} + var reply ServerDrivesPerfInfo + + err := rpcClient.Call(peerServiceName+".DrivePerfInfo", &args, &reply) + return reply, err +} + // NewPeerRPCClient - returns new peer RPC client. func NewPeerRPCClient(host *xnet.Host) (*PeerRPCClient, error) { scheme := "http" diff --git a/cmd/peer-rpc-server.go b/cmd/peer-rpc-server.go index c243996c4..e0b40af9f 100644 --- a/cmd/peer-rpc-server.go +++ b/cmd/peer-rpc-server.go @@ -238,6 +238,17 @@ func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidRepl return globalConfigSys.Load(newObjectLayerFn()) } +// DrivePerfInfo - handles drive performance RPC call +func (receiver *peerRPCReceiver) DrivePerfInfo(args *AuthArgs, reply *ServerDrivesPerfInfo) error { + objAPI := newObjectLayerFn() + if objAPI == nil { + return errServerNotInitialized + } + + *reply = localEndpointsPerf(globalEndpoints) + return nil +} + // NewPeerRPCServer - returns new peer RPC server. func NewPeerRPCServer() (*xrpc.Server, error) { rpcServer := xrpc.NewServer() diff --git a/pkg/disk/disk.go b/pkg/disk/disk.go index 04664b3d4..7660cce5b 100644 --- a/pkg/disk/disk.go +++ b/pkg/disk/disk.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015 Minio, Inc. + * Minio Cloud Storage, (C) 2018 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,26 @@ package disk +import ( + "bytes" + "crypto/rand" + "errors" + "os" + "path" + "strconv" + "time" + + humanize "github.com/dustin/go-humanize" +) + +// file size for performance read and write checks +const randBufSize = 1 * humanize.KiByte +const randParts = 1024 +const fileSize = randParts * randBufSize + +// Total count of read / write iteration for performance measurement +const iterations = 10 + // Info stat fs struct is container which holds following values // Total - total size of the volume / disk // Free - free size of the volume / disk @@ -32,3 +52,87 @@ type Info struct { // Usage is calculated per tenant. Usage uint64 } + +// Performance holds informantion about read and write speed of a disk +type Performance struct { + Path string `json:"path"` + Error string `json:"error,omitempty"` + WriteSpeed float64 `json:"writeSpeed"` + ReadSpeed float64 `json:"readSpeed"` +} + +// GetPerformance returns given disk's read and write performance +func GetPerformance(path string) Performance { + perf := Performance{} + write, read, err := doPerfMeasure(path) + if err != nil { + perf.Error = err.Error() + return perf + } + perf.WriteSpeed = write + perf.ReadSpeed = read + return perf +} + +// Calculate the write and read performance - write and read 10 tmp (1 MiB) +// files and find the average time taken (Bytes / Sec) +func doPerfMeasure(fsPath string) (write, read float64, err error) { + var count int + var totalWriteElapsed time.Duration + var totalReadElapsed time.Duration + + defer os.RemoveAll(fsPath) + + randBuf := make([]byte, randBufSize) + rand.Read(randBuf) + buf := bytes.Repeat(randBuf, randParts) + + // create the enclosing directory + err = os.MkdirAll(fsPath, 0777) + if err != nil { + return 0, 0, err + } + + for count = 1; count <= iterations; count++ { + fsTempObjPath := path.Join(fsPath, strconv.Itoa(count)) + + // Write performance calculation + writeStart := time.Now() + n, err := writeFile(fsTempObjPath, buf) + + if err != nil { + return 0, 0, err + } + if n != fileSize { + return 0, 0, errors.New("Could not write temporary data to disk") + } + + writeElapsed := time.Since(writeStart) + totalWriteElapsed += writeElapsed + + // Read performance calculation + readStart := time.Now() + n, err = readFile(fsTempObjPath, buf) + + if err != nil { + return 0, 0, err + } + if n != fileSize { + return 0, 0, errors.New("Could not read temporary data from disk") + } + + readElapsed := time.Since(readStart) + totalReadElapsed += readElapsed + } + // Average time spent = total time elapsed / number of writes + avgWriteTime := totalWriteElapsed.Seconds() / float64(count) + // Write perf = fileSize (in Bytes) / average time spent writing (in seconds) + write = fileSize / avgWriteTime + + // Average time spent = total time elapsed / number of writes + avgReadTime := totalReadElapsed.Seconds() / float64(count) + // read perf = fileSize (in Bytes) / average time spent reading (in seconds) + read = fileSize / avgReadTime + + return write, read, nil +} diff --git a/pkg/disk/helpers.go b/pkg/disk/helpers.go new file mode 100644 index 000000000..c0da6da0e --- /dev/null +++ b/pkg/disk/helpers.go @@ -0,0 +1,52 @@ +/* + * Minio Cloud Storage, (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package disk + +import ( + "os" +) + +func readFile(path string, buf []byte) (int, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + + n, err := f.Read(buf) + if err != nil { + return 0, err + } + + return n, nil +} + +func writeFile(path string, data []byte) (int, error) { + f, err := os.Create(path) + if err != nil { + return 0, err + } + defer f.Close() + + n, err := f.Write(data) + if err != nil { + return 0, err + } + + f.Sync() + return n, nil +} diff --git a/pkg/madmin/API.md b/pkg/madmin/API.md index 04e8642fc..0f8b1ecb5 100644 --- a/pkg/madmin/API.md +++ b/pkg/madmin/API.md @@ -39,7 +39,7 @@ func main() { | Service operations | Info operations | Healing operations | Config operations | IAM operations | Misc | |:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:------------------------------------|:------------------------------------| | [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`AddUser`](#AddUser) | [`SetAdminCredentials`](#SetAdminCredentials) | -| [`ServiceSendAction`](#ServiceSendAction) | | | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) | +| [`ServiceSendAction`](#ServiceSendAction) | [`ServerDrivesPerfInfo`](#ServerDrivesPerfInfo) | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) | | | | | [`GetConfigKeys`](#GetConfigKeys) | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) | | | | | [`SetConfigKeys`](#SetConfigKeys) | [`AddCannedPolicy`](#AddCannedPolicy) | | @@ -204,6 +204,23 @@ Fetches information for all cluster nodes, such as server properties, storage in ``` + +### ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error) + +Fetches drive performance information for all cluster nodes. Returned value is in Bytes/s. + +| Param | Type | Description | +|---|---|---| +|`di.Addr` | _string_ | Address of the server the following information is retrieved from. | +|`di.Error` | _string _ | Errors (if any) encountered while reaching this node | +|`di.DrivesPerf` | _disk.Performance_ | Path of the drive mount on above server and read, write speed. | + +| Param | Type | Description | +|---|---|---| +|`disk.Performance.Path` | _string_ | Path of drive mount. | +|`disk.Performance.Error` | _string_ | Error (if any) encountered while accessing this drive. | +|`disk.Performance.WriteSpeed` | _float64_ | Write speed on above path in Bytes/s. | +|`disk.Performance.ReadSpeed` | _float64_ | Read speed on above path in Bytes/s. | ## 6. Heal operations diff --git a/pkg/madmin/examples/drives-perf-info.go b/pkg/madmin/examples/drives-perf-info.go new file mode 100644 index 000000000..f287e2ee9 --- /dev/null +++ b/pkg/madmin/examples/drives-perf-info.go @@ -0,0 +1,44 @@ +// +build ignore + +/* + * Minio Cloud Storage, (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "log" + + "github.com/minio/minio/pkg/madmin" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise. + // New returns an Minio Admin client object. + madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + st, err := madmClnt.ServerDrivesPerfInfo() + if err != nil { + log.Fatalln(err) + } + log.Println(st) +} diff --git a/pkg/madmin/info-commands.go b/pkg/madmin/info-commands.go index 1a0d71af6..e2920341c 100644 --- a/pkg/madmin/info-commands.go +++ b/pkg/madmin/info-commands.go @@ -21,7 +21,10 @@ import ( "encoding/json" "io/ioutil" "net/http" + "net/url" "time" + + "github.com/minio/minio/pkg/disk" ) // BackendType - represents different backend types. @@ -147,3 +150,46 @@ func (adm *AdminClient) ServerInfo() ([]ServerInfo, error) { return serversInfo, nil } + +// ServerDrivesPerfInfo holds informantion about address and write speed of +// all drives in a single server node +type ServerDrivesPerfInfo struct { + Addr string `json:"addr"` + Error string `json:"error,omitempty"` + Perf []disk.Performance `json:"perf"` +} + +// ServerDrivesPerfInfo - Returns drive's read and write performance information +func (adm *AdminClient) ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error) { + v := url.Values{} + v.Set("perfType", string("drive")) + resp, err := adm.executeMethod("GET", requestData{ + relPath: "/v1/performance", + queryValues: v, + }) + + defer closeResponse(resp) + if err != nil { + return nil, err + } + + // Check response http status code + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp) + } + + // Unmarshal the server's json response + var info []ServerDrivesPerfInfo + + respBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(respBytes, &info) + if err != nil { + return nil, err + } + + return info, nil +}