Add uptime to ServiceStatus (#3690)

master
Krishnan Parthasarathi 8 years ago committed by Harshavardhana
parent 7547f3c8a3
commit ce9aa2f2b2
  1. 19
      cmd/admin-handlers.go
  2. 7
      cmd/admin-handlers_test.go
  3. 86
      cmd/admin-rpc-client.go
  4. 27
      cmd/admin-rpc-server.go
  5. 4
      cmd/globals.go
  6. 4
      cmd/server-main.go
  7. 2
      pkg/madmin/service-commands.go

@ -19,7 +19,6 @@ package cmd
import ( import (
"encoding/json" "encoding/json"
"encoding/xml" "encoding/xml"
"fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
@ -54,8 +53,8 @@ type ServerVersion struct {
// ServerStatus - contains the response of service status API // ServerStatus - contains the response of service status API
type ServerStatus struct { type ServerStatus struct {
StorageInfo StorageInfo `json:"storageInfo"`
ServerVersion ServerVersion `json:"serverVersion"` ServerVersion ServerVersion `json:"serverVersion"`
Uptime time.Duration `json:"uptime"`
} }
// ServiceStatusHandler - GET /?service // ServiceStatusHandler - GET /?service
@ -70,15 +69,22 @@ func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *
return return
} }
// Fetch storage backend information
storageInfo := newObjectLayerFn().StorageInfo()
// Fetch server version // Fetch server version
serverVersion := ServerVersion{Version: Version, CommitID: CommitID} serverVersion := ServerVersion{Version: Version, CommitID: CommitID}
// Fetch uptimes from all peers. This may fail to due to lack
// of read-quorum availability.
uptime, err := getPeerUptimes(globalAdminPeers)
if err != nil {
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
errorIf(err, "Possibly failed to get uptime from majority of servers.")
return
}
// Create API response // Create API response
serverStatus := ServerStatus{ serverStatus := ServerStatus{
StorageInfo: storageInfo,
ServerVersion: serverVersion, ServerVersion: serverVersion,
Uptime: uptime,
} }
// Marshal API response // Marshal API response
@ -542,7 +548,6 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
// Create a new set of storage instances to heal format.json. // Create a new set of storage instances to heal format.json.
bootstrapDisks, err := initStorageDisks(globalEndpoints) bootstrapDisks, err := initStorageDisks(globalEndpoints)
if err != nil { if err != nil {
fmt.Println(traceError(err))
writeErrorResponse(w, toAPIErrorCode(err), r.URL) writeErrorResponse(w, toAPIErrorCode(err), r.URL)
return return
} }
@ -550,7 +555,6 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
// Heal format.json on available storage. // Heal format.json on available storage.
err = healFormatXL(bootstrapDisks) err = healFormatXL(bootstrapDisks)
if err != nil { if err != nil {
fmt.Println(traceError(err))
writeErrorResponse(w, toAPIErrorCode(err), r.URL) writeErrorResponse(w, toAPIErrorCode(err), r.URL)
return return
} }
@ -558,7 +562,6 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
// Instantiate new object layer with newly formatted storage. // Instantiate new object layer with newly formatted storage.
newObjectAPI, err := newXLObjects(bootstrapDisks) newObjectAPI, err := newXLObjects(bootstrapDisks)
if err != nil { if err != nil {
fmt.Println(traceError(err))
writeErrorResponse(w, toAPIErrorCode(err), r.URL) writeErrorResponse(w, toAPIErrorCode(err), r.URL)
return return
} }

@ -25,6 +25,7 @@ import (
"net/http/httptest" "net/http/httptest"
"net/url" "net/url"
"testing" "testing"
"time"
router "github.com/gorilla/mux" router "github.com/gorilla/mux"
) )
@ -55,6 +56,9 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
return nil, xlErr return nil, xlErr
} }
// Initialize boot time
globalBootTime = time.Now().UTC()
// Set globalEndpoints for a single node XL setup. // Set globalEndpoints for a single node XL setup.
for _, xlDir := range xlDirs { for _, xlDir := range xlDirs {
globalEndpoints = append(globalEndpoints, &url.URL{ globalEndpoints = append(globalEndpoints, &url.URL{
@ -225,14 +229,13 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
if cmd == statusCmd { if cmd == statusCmd {
expectedInfo := ServerStatus{ expectedInfo := ServerStatus{
StorageInfo: newObjectLayerFn().StorageInfo(),
ServerVersion: ServerVersion{Version: Version, CommitID: CommitID}, ServerVersion: ServerVersion{Version: Version, CommitID: CommitID},
} }
receivedInfo := ServerStatus{} receivedInfo := ServerStatus{}
if jsonErr := json.Unmarshal(rec.Body.Bytes(), &receivedInfo); jsonErr != nil { if jsonErr := json.Unmarshal(rec.Body.Bytes(), &receivedInfo); jsonErr != nil {
t.Errorf("Failed to unmarshal StorageInfo - %v", jsonErr) t.Errorf("Failed to unmarshal StorageInfo - %v", jsonErr)
} }
if expectedInfo != receivedInfo { if expectedInfo.ServerVersion != receivedInfo.ServerVersion {
t.Errorf("Expected storage info and received storage info differ, %v %v", expectedInfo, receivedInfo) t.Errorf("Expected storage info and received storage info differ, %v %v", expectedInfo, receivedInfo)
} }
} }

@ -19,6 +19,7 @@ package cmd
import ( import (
"net/url" "net/url"
"path" "path"
"sort"
"sync" "sync"
"time" "time"
) )
@ -39,6 +40,7 @@ type adminCmdRunner interface {
Restart() error Restart() error
ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
ReInitDisks() error ReInitDisks() error
Uptime() (time.Duration, error)
} }
// Restart - Sends a message over channel to the go-routine // Restart - Sends a message over channel to the go-routine
@ -88,6 +90,28 @@ func (rc remoteAdminClient) ReInitDisks() error {
return rc.Call("Admin.ReInitDisks", &args, &reply) return rc.Call("Admin.ReInitDisks", &args, &reply)
} }
// Uptime - Returns the uptime of this server. Timestamp is taken
// after object layer is initialized.
func (lc localAdminClient) Uptime() (time.Duration, error) {
if globalBootTime.IsZero() {
return time.Duration(0), errServerNotInitialized
}
return time.Now().UTC().Sub(globalBootTime), nil
}
// Uptime - returns the uptime of the server to which the RPC call is made.
func (rc remoteAdminClient) Uptime() (time.Duration, error) {
args := AuthRPCArgs{}
reply := UptimeReply{}
err := rc.Call("Admin.Uptime", &args, &reply)
if err != nil {
return time.Duration(0), err
}
return reply.Uptime, nil
}
// adminPeer - represents an entity that implements Restart methods. // adminPeer - represents an entity that implements Restart methods.
type adminPeer struct { type adminPeer struct {
addr string addr string
@ -241,3 +265,65 @@ func reInitPeerDisks(peers adminPeers) error {
wg.Wait() wg.Wait()
return nil return nil
} }
// uptimeSlice - used to sort uptimes in chronological order.
type uptimeSlice []struct {
err error
uptime time.Duration
}
func (ts uptimeSlice) Len() int {
return len(ts)
}
func (ts uptimeSlice) Less(i, j int) bool {
return ts[i].uptime < ts[j].uptime
}
func (ts uptimeSlice) Swap(i, j int) {
ts[i], ts[j] = ts[j], ts[i]
}
// getPeerUptimes - returns the uptime since the last time read quorum
// was established on success. Otherwise returns errXLReadQuorum.
func getPeerUptimes(peers adminPeers) (time.Duration, error) {
uptimes := make(uptimeSlice, len(peers))
// Get up time of all servers.
wg := sync.WaitGroup{}
for i, peer := range peers {
wg.Add(1)
go func(idx int, peer adminPeer) {
defer wg.Done()
uptimes[idx].uptime, uptimes[idx].err = peer.cmdRunner.Uptime()
}(i, peer)
}
wg.Wait()
// Sort uptimes in chronological order.
sort.Sort(uptimes)
// Pick the readQuorum'th uptime in chronological order. i.e,
// the time at which read quorum was (re-)established.
readQuorum := len(uptimes) / 2
validCount := 0
latestUptime := time.Duration(0)
for _, uptime := range uptimes {
if uptime.err != nil {
continue
}
validCount++
if validCount >= readQuorum {
latestUptime = uptime.uptime
break
}
}
// This implies there weren't read quorum number of servers up.
if latestUptime == time.Duration(0) {
return time.Duration(0), InsufficientReadQuorum{}
}
return latestUptime, nil
}

@ -48,6 +48,12 @@ type ListLocksReply struct {
volLocks []VolumeLockInfo volLocks []VolumeLockInfo
} }
// UptimeReply - wraps the uptime response over RPC.
type UptimeReply struct {
AuthRPCReply
Uptime time.Duration
}
// Restart - Restart this instance of minio server. // Restart - Restart this instance of minio server.
func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error { func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
if err := args.IsAuthenticated(); err != nil { if err := args.IsAuthenticated(); err != nil {
@ -105,6 +111,27 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
return nil return nil
} }
// Uptime - returns the time when object layer was initialized on this server.
func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
if err := args.IsAuthenticated(); err != nil {
return err
}
if globalBootTime.IsZero() {
return errServerNotInitialized
}
// N B The uptime is computed assuming that the system time is
// monotonic. This is not the case in time pkg in Go, see
// https://github.com/golang/go/issues/12914. This is expected
// to be fixed by go1.9.
*reply = UptimeReply{
Uptime: time.Now().UTC().Sub(globalBootTime),
}
return nil
}
// registerAdminRPCRouter - registers RPC methods for service status, // registerAdminRPCRouter - registers RPC methods for service status,
// stop and restart commands. // stop and restart commands.
func registerAdminRPCRouter(mux *router.Router) error { func registerAdminRPCRouter(mux *router.Router) error {

@ -124,9 +124,13 @@ var (
// Global server's network statistics // Global server's network statistics
globalConnStats = newConnStats() globalConnStats = newConnStats()
// Global HTTP request statisitics // Global HTTP request statisitics
globalHTTPStats = newHTTPStats() globalHTTPStats = newHTTPStats()
// Time when object layer was initialized on start up.
globalBootTime time.Time
// Add new variable global values here. // Add new variable global values here.
) )

@ -25,6 +25,7 @@ import (
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
"time"
"runtime" "runtime"
@ -465,6 +466,9 @@ func serverMain(c *cli.Context) {
globalObjectAPI = newObject globalObjectAPI = newObject
globalObjLayerMutex.Unlock() globalObjLayerMutex.Unlock()
// Set startup time
globalBootTime = time.Now().UTC()
// Prints the formatted startup message once object layer is initialized. // Prints the formatted startup message once object layer is initialized.
printStartupMessage(apiEndPoints) printStartupMessage(apiEndPoints)

@ -25,6 +25,7 @@ import (
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"time"
) )
// BackendType - represents different backend types. // BackendType - represents different backend types.
@ -70,6 +71,7 @@ type ServerVersion struct {
type ServiceStatusMetadata struct { type ServiceStatusMetadata struct {
StorageInfo StorageInfo `json:"storageInfo"` StorageInfo StorageInfo `json:"storageInfo"`
ServerVersion ServerVersion `json:"serverVersion"` ServerVersion ServerVersion `json:"serverVersion"`
Uptime time.Duration `json:"uptime"`
} }
// ServiceStatus - Connect to a minio server and call Service Status Management API // ServiceStatus - Connect to a minio server and call Service Status Management API

Loading…
Cancel
Save