obd: Add console log to OBD output (#10372)

master
Anis Elleuch 4 years ago committed by GitHub
parent 80e3dce631
commit 8ea55f9dba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 25
      cmd/admin-handlers.go
  2. 14
      cmd/admin-router.go
  3. 4
      cmd/common-main.go
  4. 18
      cmd/consolelogger.go
  5. 4
      cmd/gateway-main.go
  6. 30
      cmd/notification.go
  7. 13
      cmd/obdinfo.go
  8. 11
      cmd/peer-rest-client.go
  9. 1
      cmd/peer-rest-common.go
  10. 17
      cmd/peer-rest-server.go
  11. 1
      cmd/server-main.go
  12. 17
      pkg/madmin/obd.go

@ -1241,7 +1241,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
return
}
vars := mux.Vars(r)
query := r.URL.Query()
obdInfo := madmin.OBDInfo{}
obdInfoCh := make(chan madmin.OBDInfo)
@ -1287,7 +1287,13 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
go func() {
defer close(obdInfoCh)
if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
if log := query.Get("log"); log == "true" {
obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, getLocalLogOBD(deadlinedCtx, r))
obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, globalNotificationSys.LogOBDInfo(deadlinedCtx)...)
partialWrite(obdInfo)
}
if cpu := query.Get("syscpu"); cpu == "true" {
cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
@ -1295,7 +1301,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo)
}
if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
@ -1303,7 +1309,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo)
}
if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
if osInfo := query.Get("sysosinfo"); osInfo == "true" {
osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
@ -1311,7 +1317,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo)
}
if mem, ok := vars["sysmem"]; ok && mem == "true" {
if mem := query.Get("sysmem"); mem == "true" {
memInfo := getLocalMemOBD(deadlinedCtx, r)
obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
@ -1319,7 +1325,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo)
}
if proc, ok := vars["sysprocess"]; ok && proc == "true" {
if proc := query.Get("sysprocess"); proc == "true" {
procInfo := getLocalProcOBD(deadlinedCtx, r)
obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
@ -1327,14 +1333,14 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo)
}
if config, ok := vars["minioconfig"]; ok && config == "true" {
if config := query.Get("minioconfig"); config == "true" {
cfg, err := readServerConfig(ctx, objectAPI)
logger.LogIf(ctx, err)
obdInfo.Minio.Config = cfg
partialWrite(obdInfo)
}
if drive, ok := vars["perfdrive"]; ok && drive == "true" {
if drive := query.Get("perfdrive"); drive == "true" {
// Get drive obd details from local server's drive(s)
driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
@ -1365,7 +1371,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
partialWrite(obdInfo)
}
if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistErasure {
if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
partialWrite(obdInfo)
@ -1379,6 +1385,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
partialWrite(obdInfo)
}
}()
ticker := time.NewTicker(30 * time.Second)

@ -212,18 +212,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
if !globalIsGateway {
// -- OBD API --
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").
HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).
Queries("perfdrive", "{perfdrive:true|false}",
"perfnet", "{perfnet:true|false}",
"minioinfo", "{minioinfo:true|false}",
"minioconfig", "{minioconfig:true|false}",
"syscpu", "{syscpu:true|false}",
"sysdiskhw", "{sysdiskhw:true|false}",
"sysosinfo", "{sysosinfo:true|false}",
"sysmem", "{sysmem:true|false}",
"sysprocess", "{sysprocess:true|false}",
)
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
}
}

@ -43,10 +43,6 @@ func init() {
logger.Init(GOPATH, GOROOT)
logger.RegisterError(config.FmtError)
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(GlobalContext)
logger.AddTarget(globalConsoleSys)
gob.Register(StorageErr(""))
}

@ -122,6 +122,24 @@ func (sys *HTTPConsoleLoggerSys) Validate() error {
return nil
}
// Content returns the console stdout log
func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
sys.RLock()
sys.logBuf.Do(func(p interface{}) {
if p != nil {
lg, ok := p.(log.Info)
if ok {
if (lg.Entry != log.Entry{}) {
logs = append(logs, lg.Entry)
}
}
}
})
sys.RUnlock()
return
}
// Send log message 'e' to console and publish to console
// log pubsub system
func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {

@ -168,6 +168,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
}
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(GlobalContext)
logger.AddTarget(globalConsoleSys)
// Handle common command args.
handleCommonCmdArgs(ctx)

@ -1118,6 +1118,36 @@ func (sys *NotificationSys) ProcOBDInfo(ctx context.Context) []madmin.ServerProc
return reply
}
// LogOBDInfo - Logs OBD information
func (sys *NotificationSys) LogOBDInfo(ctx context.Context) []madmin.ServerLogOBDInfo {
reply := make([]madmin.ServerLogOBDInfo, len(sys.peerClients))
g := errgroup.WithNErrs(len(sys.peerClients))
for index, client := range sys.peerClients {
if client == nil {
continue
}
index := index
g.Go(func() error {
var err error
reply[index], err = sys.peerClients[index].LogOBDInfo(ctx)
return err
}, index)
}
for index, err := range g.Wait() {
if err != nil {
addr := sys.peerClients[index].host.String()
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
ctx := logger.SetReqInfo(GlobalContext, reqInfo)
logger.LogIf(ctx, err)
reply[index].Addr = addr
reply[index].Error = err.Error()
}
}
return reply
}
// ServerInfo - calls ServerInfo RPC call on all peers.
func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
reply := make([]madmin.ServerProperties, len(sys.peerClients))

@ -148,6 +148,19 @@ func getLocalMemOBD(ctx context.Context, r *http.Request) madmin.ServerMemOBDInf
}
}
func getLocalLogOBD(ctx context.Context, r *http.Request) madmin.ServerLogOBDInfo {
addr := r.Host
if globalIsDistErasure {
addr = GetLocalPeer(globalEndpoints)
}
log := globalConsoleSys.Content()
return madmin.ServerLogOBDInfo{
Addr: addr,
Entries: log,
}
}
func getLocalProcOBD(ctx context.Context, r *http.Request) madmin.ServerProcOBDInfo {
addr := r.Host
if globalIsDistErasure {

@ -420,6 +420,17 @@ func (client *peerRESTClient) ProcOBDInfo(ctx context.Context) (info madmin.Serv
return info, err
}
// LogOBDInfo - fetch Log OBD information for a remote node.
func (client *peerRESTClient) LogOBDInfo(ctx context.Context) (info madmin.ServerLogOBDInfo, err error) {
respBody, err := client.callWithContext(ctx, peerRESTMethodLogOBDInfo, nil, nil, -1)
if err != nil {
return
}
defer http.DrainBody(respBody)
err = gob.NewDecoder(respBody).Decode(&info)
return info, err
}
// StartProfiling - Issues profiling command on the peer node.
func (client *peerRESTClient) StartProfiling(profiler string) error {
values := make(url.Values)

@ -33,6 +33,7 @@ const (
peerRESTMethodOsInfoOBDInfo = "/osinfoobdinfo"
peerRESTMethodMemOBDInfo = "/memobdinfo"
peerRESTMethodProcOBDInfo = "/procobdinfo"
peerRESTMethodLogOBDInfo = "/logobdinfo"
peerRESTMethodDispatchNetOBDInfo = "/dispatchnetobdinfo"
peerRESTMethodDeleteBucketMetadata = "/deletebucketmetadata"
peerRESTMethodLoadBucketMetadata = "/loadbucketmetadata"

@ -545,6 +545,22 @@ func (s *peerRESTServer) ProcOBDInfoHandler(w http.ResponseWriter, r *http.Reque
logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
}
// LogOBDInfoHandler - returns Log OBD info.
func (s *peerRESTServer) LogOBDInfoHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
s.writeErrorResponse(w, errors.New("Invalid request"))
return
}
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
info := getLocalLogOBD(ctx, r)
defer w.(http.Flusher).Flush()
logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
}
// MemOBDInfoHandler - returns Mem OBD info.
func (s *peerRESTServer) MemOBDInfoHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
@ -1039,6 +1055,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcOBDInfo).HandlerFunc(httpTraceHdrs(server.ProcOBDInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLogOBDInfo).HandlerFunc(httpTraceHdrs(server.LogOBDInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemOBDInfo).HandlerFunc(httpTraceHdrs(server.MemOBDInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfoOBDInfo).HandlerFunc(httpTraceHdrs(server.OsOBDInfoHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwOBDInfo).HandlerFunc(httpTraceHdrs(server.DiskHwOBDInfoHandler))

@ -391,6 +391,7 @@ func serverMain(ctx *cli.Context) {
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(GlobalContext)
logger.AddTarget(globalConsoleSys)
// Handle all server command args.
serverHandleCmdArgs(ctx)

@ -25,6 +25,7 @@ import (
"net/url"
"time"
"github.com/minio/minio/cmd/logger/message/log"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/net"
@ -42,6 +43,19 @@ type OBDInfo struct {
Perf PerfOBDInfo `json:"perf,omitempty"`
Minio MinioOBDInfo `json:"minio,omitempty"`
Sys SysOBDInfo `json:"sys,omitempty"`
Logging LogOBDInfo `json:"logging,omitempty"`
}
// LogOBDInfo holds different type of logs of all clusters
type LogOBDInfo struct {
ServersLog []ServerLogOBDInfo `json:"serverLogs"`
}
// ServerLogOBDInfo holds server's stdout log
type ServerLogOBDInfo struct {
Addr string `json:"nodeName"`
Entries []log.Entry `json:"entries"`
Error string `json:"error,omitempty"`
}
// SysOBDInfo - Includes hardware and system information of the MinIO cluster
@ -185,6 +199,7 @@ const (
OBDDataTypeSysMem OBDDataType = "sysmem"
OBDDataTypeSysNet OBDDataType = "sysnet"
OBDDataTypeSysProcess OBDDataType = "sysprocess"
OBDDataTypeLog OBDDataType = "log"
)
// OBDDataTypesMap - Map of OBD datatypes
@ -201,6 +216,7 @@ var OBDDataTypesMap = map[string]OBDDataType{
"sysmem": OBDDataTypeSysMem,
"sysnet": OBDDataTypeSysNet,
"sysprocess": OBDDataTypeSysProcess,
"log": OBDDataTypeLog,
}
// OBDDataTypesList - List of OBD datatypes
@ -217,6 +233,7 @@ var OBDDataTypesList = []OBDDataType{
OBDDataTypeSysMem,
OBDDataTypeSysNet,
OBDDataTypeSysProcess,
OBDDataTypeLog,
}
// ServerOBDInfo - Connect to a minio server and call OBD Info Management API

Loading…
Cancel
Save