|
|
|
@ -982,16 +982,13 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) { |
|
|
|
|
|
|
|
|
|
w.Header().Set(xhttp.ContentType, "text/event-stream") |
|
|
|
|
|
|
|
|
|
doneCh := make(chan struct{}) |
|
|
|
|
defer close(doneCh) |
|
|
|
|
|
|
|
|
|
// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
|
|
|
|
|
// Use buffered channel to take care of burst sends or slow w.Write()
|
|
|
|
|
traceCh := make(chan interface{}, 4000) |
|
|
|
|
|
|
|
|
|
peers := getRestClients(globalEndpoints) |
|
|
|
|
|
|
|
|
|
globalHTTPTrace.Subscribe(traceCh, doneCh, func(entry interface{}) bool { |
|
|
|
|
globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool { |
|
|
|
|
return mustTrace(entry, trcAll, trcErr) |
|
|
|
|
}) |
|
|
|
|
|
|
|
|
@ -999,7 +996,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) { |
|
|
|
|
if peer == nil { |
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
peer.Trace(traceCh, doneCh, trcAll, trcErr) |
|
|
|
|
peer.Trace(traceCh, ctx.Done(), trcAll, trcErr) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
keepAliveTicker := time.NewTicker(500 * time.Millisecond) |
|
|
|
@ -1018,7 +1015,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) { |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
w.(http.Flusher).Flush() |
|
|
|
|
case <-GlobalServiceDoneCh: |
|
|
|
|
case <-ctx.Done(): |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
@ -1052,20 +1049,18 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque |
|
|
|
|
w.Header().Add("Connection", "close") |
|
|
|
|
w.Header().Set(xhttp.ContentType, "text/event-stream") |
|
|
|
|
|
|
|
|
|
doneCh := make(chan struct{}) |
|
|
|
|
defer close(doneCh) |
|
|
|
|
logCh := make(chan interface{}, 4000) |
|
|
|
|
|
|
|
|
|
peers := getRestClients(globalEndpoints) |
|
|
|
|
|
|
|
|
|
globalConsoleSys.Subscribe(logCh, doneCh, node, limitLines, logKind, nil) |
|
|
|
|
globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil) |
|
|
|
|
|
|
|
|
|
for _, peer := range peers { |
|
|
|
|
if peer == nil { |
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
if node == "" || strings.EqualFold(peer.host.Name, node) { |
|
|
|
|
peer.ConsoleLog(logCh, doneCh) |
|
|
|
|
peer.ConsoleLog(logCh, ctx.Done()) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -1089,7 +1084,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
w.(http.Flusher).Flush() |
|
|
|
|
case <-GlobalServiceDoneCh: |
|
|
|
|
case <-ctx.Done(): |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
@ -1175,65 +1170,39 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
vars := mux.Vars(r) |
|
|
|
|
pulse := make(chan struct{}) |
|
|
|
|
obdDone := make(chan struct{}) |
|
|
|
|
obdInfo := madmin.OBDInfo{} |
|
|
|
|
obdInfoCh := make(chan madmin.OBDInfo) |
|
|
|
|
|
|
|
|
|
enc := json.NewEncoder(w) |
|
|
|
|
doPartialWrite := func() { |
|
|
|
|
logger.LogIf(ctx, enc.Encode(obdInfo)) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
partialWrite := func() { |
|
|
|
|
pulse <- struct{}{} |
|
|
|
|
partialWrite := func(oinfo madmin.OBDInfo) { |
|
|
|
|
obdInfoCh <- oinfo |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
finish := func() { |
|
|
|
|
obdDone <- struct{}{} |
|
|
|
|
} |
|
|
|
|
setCommonHeaders(w) |
|
|
|
|
w.Header().Set(xhttp.ContentType, "text/event-stream") |
|
|
|
|
w.WriteHeader(http.StatusOK) |
|
|
|
|
|
|
|
|
|
errResp := func(err error) { |
|
|
|
|
errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(), |
|
|
|
|
w.Header().Get(xhttp.AmzRequestID), globalDeploymentID) |
|
|
|
|
encodedErrorResponse := encodeResponse(errorResponse) |
|
|
|
|
obdInfo.Error = string(encodedErrorResponse) |
|
|
|
|
finish() |
|
|
|
|
logger.LogIf(ctx, enc.Encode(obdInfo)) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
deadline := 3600 * time.Second |
|
|
|
|
deadlineStr := r.URL.Query().Get("deadline") |
|
|
|
|
if deadlineStr != "" { |
|
|
|
|
if dstr := r.URL.Query().Get("deadline"); dstr != "" { |
|
|
|
|
var err error |
|
|
|
|
deadline, err = time.ParseDuration(deadlineStr) |
|
|
|
|
deadline, err = time.ParseDuration(dstr) |
|
|
|
|
if err != nil { |
|
|
|
|
errResp(err) |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(deadline)) |
|
|
|
|
|
|
|
|
|
setCommonHeaders(w) |
|
|
|
|
w.Header().Set(xhttp.ContentType, string(mimeJSON)) |
|
|
|
|
w.WriteHeader(http.StatusOK) |
|
|
|
|
deadlinedCtx, cancel := context.WithTimeout(ctx, deadline) |
|
|
|
|
|
|
|
|
|
ticker := time.NewTicker(30 * time.Second) |
|
|
|
|
defer ticker.Stop() |
|
|
|
|
|
|
|
|
|
go func() { |
|
|
|
|
loop: |
|
|
|
|
for { |
|
|
|
|
select { |
|
|
|
|
case <-ticker.C: |
|
|
|
|
doPartialWrite() |
|
|
|
|
case <-pulse: |
|
|
|
|
doPartialWrite() |
|
|
|
|
case <-obdDone: |
|
|
|
|
break loop |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
w.(http.Flusher).Flush() |
|
|
|
|
cancel() |
|
|
|
|
}() |
|
|
|
|
defer cancel() |
|
|
|
|
|
|
|
|
|
nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress") |
|
|
|
|
if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
|
|
|
|
@ -1242,89 +1211,114 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) |
|
|
|
|
} |
|
|
|
|
defer nsLock.Unlock() |
|
|
|
|
|
|
|
|
|
if cpu, ok := vars["syscpu"]; ok && cpu == "true" { |
|
|
|
|
cpuInfo := getLocalCPUOBDInfo(deadlinedCtx) |
|
|
|
|
|
|
|
|
|
obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo) |
|
|
|
|
obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite() |
|
|
|
|
} |
|
|
|
|
go func() { |
|
|
|
|
defer close(obdInfoCh) |
|
|
|
|
|
|
|
|
|
if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" { |
|
|
|
|
diskHwInfo := getLocalDiskHwOBD(deadlinedCtx) |
|
|
|
|
if cpu, ok := vars["syscpu"]; ok && cpu == "true" { |
|
|
|
|
cpuInfo := getLocalCPUOBDInfo(deadlinedCtx) |
|
|
|
|
|
|
|
|
|
obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo) |
|
|
|
|
obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite() |
|
|
|
|
} |
|
|
|
|
obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo) |
|
|
|
|
obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite(obdInfo) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" { |
|
|
|
|
osInfo := getLocalOsInfoOBD(deadlinedCtx) |
|
|
|
|
if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" { |
|
|
|
|
diskHwInfo := getLocalDiskHwOBD(deadlinedCtx) |
|
|
|
|
|
|
|
|
|
obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo) |
|
|
|
|
obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if mem, ok := vars["sysmem"]; ok && mem == "true" { |
|
|
|
|
memInfo := getLocalMemOBD(deadlinedCtx) |
|
|
|
|
obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo) |
|
|
|
|
obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite(obdInfo) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo) |
|
|
|
|
obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite() |
|
|
|
|
} |
|
|
|
|
if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" { |
|
|
|
|
osInfo := getLocalOsInfoOBD(deadlinedCtx) |
|
|
|
|
|
|
|
|
|
if proc, ok := vars["sysprocess"]; ok && proc == "true" { |
|
|
|
|
procInfo := getLocalProcOBD(deadlinedCtx) |
|
|
|
|
obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo) |
|
|
|
|
obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite(obdInfo) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo) |
|
|
|
|
obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite() |
|
|
|
|
} |
|
|
|
|
if mem, ok := vars["sysmem"]; ok && mem == "true" { |
|
|
|
|
memInfo := getLocalMemOBD(deadlinedCtx) |
|
|
|
|
|
|
|
|
|
if config, ok := vars["minioconfig"]; ok && config == "true" { |
|
|
|
|
cfg, err := readServerConfig(ctx, objectAPI) |
|
|
|
|
logger.LogIf(ctx, err) |
|
|
|
|
obdInfo.Minio.Config = cfg |
|
|
|
|
partialWrite() |
|
|
|
|
} |
|
|
|
|
obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo) |
|
|
|
|
obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite(obdInfo) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if drive, ok := vars["perfdrive"]; ok && drive == "true" { |
|
|
|
|
// Get drive obd details from local server's drive(s)
|
|
|
|
|
driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r) |
|
|
|
|
driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r) |
|
|
|
|
if proc, ok := vars["sysprocess"]; ok && proc == "true" { |
|
|
|
|
procInfo := getLocalProcOBD(deadlinedCtx) |
|
|
|
|
|
|
|
|
|
errStr := "" |
|
|
|
|
if driveOBDSerial.Error != "" { |
|
|
|
|
errStr = "serial: " + driveOBDSerial.Error |
|
|
|
|
obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo) |
|
|
|
|
obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...) |
|
|
|
|
partialWrite(obdInfo) |
|
|
|
|
} |
|
|
|
|
if driveOBDParallel.Error != "" { |
|
|
|
|
errStr = errStr + " parallel: " + driveOBDParallel.Error |
|
|
|
|
|
|
|
|
|
if config, ok := vars["minioconfig"]; ok && config == "true" { |
|
|
|
|
cfg, err := readServerConfig(ctx, objectAPI) |
|
|
|
|
logger.LogIf(ctx, err) |
|
|
|
|
obdInfo.Minio.Config = cfg |
|
|
|
|
partialWrite(obdInfo) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
driveOBD := madmin.ServerDrivesOBDInfo{ |
|
|
|
|
Addr: driveOBDSerial.Addr, |
|
|
|
|
Serial: driveOBDSerial.Serial, |
|
|
|
|
Parallel: driveOBDParallel.Parallel, |
|
|
|
|
Error: errStr, |
|
|
|
|
if drive, ok := vars["perfdrive"]; ok && drive == "true" { |
|
|
|
|
// Get drive obd details from local server's drive(s)
|
|
|
|
|
driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r) |
|
|
|
|
driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r) |
|
|
|
|
|
|
|
|
|
errStr := "" |
|
|
|
|
if driveOBDSerial.Error != "" { |
|
|
|
|
errStr = "serial: " + driveOBDSerial.Error |
|
|
|
|
} |
|
|
|
|
if driveOBDParallel.Error != "" { |
|
|
|
|
errStr = errStr + " parallel: " + driveOBDParallel.Error |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
driveOBD := madmin.ServerDrivesOBDInfo{ |
|
|
|
|
Addr: driveOBDSerial.Addr, |
|
|
|
|
Serial: driveOBDSerial.Serial, |
|
|
|
|
Parallel: driveOBDParallel.Parallel, |
|
|
|
|
Error: errStr, |
|
|
|
|
} |
|
|
|
|
obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD) |
|
|
|
|
|
|
|
|
|
// Notify all other MinIO peers to report drive obd numbers
|
|
|
|
|
driveOBDs := globalNotificationSys.DriveOBDInfo(deadlinedCtx) |
|
|
|
|
obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBDs...) |
|
|
|
|
|
|
|
|
|
partialWrite(obdInfo) |
|
|
|
|
} |
|
|
|
|
obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD) |
|
|
|
|
|
|
|
|
|
// Notify all other MinIO peers to report drive obd numbers
|
|
|
|
|
driveOBDs := globalNotificationSys.DriveOBDInfo(deadlinedCtx) |
|
|
|
|
obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBDs...) |
|
|
|
|
if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL { |
|
|
|
|
obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx)) |
|
|
|
|
obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.DispatchNetOBDInfo(deadlinedCtx)...) |
|
|
|
|
obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx) |
|
|
|
|
partialWrite(obdInfo) |
|
|
|
|
} |
|
|
|
|
}() |
|
|
|
|
|
|
|
|
|
partialWrite() |
|
|
|
|
} |
|
|
|
|
ticker := time.NewTicker(30 * time.Second) |
|
|
|
|
defer ticker.Stop() |
|
|
|
|
|
|
|
|
|
if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL { |
|
|
|
|
obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx)) |
|
|
|
|
obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.DispatchNetOBDInfo(deadlinedCtx)...) |
|
|
|
|
obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx) |
|
|
|
|
partialWrite() |
|
|
|
|
for { |
|
|
|
|
select { |
|
|
|
|
case oinfo, ok := <-obdInfoCh: |
|
|
|
|
if !ok { |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
logger.LogIf(ctx, enc.Encode(oinfo)) |
|
|
|
|
w.(http.Flusher).Flush() |
|
|
|
|
case <-ticker.C: |
|
|
|
|
if _, err := w.Write([]byte(" ")); err != nil { |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
w.(http.Flusher).Flush() |
|
|
|
|
case <-deadlinedCtx.Done(): |
|
|
|
|
w.(http.Flusher).Flush() |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
finish() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// ServerInfoHandler - GET /minio/admin/v3/info
|
|
|
|
@ -1488,7 +1482,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque |
|
|
|
|
func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus { |
|
|
|
|
|
|
|
|
|
// Fetch the configured targets
|
|
|
|
|
targetList, err := notify.FetchRegisteredTargets(cfg, GlobalServiceDoneCh, NewGatewayHTTPTransport(), true, false) |
|
|
|
|
targetList, err := notify.FetchRegisteredTargets(cfg, GlobalContext.Done(), NewGatewayHTTPTransport(), true, false) |
|
|
|
|
if err != nil && err != notify.ErrTargetsOffline { |
|
|
|
|
logger.LogIf(GlobalContext, err) |
|
|
|
|
return nil |
|
|
|
|