Profiling: Add base, fix memory profiling (#8850)

For 'snapshot' type profiles, record a 'before' profile that can be used 
as `go tool pprof -base=before ...` to compare before and after.

"Before" profiles are included in the zipped package.

[`runtime.MemProfileRate`](https://golang.org/pkg/runtime/#pkg-variables) 
should not be updated while the application is running, so we set it at startup.

Co-authored-by: Harshavardhana <harsha@minio.io>
master
Klaus Post 5 years ago committed by Harshavardhana
parent f14f60a487
commit c7178d2066
  1. 1
      cmd/server-main.go
  2. 2
      cmd/signals.go
  3. 63
      cmd/utils.go
  4. 9
      pkg/madmin/profiling-commands.go

@ -255,6 +255,7 @@ func serverMain(ctx *cli.Context) {
if ctx.Args().First() == "help" || !endpointsPresent(ctx) {
cli.ShowCommandHelpAndExit(ctx, "server", 1)
}
setDefaultProfilerRates()
// Initialize globalConsoleSys system
globalConsoleSys = NewConsoleLogger(context.Background())

@ -30,11 +30,9 @@ func handleSignals() {
// If global profiler is set stop before we exit.
globalProfilerMu.Lock()
defer globalProfilerMu.Unlock()
if len(globalProfiler) > 0 {
for _, p := range globalProfiler {
p.Stop()
}
}
if success {
os.Exit(0)

@ -42,6 +42,7 @@ import (
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/handlers"
"github.com/minio/minio/pkg/madmin"
humanize "github.com/dustin/go-humanize"
"github.com/gorilla/mux"
@ -185,9 +186,28 @@ func contains(slice interface{}, elem interface{}) bool {
// provide any API to calculate the profiler file path in the
// disk since the name of this latter is randomly generated.
type profilerWrapper struct {
// Profile recorded at start of benchmark.
base []byte
stopFn func() ([]byte, error)
}
// recordBase will record the profile and store it as the base.
func (p *profilerWrapper) recordBase(name string) {
var buf bytes.Buffer
p.base = nil
err := pprof.Lookup(name).WriteTo(&buf, 0)
if err != nil {
return
}
p.base = buf.Bytes()
}
// Base returns the recorded base if any.
func (p profilerWrapper) Base() []byte {
return p.base
}
// Stop the currently running benchmark.
func (p profilerWrapper) Stop() ([]byte, error) {
return p.stopFn()
}
@ -211,18 +231,28 @@ func getProfileData() (map[string][]byte, error) {
if err == nil {
dst[typ] = buf
}
buf = prof.Base()
if len(buf) > 0 {
dst[typ+"-before"] = buf
}
}
return dst, nil
}
func setDefaultProfilerRates() {
runtime.MemProfileRate = 4096 // 512K -> 4K - Must be constant throughout application lifetime.
runtime.SetMutexProfileFraction(0) // Disable until needed
runtime.SetBlockProfileRate(0) // Disable until needed
}
// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
func startProfiler(profilerType string) (minioProfiler, error) {
var prof profilerWrapper
// Enable profiler and set the name of the file that pkg/pprof
// library creates to store profiling data.
switch profilerType {
case "cpu":
switch madmin.ProfilerType(profilerType) {
case madmin.ProfilerCPU:
dirPath, err := ioutil.TempDir("", "profile")
if err != nil {
return nil, err
@ -245,32 +275,41 @@ func startProfiler(profilerType string) (minioProfiler, error) {
defer os.RemoveAll(dirPath)
return ioutil.ReadFile(fn)
}
case "mem":
old := runtime.MemProfileRate
runtime.MemProfileRate = 4096
case madmin.ProfilerMEM:
runtime.GC()
prof.recordBase("heap")
prof.stopFn = func() ([]byte, error) {
runtime.GC()
var buf bytes.Buffer
runtime.MemProfileRate = old
err := pprof.Lookup("heap").WriteTo(&buf, 0)
return buf.Bytes(), err
}
case "block":
case madmin.ProfilerBlock:
prof.recordBase("block")
runtime.SetBlockProfileRate(1)
prof.stopFn = func() ([]byte, error) {
var buf bytes.Buffer
runtime.SetBlockProfileRate(0)
err := pprof.Lookup("block").WriteTo(&buf, 0)
runtime.SetBlockProfileRate(0)
return buf.Bytes(), err
}
case "mutex":
case madmin.ProfilerMutex:
prof.recordBase("mutex")
runtime.SetMutexProfileFraction(1)
prof.stopFn = func() ([]byte, error) {
var buf bytes.Buffer
runtime.SetMutexProfileFraction(0)
err := pprof.Lookup("mutex").WriteTo(&buf, 0)
runtime.SetMutexProfileFraction(0)
return buf.Bytes(), err
}
case madmin.ProfilerThreads:
prof.recordBase("threadcreate")
prof.stopFn = func() ([]byte, error) {
var buf bytes.Buffer
err := pprof.Lookup("threadcreate").WriteTo(&buf, 0)
return buf.Bytes(), err
}
case "trace":
case madmin.ProfilerTrace:
dirPath, err := ioutil.TempDir("", "profile")
if err != nil {
return nil, err
@ -302,6 +341,8 @@ func startProfiler(profilerType string) (minioProfiler, error) {
// minioProfiler - minio profiler interface.
type minioProfiler interface {
// Return base profile. 'nil' if none.
Base() []byte
// Stop the profiler
Stop() ([]byte, error)
}

@ -34,10 +34,11 @@ type ProfilerType string
// Different supported profiler types.
const (
ProfilerCPU ProfilerType = "cpu" // represents CPU profiler type
ProfilerMEM = "mem" // represents MEM profiler type
ProfilerBlock = "block" // represents Block profiler type
ProfilerMutex = "mutex" // represents Mutex profiler type
ProfilerTrace = "trace" // represents Trace profiler type
ProfilerMEM ProfilerType = "mem" // represents MEM profiler type
ProfilerBlock ProfilerType = "block" // represents Block profiler type
ProfilerMutex ProfilerType = "mutex" // represents Mutex profiler type
ProfilerTrace ProfilerType = "trace" // represents Trace profiler type
ProfilerThreads ProfilerType = "threads" // represents ThreadCreate profiler type
)
// StartProfilingResult holds the result of starting

Loading…
Cancel
Save