From f1b246219325326bbecb674286d803e75574f407 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 4 Mar 2020 15:58:12 +0100 Subject: [PATCH] Add goroutine profiles (#9078) Allow downloading goroutine dump to help detect leaks or overuse of goroutines. Extensions are now type dependent. Change `profiling` -> `profile` prefix, since that is what they are not the abstract concept. --- cmd/notification.go | 2 +- cmd/utils.go | 35 ++++++++++++++++++++++++-------- pkg/madmin/profiling-commands.go | 13 ++++++------ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/cmd/notification.go b/cmd/notification.go index 3f0de203d..226eb0ada 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -375,7 +375,7 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io // Send profiling data to zip as file for typ, data := range data { header, zerr := zip.FileInfoHeader(dummyFileInfo{ - name: fmt.Sprintf("profiling-%s-%s.pprof", thisAddr, typ), + name: fmt.Sprintf("profile-%s-%s", thisAddr, typ), size: int64(len(data)), mode: 0600, modTime: UTCNow(), diff --git a/cmd/utils.go b/cmd/utils.go index 913ab86af..2439f62ba 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -190,13 +190,14 @@ type profilerWrapper struct { // Profile recorded at start of benchmark. base []byte stopFn func() ([]byte, error) + ext string } // recordBase will record the profile and store it as the base. -func (p *profilerWrapper) recordBase(name string) { +func (p *profilerWrapper) recordBase(name string, debug int) { var buf bytes.Buffer p.base = nil - err := pprof.Lookup(name).WriteTo(&buf, 0) + err := pprof.Lookup(name).WriteTo(&buf, debug) if err != nil { return } @@ -213,6 +214,11 @@ func (p profilerWrapper) Stop() ([]byte, error) { return p.stopFn() } +// Extension returns the extension without dot prefix. +func (p profilerWrapper) Extension() string { + return p.ext +} + // Returns current profile data, returns error if there is no active // profiling in progress. Stops an active profile. func getProfileData() (map[string][]byte, error) { @@ -230,11 +236,11 @@ func getProfileData() (map[string][]byte, error) { buf, err := prof.Stop() delete(globalProfiler, typ) if err == nil { - dst[typ] = buf + dst[typ+"."+prof.Extension()] = buf } buf = prof.Base() if len(buf) > 0 { - dst[typ+"-before"] = buf + dst[typ+"-before"+"."+prof.Extension()] = buf } } return dst, nil @@ -249,7 +255,7 @@ func setDefaultProfilerRates() { // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this. func startProfiler(profilerType string) (minioProfiler, error) { var prof profilerWrapper - + prof.ext = "pprof" // Enable profiler and set the name of the file that pkg/pprof // library creates to store profiling data. switch madmin.ProfilerType(profilerType) { @@ -278,7 +284,7 @@ func startProfiler(profilerType string) (minioProfiler, error) { } case madmin.ProfilerMEM: runtime.GC() - prof.recordBase("heap") + prof.recordBase("heap", 0) prof.stopFn = func() ([]byte, error) { runtime.GC() var buf bytes.Buffer @@ -286,7 +292,7 @@ func startProfiler(profilerType string) (minioProfiler, error) { return buf.Bytes(), err } case madmin.ProfilerBlock: - prof.recordBase("block") + prof.recordBase("block", 0) runtime.SetBlockProfileRate(1) prof.stopFn = func() ([]byte, error) { var buf bytes.Buffer @@ -295,7 +301,7 @@ func startProfiler(profilerType string) (minioProfiler, error) { return buf.Bytes(), err } case madmin.ProfilerMutex: - prof.recordBase("mutex") + prof.recordBase("mutex", 0) runtime.SetMutexProfileFraction(1) prof.stopFn = func() ([]byte, error) { var buf bytes.Buffer @@ -304,12 +310,20 @@ func startProfiler(profilerType string) (minioProfiler, error) { return buf.Bytes(), err } case madmin.ProfilerThreads: - prof.recordBase("threadcreate") + prof.recordBase("threadcreate", 0) prof.stopFn = func() ([]byte, error) { var buf bytes.Buffer err := pprof.Lookup("threadcreate").WriteTo(&buf, 0) return buf.Bytes(), err } + case madmin.ProfilerGoroutines: + prof.ext = "txt" + prof.recordBase("goroutines", 1) + prof.stopFn = func() ([]byte, error) { + var buf bytes.Buffer + err := pprof.Lookup("goroutines").WriteTo(&buf, 1) + return buf.Bytes(), err + } case madmin.ProfilerTrace: dirPath, err := ioutil.TempDir("", "profile") if err != nil { @@ -324,6 +338,7 @@ func startProfiler(profilerType string) (minioProfiler, error) { if err != nil { return nil, err } + prof.ext = "trace" prof.stopFn = func() ([]byte, error) { trace.Stop() err := f.Close() @@ -346,6 +361,8 @@ type minioProfiler interface { Base() []byte // Stop the profiler Stop() ([]byte, error) + // Return extension of profile + Extension() string } // Global profiler to be used by service go-routine. diff --git a/pkg/madmin/profiling-commands.go b/pkg/madmin/profiling-commands.go index fb0796586..716c65fec 100644 --- a/pkg/madmin/profiling-commands.go +++ b/pkg/madmin/profiling-commands.go @@ -33,12 +33,13 @@ type ProfilerType string // Different supported profiler types. const ( - ProfilerCPU ProfilerType = "cpu" // represents CPU profiler type - ProfilerMEM ProfilerType = "mem" // represents MEM profiler type - ProfilerBlock ProfilerType = "block" // represents Block profiler type - ProfilerMutex ProfilerType = "mutex" // represents Mutex profiler type - ProfilerTrace ProfilerType = "trace" // represents Trace profiler type - ProfilerThreads ProfilerType = "threads" // represents ThreadCreate profiler type + ProfilerCPU ProfilerType = "cpu" // represents CPU profiler type + ProfilerMEM ProfilerType = "mem" // represents MEM profiler type + ProfilerBlock ProfilerType = "block" // represents Block profiler type + ProfilerMutex ProfilerType = "mutex" // represents Mutex profiler type + ProfilerTrace ProfilerType = "trace" // represents Trace profiler type + ProfilerThreads ProfilerType = "threads" // represents ThreadCreate profiler type + ProfilerGoroutines ProfilerType = "goroutines" // represents Goroutine dumps. ) // StartProfilingResult holds the result of starting