Add goroutine profiles (#9078)

Allow downloading goroutine dump to help detect leaks
or overuse of goroutines.

Extensions are now type dependent.

Change `profiling` -> `profile` prefix, since that is what they are 
not the abstract concept.
master
Klaus Post 5 years ago committed by GitHub
parent 8fbf2b0b2a
commit f1b2462193
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      cmd/notification.go
  2. 35
      cmd/utils.go
  3. 1
      pkg/madmin/profiling-commands.go

@ -375,7 +375,7 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
// Send profiling data to zip as file // Send profiling data to zip as file
for typ, data := range data { for typ, data := range data {
header, zerr := zip.FileInfoHeader(dummyFileInfo{ header, zerr := zip.FileInfoHeader(dummyFileInfo{
name: fmt.Sprintf("profiling-%s-%s.pprof", thisAddr, typ), name: fmt.Sprintf("profile-%s-%s", thisAddr, typ),
size: int64(len(data)), size: int64(len(data)),
mode: 0600, mode: 0600,
modTime: UTCNow(), modTime: UTCNow(),

@ -190,13 +190,14 @@ type profilerWrapper struct {
// Profile recorded at start of benchmark. // Profile recorded at start of benchmark.
base []byte base []byte
stopFn func() ([]byte, error) stopFn func() ([]byte, error)
ext string
} }
// recordBase will record the profile and store it as the base. // recordBase will record the profile and store it as the base.
func (p *profilerWrapper) recordBase(name string) { func (p *profilerWrapper) recordBase(name string, debug int) {
var buf bytes.Buffer var buf bytes.Buffer
p.base = nil p.base = nil
err := pprof.Lookup(name).WriteTo(&buf, 0) err := pprof.Lookup(name).WriteTo(&buf, debug)
if err != nil { if err != nil {
return return
} }
@ -213,6 +214,11 @@ func (p profilerWrapper) Stop() ([]byte, error) {
return p.stopFn() return p.stopFn()
} }
// Extension returns the extension without dot prefix.
func (p profilerWrapper) Extension() string {
return p.ext
}
// Returns current profile data, returns error if there is no active // Returns current profile data, returns error if there is no active
// profiling in progress. Stops an active profile. // profiling in progress. Stops an active profile.
func getProfileData() (map[string][]byte, error) { func getProfileData() (map[string][]byte, error) {
@ -230,11 +236,11 @@ func getProfileData() (map[string][]byte, error) {
buf, err := prof.Stop() buf, err := prof.Stop()
delete(globalProfiler, typ) delete(globalProfiler, typ)
if err == nil { if err == nil {
dst[typ] = buf dst[typ+"."+prof.Extension()] = buf
} }
buf = prof.Base() buf = prof.Base()
if len(buf) > 0 { if len(buf) > 0 {
dst[typ+"-before"] = buf dst[typ+"-before"+"."+prof.Extension()] = buf
} }
} }
return dst, nil return dst, nil
@ -249,7 +255,7 @@ func setDefaultProfilerRates() {
// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this. // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
func startProfiler(profilerType string) (minioProfiler, error) { func startProfiler(profilerType string) (minioProfiler, error) {
var prof profilerWrapper var prof profilerWrapper
prof.ext = "pprof"
// Enable profiler and set the name of the file that pkg/pprof // Enable profiler and set the name of the file that pkg/pprof
// library creates to store profiling data. // library creates to store profiling data.
switch madmin.ProfilerType(profilerType) { switch madmin.ProfilerType(profilerType) {
@ -278,7 +284,7 @@ func startProfiler(profilerType string) (minioProfiler, error) {
} }
case madmin.ProfilerMEM: case madmin.ProfilerMEM:
runtime.GC() runtime.GC()
prof.recordBase("heap") prof.recordBase("heap", 0)
prof.stopFn = func() ([]byte, error) { prof.stopFn = func() ([]byte, error) {
runtime.GC() runtime.GC()
var buf bytes.Buffer var buf bytes.Buffer
@ -286,7 +292,7 @@ func startProfiler(profilerType string) (minioProfiler, error) {
return buf.Bytes(), err return buf.Bytes(), err
} }
case madmin.ProfilerBlock: case madmin.ProfilerBlock:
prof.recordBase("block") prof.recordBase("block", 0)
runtime.SetBlockProfileRate(1) runtime.SetBlockProfileRate(1)
prof.stopFn = func() ([]byte, error) { prof.stopFn = func() ([]byte, error) {
var buf bytes.Buffer var buf bytes.Buffer
@ -295,7 +301,7 @@ func startProfiler(profilerType string) (minioProfiler, error) {
return buf.Bytes(), err return buf.Bytes(), err
} }
case madmin.ProfilerMutex: case madmin.ProfilerMutex:
prof.recordBase("mutex") prof.recordBase("mutex", 0)
runtime.SetMutexProfileFraction(1) runtime.SetMutexProfileFraction(1)
prof.stopFn = func() ([]byte, error) { prof.stopFn = func() ([]byte, error) {
var buf bytes.Buffer var buf bytes.Buffer
@ -304,12 +310,20 @@ func startProfiler(profilerType string) (minioProfiler, error) {
return buf.Bytes(), err return buf.Bytes(), err
} }
case madmin.ProfilerThreads: case madmin.ProfilerThreads:
prof.recordBase("threadcreate") prof.recordBase("threadcreate", 0)
prof.stopFn = func() ([]byte, error) { prof.stopFn = func() ([]byte, error) {
var buf bytes.Buffer var buf bytes.Buffer
err := pprof.Lookup("threadcreate").WriteTo(&buf, 0) err := pprof.Lookup("threadcreate").WriteTo(&buf, 0)
return buf.Bytes(), err return buf.Bytes(), err
} }
case madmin.ProfilerGoroutines:
prof.ext = "txt"
prof.recordBase("goroutines", 1)
prof.stopFn = func() ([]byte, error) {
var buf bytes.Buffer
err := pprof.Lookup("goroutines").WriteTo(&buf, 1)
return buf.Bytes(), err
}
case madmin.ProfilerTrace: case madmin.ProfilerTrace:
dirPath, err := ioutil.TempDir("", "profile") dirPath, err := ioutil.TempDir("", "profile")
if err != nil { if err != nil {
@ -324,6 +338,7 @@ func startProfiler(profilerType string) (minioProfiler, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
prof.ext = "trace"
prof.stopFn = func() ([]byte, error) { prof.stopFn = func() ([]byte, error) {
trace.Stop() trace.Stop()
err := f.Close() err := f.Close()
@ -346,6 +361,8 @@ type minioProfiler interface {
Base() []byte Base() []byte
// Stop the profiler // Stop the profiler
Stop() ([]byte, error) Stop() ([]byte, error)
// Return extension of profile
Extension() string
} }
// Global profiler to be used by service go-routine. // Global profiler to be used by service go-routine.

@ -39,6 +39,7 @@ const (
ProfilerMutex ProfilerType = "mutex" // represents Mutex profiler type ProfilerMutex ProfilerType = "mutex" // represents Mutex profiler type
ProfilerTrace ProfilerType = "trace" // represents Trace profiler type ProfilerTrace ProfilerType = "trace" // represents Trace profiler type
ProfilerThreads ProfilerType = "threads" // represents ThreadCreate profiler type ProfilerThreads ProfilerType = "threads" // represents ThreadCreate profiler type
ProfilerGoroutines ProfilerType = "goroutines" // represents Goroutine dumps.
) )
// StartProfilingResult holds the result of starting // StartProfilingResult holds the result of starting

Loading…
Cancel
Save