posix: Use sync.Pool buffers to copy in large buffers. (#3106)

These fixes are borrowed from the fixes required for GlusterFS i/o throughput.

.travis.yml

@@ -7,9 +7,6 @@ language: go
 
 os:
 - linux
-- osx
-
-osx_image: xcode7.2
 
 env:
 - ARCH=x86_64

cmd/object-common.go

@@ -27,7 +27,7 @@ const (
 	blockSizeV1 = 10 * 1024 * 1024 // 10MiB.
 
 	// Staging buffer read size for all internal operations version 1.
-	readSizeV1 = 128 * 1024 // 128KiB.
+	readSizeV1 = 1 * 1024 * 1024 // 1MiB.
 
 	// Buckets meta prefix.
 	bucketMetaPrefix = "buckets"

cmd/posix-list-dir-nix.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 	"unsafe"
 )
@@ -30,9 +31,9 @@ import (
 const (
 	// readDirentBufSize for syscall.ReadDirent() to hold multiple
 	// directory entries in one buffer. golang source uses 4096 as
-	// buffer size whereas we want 25 times larger to save lots of
+	// buffer size whereas we want 64 times larger to save lots of
 	// entries to avoid multiple syscall.ReadDirent() call.
-	readDirentBufSize = 4096 * 25
+	readDirentBufSize = 4096 * 64
 )
 
 // actual length of the byte array from the c - world.
@@ -106,9 +107,19 @@ func parseDirents(dirPath string, buf []byte) (entries []string, err error) {
 	return entries, nil
 }
 
+var readDirBufPool = sync.Pool{
+	New: func() interface{} {
+		b := make([]byte, readDirentBufSize)
+		return &b
+	},
+}
+
 // Return all the entries at the directory dirPath.
 func readDir(dirPath string) (entries []string, err error) {
-	buf := make([]byte, readDirentBufSize)
+	bufp := readDirBufPool.Get().(*[]byte)
+	buf := *bufp
+	defer readDirBufPool.Put(bufp)
+
 	d, err := os.Open(dirPath)
 	if err != nil {
 		// File is really not found.

cmd/posix-utils_windows_test.go

@@ -36,7 +36,7 @@ func TestUNCPaths(t *testing.T) {
 		{string(bytes.Repeat([]byte("界"), 85)), true},
 		// Each path component must be <= 255 bytes long.
 		{string(bytes.Repeat([]byte("界"), 100)), false},
-		{`\\p\q\r\s\t`, true},
+		{`/p/q/r/s/t`, true},
 	}
 	// Instantiate posix object to manage a disk
 	var err error

cmd/posix.go

@@ -26,6 +26,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"syscall"
 
@@ -45,6 +46,7 @@ type posix struct {
 	suppliedDiskPath string
 	minFreeSpace     int64
 	minFreeInodes    int64
+	pool             sync.Pool
 }
 
 var errFaultyDisk = errors.New("Faulty disk")
@@ -114,6 +116,13 @@ func newPosix(diskPath string) (StorageAPI, error) {
 		diskPath:         diskPath,
 		minFreeSpace:     fsMinFreeSpace,
 		minFreeInodes:    fsMinFreeInodesPercent,
+		// 1MiB buffer pool for posix internal operations.
+		pool: sync.Pool{
+			New: func() interface{} {
+				b := make([]byte, readSizeV1)
+				return &b
+			},
+		},
 	}
 	st, err := os.Stat(preparePath(diskPath))
 	if err != nil {
@@ -144,7 +153,7 @@ func getDiskInfo(diskPath string) (di disk.Info, err error) {
 }
 
 // checkDiskFree verifies if disk path has sufficient minimum free disk space and files.
-func (s posix) checkDiskFree() (err error) {
+func (s *posix) checkDiskFree() (err error) {
 	di, err := getDiskInfo(s.diskPath)
 	if err != nil {
 		return err
@@ -584,7 +593,7 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 	}
 	// Create top level directories if they don't exist.
 	// with mode 0777 mkdir honors system umask.
-	if err = mkdirAll(filepath.Dir(filePath), 0777); err != nil {
+	if err = mkdirAll(preparePath(slashpath.Dir(filePath)), 0777); err != nil {
 		// File path cannot be verified since one of the parents is a file.
 		if isSysErrNotDir(err) {
 			return errFileAccessDenied
@@ -609,8 +618,13 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 	// Close upon return.
 	defer w.Close()
 
+	bufp := s.pool.Get().(*[]byte)
+
+	// Reuse buffer.
+	defer s.pool.Put(bufp)
+
 	// Return io.Copy
-	_, err = io.Copy(w, bytes.NewReader(buf))
+	_, err = io.CopyBuffer(w, bytes.NewReader(buf), *bufp)
 	return err
 }