oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Optimize FileInfo(Version) transfer (#10775)

Browse Source 
File Info decoding, in particular, is showing up as a major 
allocator and time consumer for internode data transfers

Switch to message pack for cross-server transfers:

```
MSGP:

Size: 945 bytes

BenchmarkEncodeFileInfoMsgp-32    	 1558444	       866 ns/op	   1.16 MB/s	       0 B/op	       0 allocs/op
BenchmarkDecodeFileInfoMsgp-32    	  479968	      2487 ns/op	   0.40 MB/s	     848 B/op	      18 allocs/op

GOB:

Size: 1409 bytes

BenchmarkEncodeFileInfoGOB-32    	  333339	      3237 ns/op	   0.31 MB/s	     576 B/op	      19 allocs/op
BenchmarkDecodeFileInfoGOB-32    	   20869	     57837 ns/op	   0.02 MB/s	   16439 B/op	     428 allocs/op
```
master
Klaus Post 4 years ago committed by GitHub
parent 86e0d272f3
commit 37749f4623
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1777 additions and 41 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      cmd/metacache-entries.go
  2. 7
      cmd/storage-datatypes.go
  3. 1082
      cmd/storage-datatypes_gen.go
  4. 575
      cmd/storage-datatypes_gen_test.go
  5. 73
      cmd/storage-datatypes_test.go
  6. 31
      cmd/storage-rest-client.go
  7. 40
      cmd/storage-rest-server.go
  8. 6
      cmd/xl-storage.go

4
cmd/metacache-entries.go
Unescape View File

@ -122,7 +122,7 @@ func (e *metaCacheEntry) fileInfo(bucket string) (*FileInfo, error) {
return &FileInfo{
Volume: bucket,
Name: e.name,
Mode: os.ModeDir,
Mode: uint32(os.ModeDir),
}, nil
}
if e.cached == nil {
@ -146,7 +146,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro
{
Volume: bucket,
Name: e.name,
Mode: os.ModeDir,
Mode: uint32(os.ModeDir),
},
},
}, nil

7
cmd/storage-datatypes.go
Unescape View File

@ -17,10 +17,11 @@
package cmd
import (
"os"
"time"
)
//go:generate msgp -file=$GOFILE
// VolInfo - represents volume stat information.
type VolInfo struct {
// Name of the volume.
@ -74,6 +75,8 @@ func (f *FileInfoVersions) forwardPastVersion(v string) {
}
// FileInfo - represents file stat information.
//msgp:tuple FileInfo
// The above means that any added/deleted fields are incompatible.
type FileInfo struct {
// Name of the volume.
Volume string
@ -105,7 +108,7 @@ type FileInfo struct {
Size int64
// File mode bits.
Mode os.FileMode
Mode uint32
// File metadata
Metadata map[string]string

1082
cmd/storage-datatypes_gen.go
View File

File diff suppressed because it is too large Load Diff

575
cmd/storage-datatypes_gen_test.go
Unescape View File

@ -0,0 +1,575 @@
package cmd
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
import (
"bytes"
"testing"
"github.com/tinylib/msgp/msgp"
)
func TestMarshalUnmarshalFileInfo(t *testing.T) {
v := FileInfo{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgFileInfo(b *testing.B) {
v := FileInfo{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgFileInfo(b *testing.B) {
v := FileInfo{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalFileInfo(b *testing.B) {
v := FileInfo{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeFileInfo(t *testing.T) {
v := FileInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeFileInfo Msgsize() is inaccurate")
}
vn := FileInfo{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeFileInfo(b *testing.B) {
v := FileInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeFileInfo(b *testing.B) {
v := FileInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
func TestMarshalUnmarshalFileInfoVersions(t *testing.T) {
v := FileInfoVersions{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgFileInfoVersions(b *testing.B) {
v := FileInfoVersions{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgFileInfoVersions(b *testing.B) {
v := FileInfoVersions{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalFileInfoVersions(b *testing.B) {
v := FileInfoVersions{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeFileInfoVersions(t *testing.T) {
v := FileInfoVersions{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeFileInfoVersions Msgsize() is inaccurate")
}
vn := FileInfoVersions{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeFileInfoVersions(b *testing.B) {
v := FileInfoVersions{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeFileInfoVersions(b *testing.B) {
v := FileInfoVersions{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
func TestMarshalUnmarshalFilesInfo(t *testing.T) {
v := FilesInfo{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgFilesInfo(b *testing.B) {
v := FilesInfo{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgFilesInfo(b *testing.B) {
v := FilesInfo{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalFilesInfo(b *testing.B) {
v := FilesInfo{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeFilesInfo(t *testing.T) {
v := FilesInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeFilesInfo Msgsize() is inaccurate")
}
vn := FilesInfo{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeFilesInfo(b *testing.B) {
v := FilesInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeFilesInfo(b *testing.B) {
v := FilesInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
func TestMarshalUnmarshalFilesInfoVersions(t *testing.T) {
v := FilesInfoVersions{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgFilesInfoVersions(b *testing.B) {
v := FilesInfoVersions{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgFilesInfoVersions(b *testing.B) {
v := FilesInfoVersions{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalFilesInfoVersions(b *testing.B) {
v := FilesInfoVersions{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeFilesInfoVersions(t *testing.T) {
v := FilesInfoVersions{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeFilesInfoVersions Msgsize() is inaccurate")
}
vn := FilesInfoVersions{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeFilesInfoVersions(b *testing.B) {
v := FilesInfoVersions{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeFilesInfoVersions(b *testing.B) {
v := FilesInfoVersions{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
func TestMarshalUnmarshalVolInfo(t *testing.T) {
v := VolInfo{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgVolInfo(b *testing.B) {
v := VolInfo{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgVolInfo(b *testing.B) {
v := VolInfo{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalVolInfo(b *testing.B) {
v := VolInfo{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeVolInfo(t *testing.T) {
v := VolInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeVolInfo Msgsize() is inaccurate")
}
vn := VolInfo{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeVolInfo(b *testing.B) {
v := VolInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeVolInfo(b *testing.B) {
v := VolInfo{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}

73
cmd/storage-datatypes_test.go
Unescape View File

@ -0,0 +1,73 @@
package cmd
import (
"bytes"
"encoding/gob"
"io/ioutil"
"testing"
"github.com/tinylib/msgp/msgp"
)
func BenchmarkDecodeFileInfoMsgp(b *testing.B) {
v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(1)
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.Log("Size:", buf.Len(), "bytes")
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkDecodeFileInfoGOB(b *testing.B) {
v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}}
var buf bytes.Buffer
gob.NewEncoder(&buf).Encode(v)
encoded := buf.Bytes()
b.Log("Size:", buf.Len(), "bytes")
b.SetBytes(1)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
dec := gob.NewDecoder(bytes.NewBuffer(encoded))
err := dec.Decode(&v)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkEncodeFileInfoMsgp(b *testing.B) {
v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}}
b.SetBytes(1)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := msgp.Encode(ioutil.Discard, &v)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkEncodeFileInfoGOB(b *testing.B) {
v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}}
enc := gob.NewEncoder(ioutil.Discard)
b.SetBytes(1)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := enc.Encode(&v)
if err != nil {
b.Fatal(err)
}
}
}

31
cmd/storage-rest-client.go
Unescape View File

@ -34,6 +34,7 @@ import (
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/cmd/rest"
xnet "github.com/minio/minio/pkg/net"
"github.com/tinylib/msgp/msgp"
)
func isNetworkError(err error) bool {
@ -290,7 +291,7 @@ func (client *storageRESTClient) WriteMetadata(ctx context.Context, volume, path
values.Set(storageRESTFilePath, path)
var reader bytes.Buffer
if err := gob.NewEncoder(&reader).Encode(fi); err != nil {
if err := msgp.Encode(&reader, &fi); err != nil {
return err
}
@ -305,7 +306,7 @@ func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path
values.Set(storageRESTFilePath, path)
var buffer bytes.Buffer
if err := gob.NewEncoder(&buffer).Encode(fi); err != nil {
if err := msgp.Encode(&buffer, &fi); err != nil {
return err
}
@ -341,7 +342,7 @@ func (client *storageRESTClient) CheckParts(ctx context.Context, volume string,
values.Set(storageRESTFilePath, path)
var reader bytes.Buffer
if err := gob.NewEncoder(&reader).Encode(fi); err != nil {
if err := msgp.Encode(&reader, &fi); err != nil {
logger.LogIf(context.Background(), err)
return err
}
@ -377,8 +378,7 @@ func (client *storageRESTClient) ReadVersion(ctx context.Context, volume, path,
return fi, err
}
defer http.DrainBody(respBody)
err = gob.NewDecoder(respBody).Decode(&fi)
err = msgp.Decode(respBody, &fi)
return fi, err
}
@ -447,10 +447,10 @@ func (client *storageRESTClient) WalkSplunk(ctx context.Context, volume, dirPath
defer close(ch)
defer http.DrainBody(respBody)
decoder := gob.NewDecoder(respBody)
decoder := msgp.NewReader(respBody)
for {
var fi FileInfo
if gerr := decoder.Decode(&fi); gerr != nil {
if gerr := fi.DecodeMsg(decoder); gerr != nil {
// Upon error return
return
}
@ -482,12 +482,12 @@ func (client *storageRESTClient) WalkVersions(ctx context.Context, volume, dirPa
defer close(ch)
defer http.DrainBody(respBody)
decoder := gob.NewDecoder(respBody)
decoder := msgp.NewReader(respBody)
for {
var fi FileInfoVersions
if gerr := decoder.Decode(&fi); gerr != nil {
if gerr := fi.DecodeMsg(decoder); gerr != nil {
// Upon error return
if gerr != io.EOF {
if msgp.Cause(gerr) != io.EOF {
logger.LogIf(GlobalContext, gerr)
}
return
@ -519,10 +519,10 @@ func (client *storageRESTClient) Walk(ctx context.Context, volume, dirPath, mark
defer close(ch)
defer http.DrainBody(respBody)
decoder := gob.NewDecoder(respBody)
decoder := msgp.NewReader(respBody)
for {
var fi FileInfo
if gerr := decoder.Decode(&fi); gerr != nil {
if gerr := fi.DecodeMsg(decoder); gerr != nil {
// Upon error return
return
}
@ -575,10 +575,11 @@ func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume stri
values.Set(storageRESTTotalVersions, strconv.Itoa(len(versions)))
var buffer bytes.Buffer
encoder := gob.NewEncoder(&buffer)
encoder := msgp.NewWriter(&buffer)
for _, version := range versions {
encoder.Encode(&version)
version.EncodeMsg(encoder)
}
logger.LogIf(ctx, encoder.Flush())
errs = make([]error, len(versions))
@ -632,7 +633,7 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st
values.Set(storageRESTFilePath, path)
var reader bytes.Buffer
if err := gob.NewEncoder(&reader).Encode(fi); err != nil {
if err := msgp.Encode(&reader, &fi); err != nil {
return err
}

40
cmd/storage-rest-server.go
Unescape View File

@ -33,6 +33,8 @@ import (
"strings"
"time"
"github.com/tinylib/msgp/msgp"
jwtreq "github.com/dgrijalva/jwt-go/request"
"github.com/gorilla/mux"
"github.com/minio/minio/cmd/config"
@ -308,7 +310,7 @@ func (s *storageRESTServer) DeleteVersionHandler(w http.ResponseWriter, r *http.
}
var fi FileInfo
if err := gob.NewDecoder(r.Body).Decode(&fi); err != nil {
if err := msgp.Decode(r.Body, &fi); err != nil {
s.writeErrorResponse(w, err)
return
}
@ -339,9 +341,7 @@ func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Re
s.writeErrorResponse(w, err)
return
}
gob.NewEncoder(w).Encode(fi)
w.(http.Flusher).Flush()
logger.LogIf(r.Context(), msgp.Encode(w, &fi))
}
// WriteMetadata write new updated metadata.
@ -359,13 +359,12 @@ func (s *storageRESTServer) WriteMetadataHandler(w http.ResponseWriter, r *http.
}
var fi FileInfo
err := gob.NewDecoder(r.Body).Decode(&fi)
if err != nil {
if err := msgp.Decode(r.Body, &fi); err != nil {
s.writeErrorResponse(w, err)
return
}
err = s.storage.WriteMetadata(r.Context(), volume, filePath, fi)
err := s.storage.WriteMetadata(r.Context(), volume, filePath, fi)
if err != nil {
s.writeErrorResponse(w, err)
}
@ -411,7 +410,7 @@ func (s *storageRESTServer) CheckPartsHandler(w http.ResponseWriter, r *http.Req
}
var fi FileInfo
if err := gob.NewDecoder(r.Body).Decode(&fi); err != nil {
if err := msgp.Decode(r.Body, &fi); err != nil {
s.writeErrorResponse(w, err)
return
}
@ -541,16 +540,17 @@ func (s *storageRESTServer) WalkSplunkHandler(w http.ResponseWriter, r *http.Req
markerPath := vars[storageRESTMarkerPath]
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
fch, err := s.storage.WalkSplunk(r.Context(), volume, dirPath, markerPath, r.Context().Done())
if err != nil {
s.writeErrorResponse(w, err)
return
}
encoder := msgp.NewWriter(w)
for fi := range fch {
encoder.Encode(&fi)
logger.LogIf(r.Context(), fi.EncodeMsg(encoder))
}
logger.LogIf(r.Context(), encoder.Flush())
}
// WalkVersionsHandler - remote caller to start walking at a requested directory path.
@ -569,16 +569,17 @@ func (s *storageRESTServer) WalkVersionsHandler(w http.ResponseWriter, r *http.R
}
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
fch, err := s.storage.WalkVersions(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done())
if err != nil {
s.writeErrorResponse(w, err)
return
}
encoder := msgp.NewWriter(w)
for fi := range fch {
logger.LogIf(r.Context(), encoder.Encode(&fi))
logger.LogIf(r.Context(), fi.EncodeMsg(encoder))
}
logger.LogIf(r.Context(), encoder.Flush())
}
// WalkHandler - remote caller to start walking at a requested directory path.
@ -597,16 +598,17 @@ func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request)
}
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
fch, err := s.storage.Walk(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done())
if err != nil {
s.writeErrorResponse(w, err)
return
}
encoder := msgp.NewWriter(w)
for fi := range fch {
logger.LogIf(r.Context(), encoder.Encode(&fi))
logger.LogIf(r.Context(), fi.EncodeMsg(encoder))
}
logger.LogIf(r.Context(), encoder.Flush())
}
// ListDirHandler - list a directory.
@ -674,9 +676,10 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
}
versions := make([]FileInfo, totalVersions)
decoder := gob.NewDecoder(r.Body)
decoder := msgp.NewReader(r.Body)
for i := 0; i < totalVersions; i++ {
if err := decoder.Decode(&versions[i]); err != nil {
dst := &versions[i]
if err := dst.DecodeMsg(decoder); err != nil {
s.writeErrorResponse(w, err)
return
}
@ -979,8 +982,7 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req
}
var fi FileInfo
err := gob.NewDecoder(r.Body).Decode(&fi)
if err != nil {
if err := msgp.Decode(r.Body, &fi); err != nil {
s.writeErrorResponse(w, err)
return
}
@ -988,7 +990,7 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req
setEventStreamHeaders(w)
encoder := gob.NewEncoder(w)
done := keepHTTPResponseAlive(w)
err = s.storage.VerifyFile(r.Context(), volume, filePath, fi)
err := s.storage.VerifyFile(r.Context(), volume, filePath, fi)
done(nil)
vresp := &VerifyFileResp{}
if err != nil {

6
cmd/xl-storage.go
Unescape View File

@ -897,7 +897,7 @@ func (s *xlStorage) WalkSplunk(ctx context.Context, volume, dirPath, marker stri
fi = FileInfo{
Volume: volume,
Name: walkResult.entry,
Mode: os.ModeDir,
Mode: uint32(os.ModeDir),
}
} else {
var err error
@ -985,7 +985,7 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st
{
Volume: volume,
Name: walkResult.entry,
Mode: os.ModeDir,
Mode: uint32(os.ModeDir),
},
},
}
@ -1066,7 +1066,7 @@ func (s *xlStorage) Walk(ctx context.Context, volume, dirPath, marker string, re
fi = FileInfo{
Volume: volume,
Name: walkResult.entry,
Mode: os.ModeDir,
Mode: uint32(os.ModeDir),
}
} else {
var err error

Write Preview
Loading…
Cancel
Save