From 37749f462339063597e053851cc782b730bb6398 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 2 Nov 2020 17:07:52 -0800 Subject: [PATCH] Optimize FileInfo(Version) transfer (#10775) File Info decoding, in particular, is showing up as a major allocator and time consumer for internode data transfers Switch to message pack for cross-server transfers: ``` MSGP: Size: 945 bytes BenchmarkEncodeFileInfoMsgp-32 1558444 866 ns/op 1.16 MB/s 0 B/op 0 allocs/op BenchmarkDecodeFileInfoMsgp-32 479968 2487 ns/op 0.40 MB/s 848 B/op 18 allocs/op GOB: Size: 1409 bytes BenchmarkEncodeFileInfoGOB-32 333339 3237 ns/op 0.31 MB/s 576 B/op 19 allocs/op BenchmarkDecodeFileInfoGOB-32 20869 57837 ns/op 0.02 MB/s 16439 B/op 428 allocs/op ``` --- cmd/metacache-entries.go | 4 +- cmd/storage-datatypes.go | 7 +- cmd/storage-datatypes_gen.go | 1082 +++++++++++++++++++++++++++++ cmd/storage-datatypes_gen_test.go | 575 +++++++++++++++ cmd/storage-datatypes_test.go | 73 ++ cmd/storage-rest-client.go | 31 +- cmd/storage-rest-server.go | 40 +- cmd/xl-storage.go | 6 +- 8 files changed, 1777 insertions(+), 41 deletions(-) create mode 100644 cmd/storage-datatypes_gen.go create mode 100644 cmd/storage-datatypes_gen_test.go create mode 100644 cmd/storage-datatypes_test.go diff --git a/cmd/metacache-entries.go b/cmd/metacache-entries.go index ae686cbd7..88bbafc5d 100644 --- a/cmd/metacache-entries.go +++ b/cmd/metacache-entries.go @@ -122,7 +122,7 @@ func (e *metaCacheEntry) fileInfo(bucket string) (*FileInfo, error) { return &FileInfo{ Volume: bucket, Name: e.name, - Mode: os.ModeDir, + Mode: uint32(os.ModeDir), }, nil } if e.cached == nil { @@ -146,7 +146,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro { Volume: bucket, Name: e.name, - Mode: os.ModeDir, + Mode: uint32(os.ModeDir), }, }, }, nil diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go index 63d28da4f..2f7ed2a3a 100644 --- a/cmd/storage-datatypes.go +++ b/cmd/storage-datatypes.go @@ -17,10 +17,11 @@ package cmd import ( - "os" "time" ) +//go:generate msgp -file=$GOFILE + // VolInfo - represents volume stat information. type VolInfo struct { // Name of the volume. @@ -74,6 +75,8 @@ func (f *FileInfoVersions) forwardPastVersion(v string) { } // FileInfo - represents file stat information. +//msgp:tuple FileInfo +// The above means that any added/deleted fields are incompatible. type FileInfo struct { // Name of the volume. Volume string @@ -105,7 +108,7 @@ type FileInfo struct { Size int64 // File mode bits. - Mode os.FileMode + Mode uint32 // File metadata Metadata map[string]string diff --git a/cmd/storage-datatypes_gen.go b/cmd/storage-datatypes_gen.go new file mode 100644 index 000000000..c6c2c6979 --- /dev/null +++ b/cmd/storage-datatypes_gen.go @@ -0,0 +1,1082 @@ +package cmd + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 13 { + err = msgp.ArrayError{Wanted: 13, Got: zb0001} + return + } + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.VersionID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + z.IsLatest, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "IsLatest") + return + } + z.Deleted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + z.DataDir, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + z.XLV1, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "XLV1") + return + } + z.ModTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Mode, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if z.Metadata == nil { + z.Metadata = make(map[string]string, zb0002) + } else if len(z.Metadata) > 0 { + for key := range z.Metadata { + delete(z.Metadata, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0001 string + var za0002 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Metadata", za0001) + return + } + z.Metadata[za0001] = za0002 + } + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0003) { + z.Parts = (z.Parts)[:zb0003] + } else { + z.Parts = make([]ObjectPartInfo, zb0003) + } + for za0003 := range z.Parts { + err = z.Parts[za0003].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + err = z.Erasure.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) { + // array header, size 13 + err = en.Append(0x9d) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + err = en.WriteString(z.VersionID) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + err = en.WriteBool(z.IsLatest) + if err != nil { + err = msgp.WrapError(err, "IsLatest") + return + } + err = en.WriteBool(z.Deleted) + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + err = en.WriteString(z.DataDir) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + err = en.WriteBool(z.XLV1) + if err != nil { + err = msgp.WrapError(err, "XLV1") + return + } + err = en.WriteTime(z.ModTime) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + err = en.WriteInt64(z.Size) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + err = en.WriteUint32(z.Mode) + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + err = en.WriteMapHeader(uint32(len(z.Metadata))) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + for za0001, za0002 := range z.Metadata { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0001) + return + } + } + err = en.WriteArrayHeader(uint32(len(z.Parts))) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + for za0003 := range z.Parts { + err = z.Parts[za0003].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + err = z.Erasure.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // array header, size 13 + o = append(o, 0x9d) + o = msgp.AppendString(o, z.Volume) + o = msgp.AppendString(o, z.Name) + o = msgp.AppendString(o, z.VersionID) + o = msgp.AppendBool(o, z.IsLatest) + o = msgp.AppendBool(o, z.Deleted) + o = msgp.AppendString(o, z.DataDir) + o = msgp.AppendBool(o, z.XLV1) + o = msgp.AppendTime(o, z.ModTime) + o = msgp.AppendInt64(o, z.Size) + o = msgp.AppendUint32(o, z.Mode) + o = msgp.AppendMapHeader(o, uint32(len(z.Metadata))) + for za0001, za0002 := range z.Metadata { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + o = msgp.AppendArrayHeader(o, uint32(len(z.Parts))) + for za0003 := range z.Parts { + o, err = z.Parts[za0003].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + o, err = z.Erasure.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 13 { + err = msgp.ArrayError{Wanted: 13, Got: zb0001} + return + } + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.VersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + z.IsLatest, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsLatest") + return + } + z.Deleted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + z.DataDir, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + z.XLV1, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "XLV1") + return + } + z.ModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Mode, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if z.Metadata == nil { + z.Metadata = make(map[string]string, zb0002) + } else if len(z.Metadata) > 0 { + for key := range z.Metadata { + delete(z.Metadata, key) + } + } + for zb0002 > 0 { + var za0001 string + var za0002 string + zb0002-- + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0001) + return + } + z.Metadata[za0001] = za0002 + } + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0003) { + z.Parts = (z.Parts)[:zb0003] + } else { + z.Parts = make([]ObjectPartInfo, zb0003) + } + for za0003 := range z.Parts { + bts, err = z.Parts[za0003].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + bts, err = z.Erasure.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *FileInfo) Msgsize() (s int) { + s = 1 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.StringPrefixSize + len(z.VersionID) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.DataDir) + msgp.BoolSize + msgp.TimeSize + msgp.Int64Size + msgp.Uint32Size + msgp.MapHeaderSize + if z.Metadata != nil { + for za0001, za0002 := range z.Metadata { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += msgp.ArrayHeaderSize + for za0003 := range z.Parts { + s += z.Parts[za0003].Msgsize() + } + s += z.Erasure.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *FileInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Volume": + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "LatestModTime": + z.LatestModTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LatestModTime") + return + } + case "Versions": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + if cap(z.Versions) >= int(zb0002) { + z.Versions = (z.Versions)[:zb0002] + } else { + z.Versions = make([]FileInfo, zb0002) + } + for za0001 := range z.Versions { + err = z.Versions[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Versions", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "Volume" + err = en.Append(0x84, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + // write "Name" + err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "LatestModTime" + err = en.Append(0xad, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.LatestModTime) + if err != nil { + err = msgp.WrapError(err, "LatestModTime") + return + } + // write "Versions" + err = en.Append(0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Versions))) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + for za0001 := range z.Versions { + err = z.Versions[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Versions", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *FileInfoVersions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "Volume" + o = append(o, 0x84, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65) + o = msgp.AppendString(o, z.Volume) + // string "Name" + o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "LatestModTime" + o = append(o, 0xad, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.LatestModTime) + // string "Versions" + o = append(o, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Versions))) + for za0001 := range z.Versions { + o, err = z.Versions[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Versions", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *FileInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Volume": + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "LatestModTime": + z.LatestModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LatestModTime") + return + } + case "Versions": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + if cap(z.Versions) >= int(zb0002) { + z.Versions = (z.Versions)[:zb0002] + } else { + z.Versions = make([]FileInfo, zb0002) + } + for za0001 := range z.Versions { + bts, err = z.Versions[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Versions", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *FileInfoVersions) Msgsize() (s int) { + s = 1 + 7 + msgp.StringPrefixSize + len(z.Volume) + 5 + msgp.StringPrefixSize + len(z.Name) + 14 + msgp.TimeSize + 9 + msgp.ArrayHeaderSize + for za0001 := range z.Versions { + s += z.Versions[za0001].Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *FilesInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Files": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + if cap(z.Files) >= int(zb0002) { + z.Files = (z.Files)[:zb0002] + } else { + z.Files = make([]FileInfo, zb0002) + } + for za0001 := range z.Files { + err = z.Files[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + case "IsTruncated": + z.IsTruncated, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *FilesInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Files" + err = en.Append(0x82, 0xa5, 0x46, 0x69, 0x6c, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Files))) + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + for za0001 := range z.Files { + err = z.Files[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + // write "IsTruncated" + err = en.Append(0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.IsTruncated) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *FilesInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Files" + o = append(o, 0x82, 0xa5, 0x46, 0x69, 0x6c, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Files))) + for za0001 := range z.Files { + o, err = z.Files[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + // string "IsTruncated" + o = append(o, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.IsTruncated) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *FilesInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Files": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + if cap(z.Files) >= int(zb0002) { + z.Files = (z.Files)[:zb0002] + } else { + z.Files = make([]FileInfo, zb0002) + } + for za0001 := range z.Files { + bts, err = z.Files[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + case "IsTruncated": + z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *FilesInfo) Msgsize() (s int) { + s = 1 + 6 + msgp.ArrayHeaderSize + for za0001 := range z.Files { + s += z.Files[za0001].Msgsize() + } + s += 12 + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *FilesInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "FilesVersions": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "FilesVersions") + return + } + if cap(z.FilesVersions) >= int(zb0002) { + z.FilesVersions = (z.FilesVersions)[:zb0002] + } else { + z.FilesVersions = make([]FileInfoVersions, zb0002) + } + for za0001 := range z.FilesVersions { + err = z.FilesVersions[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "FilesVersions", za0001) + return + } + } + case "IsTruncated": + z.IsTruncated, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *FilesInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "FilesVersions" + err = en.Append(0x82, 0xad, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.FilesVersions))) + if err != nil { + err = msgp.WrapError(err, "FilesVersions") + return + } + for za0001 := range z.FilesVersions { + err = z.FilesVersions[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "FilesVersions", za0001) + return + } + } + // write "IsTruncated" + err = en.Append(0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.IsTruncated) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *FilesInfoVersions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "FilesVersions" + o = append(o, 0x82, 0xad, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.FilesVersions))) + for za0001 := range z.FilesVersions { + o, err = z.FilesVersions[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "FilesVersions", za0001) + return + } + } + // string "IsTruncated" + o = append(o, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.IsTruncated) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *FilesInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "FilesVersions": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilesVersions") + return + } + if cap(z.FilesVersions) >= int(zb0002) { + z.FilesVersions = (z.FilesVersions)[:zb0002] + } else { + z.FilesVersions = make([]FileInfoVersions, zb0002) + } + for za0001 := range z.FilesVersions { + bts, err = z.FilesVersions[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "FilesVersions", za0001) + return + } + } + case "IsTruncated": + z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *FilesInfoVersions) Msgsize() (s int) { + s = 1 + 14 + msgp.ArrayHeaderSize + for za0001 := range z.FilesVersions { + s += z.FilesVersions[za0001].Msgsize() + } + s += 12 + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *VolInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Created": + z.Created, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z VolInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Name" + err = en.Append(0x82, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "Created" + err = en.Append(0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteTime(z.Created) + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z VolInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Name" + o = append(o, 0x82, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Created" + o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendTime(o, z.Created) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *VolInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Created": + z.Created, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z VolInfo) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + return +} diff --git a/cmd/storage-datatypes_gen_test.go b/cmd/storage-datatypes_gen_test.go new file mode 100644 index 000000000..16c78d68c --- /dev/null +++ b/cmd/storage-datatypes_gen_test.go @@ -0,0 +1,575 @@ +package cmd + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalFileInfo(t *testing.T) { + v := FileInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgFileInfo(b *testing.B) { + v := FileInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgFileInfo(b *testing.B) { + v := FileInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalFileInfo(b *testing.B) { + v := FileInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeFileInfo(t *testing.T) { + v := FileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeFileInfo Msgsize() is inaccurate") + } + + vn := FileInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeFileInfo(b *testing.B) { + v := FileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeFileInfo(b *testing.B) { + v := FileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalFileInfoVersions(t *testing.T) { + v := FileInfoVersions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeFileInfoVersions(t *testing.T) { + v := FileInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeFileInfoVersions Msgsize() is inaccurate") + } + + vn := FileInfoVersions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalFilesInfo(t *testing.T) { + v := FilesInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgFilesInfo(b *testing.B) { + v := FilesInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgFilesInfo(b *testing.B) { + v := FilesInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalFilesInfo(b *testing.B) { + v := FilesInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeFilesInfo(t *testing.T) { + v := FilesInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeFilesInfo Msgsize() is inaccurate") + } + + vn := FilesInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeFilesInfo(b *testing.B) { + v := FilesInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeFilesInfo(b *testing.B) { + v := FilesInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalFilesInfoVersions(t *testing.T) { + v := FilesInfoVersions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgFilesInfoVersions(b *testing.B) { + v := FilesInfoVersions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgFilesInfoVersions(b *testing.B) { + v := FilesInfoVersions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalFilesInfoVersions(b *testing.B) { + v := FilesInfoVersions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeFilesInfoVersions(t *testing.T) { + v := FilesInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeFilesInfoVersions Msgsize() is inaccurate") + } + + vn := FilesInfoVersions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeFilesInfoVersions(b *testing.B) { + v := FilesInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeFilesInfoVersions(b *testing.B) { + v := FilesInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalVolInfo(t *testing.T) { + v := VolInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgVolInfo(b *testing.B) { + v := VolInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgVolInfo(b *testing.B) { + v := VolInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalVolInfo(b *testing.B) { + v := VolInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeVolInfo(t *testing.T) { + v := VolInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeVolInfo Msgsize() is inaccurate") + } + + vn := VolInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeVolInfo(b *testing.B) { + v := VolInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeVolInfo(b *testing.B) { + v := VolInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/storage-datatypes_test.go b/cmd/storage-datatypes_test.go new file mode 100644 index 000000000..e91aa48fa --- /dev/null +++ b/cmd/storage-datatypes_test.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "bytes" + "encoding/gob" + "io/ioutil" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func BenchmarkDecodeFileInfoMsgp(b *testing.B) { + v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(1) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.Log("Size:", buf.Len(), "bytes") + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeFileInfoGOB(b *testing.B) { + v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}} + var buf bytes.Buffer + gob.NewEncoder(&buf).Encode(v) + encoded := buf.Bytes() + b.Log("Size:", buf.Len(), "bytes") + b.SetBytes(1) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + dec := gob.NewDecoder(bytes.NewBuffer(encoded)) + err := dec.Decode(&v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeFileInfoMsgp(b *testing.B) { + v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}} + b.SetBytes(1) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := msgp.Encode(ioutil.Discard, &v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeFileInfoGOB(b *testing.B) { + v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}} + enc := gob.NewEncoder(ioutil.Discard) + b.SetBytes(1) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := enc.Encode(&v) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 45284ee6a..d83ca2768 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -34,6 +34,7 @@ import ( "github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/rest" xnet "github.com/minio/minio/pkg/net" + "github.com/tinylib/msgp/msgp" ) func isNetworkError(err error) bool { @@ -290,7 +291,7 @@ func (client *storageRESTClient) WriteMetadata(ctx context.Context, volume, path values.Set(storageRESTFilePath, path) var reader bytes.Buffer - if err := gob.NewEncoder(&reader).Encode(fi); err != nil { + if err := msgp.Encode(&reader, &fi); err != nil { return err } @@ -305,7 +306,7 @@ func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path values.Set(storageRESTFilePath, path) var buffer bytes.Buffer - if err := gob.NewEncoder(&buffer).Encode(fi); err != nil { + if err := msgp.Encode(&buffer, &fi); err != nil { return err } @@ -341,7 +342,7 @@ func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, values.Set(storageRESTFilePath, path) var reader bytes.Buffer - if err := gob.NewEncoder(&reader).Encode(fi); err != nil { + if err := msgp.Encode(&reader, &fi); err != nil { logger.LogIf(context.Background(), err) return err } @@ -377,8 +378,7 @@ func (client *storageRESTClient) ReadVersion(ctx context.Context, volume, path, return fi, err } defer http.DrainBody(respBody) - - err = gob.NewDecoder(respBody).Decode(&fi) + err = msgp.Decode(respBody, &fi) return fi, err } @@ -447,10 +447,10 @@ func (client *storageRESTClient) WalkSplunk(ctx context.Context, volume, dirPath defer close(ch) defer http.DrainBody(respBody) - decoder := gob.NewDecoder(respBody) + decoder := msgp.NewReader(respBody) for { var fi FileInfo - if gerr := decoder.Decode(&fi); gerr != nil { + if gerr := fi.DecodeMsg(decoder); gerr != nil { // Upon error return return } @@ -482,12 +482,12 @@ func (client *storageRESTClient) WalkVersions(ctx context.Context, volume, dirPa defer close(ch) defer http.DrainBody(respBody) - decoder := gob.NewDecoder(respBody) + decoder := msgp.NewReader(respBody) for { var fi FileInfoVersions - if gerr := decoder.Decode(&fi); gerr != nil { + if gerr := fi.DecodeMsg(decoder); gerr != nil { // Upon error return - if gerr != io.EOF { + if msgp.Cause(gerr) != io.EOF { logger.LogIf(GlobalContext, gerr) } return @@ -519,10 +519,10 @@ func (client *storageRESTClient) Walk(ctx context.Context, volume, dirPath, mark defer close(ch) defer http.DrainBody(respBody) - decoder := gob.NewDecoder(respBody) + decoder := msgp.NewReader(respBody) for { var fi FileInfo - if gerr := decoder.Decode(&fi); gerr != nil { + if gerr := fi.DecodeMsg(decoder); gerr != nil { // Upon error return return } @@ -575,10 +575,11 @@ func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume stri values.Set(storageRESTTotalVersions, strconv.Itoa(len(versions))) var buffer bytes.Buffer - encoder := gob.NewEncoder(&buffer) + encoder := msgp.NewWriter(&buffer) for _, version := range versions { - encoder.Encode(&version) + version.EncodeMsg(encoder) } + logger.LogIf(ctx, encoder.Flush()) errs = make([]error, len(versions)) @@ -632,7 +633,7 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st values.Set(storageRESTFilePath, path) var reader bytes.Buffer - if err := gob.NewEncoder(&reader).Encode(fi); err != nil { + if err := msgp.Encode(&reader, &fi); err != nil { return err } diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index 315ae7d45..e42f5f4e8 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -33,6 +33,8 @@ import ( "strings" "time" + "github.com/tinylib/msgp/msgp" + jwtreq "github.com/dgrijalva/jwt-go/request" "github.com/gorilla/mux" "github.com/minio/minio/cmd/config" @@ -308,7 +310,7 @@ func (s *storageRESTServer) DeleteVersionHandler(w http.ResponseWriter, r *http. } var fi FileInfo - if err := gob.NewDecoder(r.Body).Decode(&fi); err != nil { + if err := msgp.Decode(r.Body, &fi); err != nil { s.writeErrorResponse(w, err) return } @@ -339,9 +341,7 @@ func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Re s.writeErrorResponse(w, err) return } - - gob.NewEncoder(w).Encode(fi) - w.(http.Flusher).Flush() + logger.LogIf(r.Context(), msgp.Encode(w, &fi)) } // WriteMetadata write new updated metadata. @@ -359,13 +359,12 @@ func (s *storageRESTServer) WriteMetadataHandler(w http.ResponseWriter, r *http. } var fi FileInfo - err := gob.NewDecoder(r.Body).Decode(&fi) - if err != nil { + if err := msgp.Decode(r.Body, &fi); err != nil { s.writeErrorResponse(w, err) return } - err = s.storage.WriteMetadata(r.Context(), volume, filePath, fi) + err := s.storage.WriteMetadata(r.Context(), volume, filePath, fi) if err != nil { s.writeErrorResponse(w, err) } @@ -411,7 +410,7 @@ func (s *storageRESTServer) CheckPartsHandler(w http.ResponseWriter, r *http.Req } var fi FileInfo - if err := gob.NewDecoder(r.Body).Decode(&fi); err != nil { + if err := msgp.Decode(r.Body, &fi); err != nil { s.writeErrorResponse(w, err) return } @@ -541,16 +540,17 @@ func (s *storageRESTServer) WalkSplunkHandler(w http.ResponseWriter, r *http.Req markerPath := vars[storageRESTMarkerPath] setEventStreamHeaders(w) - encoder := gob.NewEncoder(w) fch, err := s.storage.WalkSplunk(r.Context(), volume, dirPath, markerPath, r.Context().Done()) if err != nil { s.writeErrorResponse(w, err) return } + encoder := msgp.NewWriter(w) for fi := range fch { - encoder.Encode(&fi) + logger.LogIf(r.Context(), fi.EncodeMsg(encoder)) } + logger.LogIf(r.Context(), encoder.Flush()) } // WalkVersionsHandler - remote caller to start walking at a requested directory path. @@ -569,16 +569,17 @@ func (s *storageRESTServer) WalkVersionsHandler(w http.ResponseWriter, r *http.R } setEventStreamHeaders(w) - encoder := gob.NewEncoder(w) fch, err := s.storage.WalkVersions(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done()) if err != nil { s.writeErrorResponse(w, err) return } + encoder := msgp.NewWriter(w) for fi := range fch { - logger.LogIf(r.Context(), encoder.Encode(&fi)) + logger.LogIf(r.Context(), fi.EncodeMsg(encoder)) } + logger.LogIf(r.Context(), encoder.Flush()) } // WalkHandler - remote caller to start walking at a requested directory path. @@ -597,16 +598,17 @@ func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request) } setEventStreamHeaders(w) - encoder := gob.NewEncoder(w) fch, err := s.storage.Walk(r.Context(), volume, dirPath, markerPath, recursive, r.Context().Done()) if err != nil { s.writeErrorResponse(w, err) return } + encoder := msgp.NewWriter(w) for fi := range fch { - logger.LogIf(r.Context(), encoder.Encode(&fi)) + logger.LogIf(r.Context(), fi.EncodeMsg(encoder)) } + logger.LogIf(r.Context(), encoder.Flush()) } // ListDirHandler - list a directory. @@ -674,9 +676,10 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http } versions := make([]FileInfo, totalVersions) - decoder := gob.NewDecoder(r.Body) + decoder := msgp.NewReader(r.Body) for i := 0; i < totalVersions; i++ { - if err := decoder.Decode(&versions[i]); err != nil { + dst := &versions[i] + if err := dst.DecodeMsg(decoder); err != nil { s.writeErrorResponse(w, err) return } @@ -979,8 +982,7 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req } var fi FileInfo - err := gob.NewDecoder(r.Body).Decode(&fi) - if err != nil { + if err := msgp.Decode(r.Body, &fi); err != nil { s.writeErrorResponse(w, err) return } @@ -988,7 +990,7 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req setEventStreamHeaders(w) encoder := gob.NewEncoder(w) done := keepHTTPResponseAlive(w) - err = s.storage.VerifyFile(r.Context(), volume, filePath, fi) + err := s.storage.VerifyFile(r.Context(), volume, filePath, fi) done(nil) vresp := &VerifyFileResp{} if err != nil { diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 3d21ea8dc..63442502e 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -897,7 +897,7 @@ func (s *xlStorage) WalkSplunk(ctx context.Context, volume, dirPath, marker stri fi = FileInfo{ Volume: volume, Name: walkResult.entry, - Mode: os.ModeDir, + Mode: uint32(os.ModeDir), } } else { var err error @@ -985,7 +985,7 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st { Volume: volume, Name: walkResult.entry, - Mode: os.ModeDir, + Mode: uint32(os.ModeDir), }, }, } @@ -1066,7 +1066,7 @@ func (s *xlStorage) Walk(ctx context.Context, volume, dirPath, marker string, re fi = FileInfo{ Volume: volume, Name: walkResult.entry, - Mode: os.ModeDir, + Mode: uint32(os.ModeDir), } } else { var err error