fix: simplify data structure before release (#9968)

- additionally upgrade to msgp@v1.1.2
- change StatModTime,StatSize fields as
  simple Size/ModTime
- reduce 50000 entries per List batch to 10000
  as client needs to wait too long to see the
  first batch some times which is not desired
  and it is worth we write the data as soon
  as we have it.
master
Harshavardhana 4 years ago committed by GitHub
parent cdb0e6ffed
commit c087a05b43
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      cmd/api-response.go
  2. 105
      cmd/xl-storage-format-v2.go
  3. 28
      cmd/xl-storage-format-v2_gen.go
  4. 2
      go.mod
  5. 4
      go.sum

@ -35,7 +35,7 @@ import (
const ( const (
// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z // RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision. iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
maxObjectList = 50000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
maxDeleteList = 10000 // Limit number of objects deleted in a delete call. maxDeleteList = 10000 // Limit number of objects deleted in a delete call.
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse. maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
maxPartsList = 10000 // Limit number of parts in a listPartsResponse. maxPartsList = 10000 // Limit number of parts in a listPartsResponse.

@ -156,8 +156,8 @@ type xlMetaV2Object struct {
PartETags []string `json:"PartETags" msg:"PartETags"` // Part ETags PartETags []string `json:"PartETags" msg:"PartETags"` // Part ETags
PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes
PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,omitempty"` // Part ActualSizes (compression) PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,omitempty"` // Part ActualSizes (compression)
StatSize int64 `json:"Size" msg:"Size"` // Object version size Size int64 `json:"Size" msg:"Size"` // Object version size
StatModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time ModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time
MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Object version internal metadata MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Object version internal metadata
MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,omitempty"` // Object version metadata set by user MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,omitempty"` // Object version metadata set by user
} }
@ -184,7 +184,7 @@ func (j xlMetaV2Version) Valid() bool {
j.ObjectV2.ErasureAlgorithm.valid() && j.ObjectV2.ErasureAlgorithm.valid() &&
j.ObjectV2.BitrotChecksumAlgo.valid() && j.ObjectV2.BitrotChecksumAlgo.valid() &&
isXLMetaErasureInfoValid(j.ObjectV2.ErasureM, j.ObjectV2.ErasureN) && isXLMetaErasureInfoValid(j.ObjectV2.ErasureM, j.ObjectV2.ErasureN) &&
j.ObjectV2.StatModTime > 0 j.ObjectV2.ModTime > 0
case DeleteType: case DeleteType:
return j.DeleteMarker != nil && return j.DeleteMarker != nil &&
j.DeleteMarker.ModTime > 0 j.DeleteMarker.ModTime > 0
@ -271,8 +271,8 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
ObjectV2: &xlMetaV2Object{ ObjectV2: &xlMetaV2Object{
VersionID: uv, VersionID: uv,
DataDir: dd, DataDir: dd,
StatSize: fi.Size, Size: fi.Size,
StatModTime: fi.ModTime.UnixNano(), ModTime: fi.ModTime.UnixNano(),
ErasureAlgorithm: ReedSolomon, ErasureAlgorithm: ReedSolomon,
ErasureM: fi.Erasure.DataBlocks, ErasureM: fi.Erasure.DataBlocks,
ErasureN: fi.Erasure.ParityBlocks, ErasureN: fi.Erasure.ParityBlocks,
@ -373,8 +373,8 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
fi := FileInfo{ fi := FileInfo{
Volume: volume, Volume: volume,
Name: path, Name: path,
Size: j.StatSize, Size: j.Size,
ModTime: time.Unix(0, j.StatModTime).UTC(), ModTime: time.Unix(0, j.ModTime).UTC(),
VersionID: versionID, VersionID: versionID,
} }
fi.Parts = make([]ObjectPartInfo, len(j.PartNumbers)) fi.Parts = make([]ObjectPartInfo, len(j.PartNumbers))
@ -463,7 +463,7 @@ func (z xlMetaV2) TotalSize() int64 {
for i := range z.Versions { for i := range z.Versions {
switch z.Versions[i].Type { switch z.Versions[i].Type {
case ObjectType: case ObjectType:
total += z.Versions[i].ObjectV2.StatSize total += z.Versions[i].ObjectV2.Size
case LegacyType: case LegacyType:
total += z.Versions[i].ObjectV1.Stat.Size total += z.Versions[i].ObjectV1.Stat.Size
} }
@ -488,8 +488,6 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
fi, err = version.DeleteMarker.ToFileInfo(volume, path) fi, err = version.DeleteMarker.ToFileInfo(volume, path)
case LegacyType: case LegacyType:
fi, err = version.ObjectV1.ToFileInfo(volume, path) fi, err = version.ObjectV1.ToFileInfo(volume, path)
default:
continue
} }
if err != nil { if err != nil {
return nil, latestModTime, err return nil, latestModTime, err
@ -498,12 +496,7 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
latestModTime = fi.ModTime latestModTime = fi.ModTime
latestVersionID = fi.VersionID latestVersionID = fi.VersionID
} }
switch version.Type { versions = append(versions, fi)
case LegacyType:
fallthrough
case ObjectType, DeleteType:
versions = append(versions, fi)
}
} }
// We didn't find the version in delete markers so latest version // We didn't find the version in delete markers so latest version
@ -521,44 +514,53 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
// ToFileInfo converts xlMetaV2 into a common FileInfo datastructure // ToFileInfo converts xlMetaV2 into a common FileInfo datastructure
// for consumption across callers. // for consumption across callers.
func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) { func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) {
var uv uuid.UUID var uv uuid.UUID
if versionID != "" { if versionID != "" {
uv, _ = uuid.Parse(versionID) uv, _ = uuid.Parse(versionID)
} }
var latestModTime time.Time
var latestIndex int
for i, version := range z.Versions {
if !version.Valid() {
logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version))
return FileInfo{}, errFileNotFound
}
var modTime time.Time
switch version.Type {
case ObjectType:
modTime = time.Unix(0, version.ObjectV2.ModTime)
case DeleteType:
modTime = time.Unix(0, version.DeleteMarker.ModTime)
case LegacyType:
modTime = version.ObjectV1.Stat.ModTime
}
if modTime.After(latestModTime) {
latestModTime = modTime
latestIndex = i
}
}
if versionID == "" { if versionID == "" {
var latestModTime time.Time if len(z.Versions) >= 1 {
var latestIndex int if !z.Versions[latestIndex].Valid() {
for i, version := range z.Versions { logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", z.Versions[latestIndex]))
if !version.Valid() {
logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version))
return FileInfo{}, errFileNotFound return FileInfo{}, errFileNotFound
} }
var modTime time.Time
switch version.Type {
case ObjectType:
modTime = time.Unix(0, version.ObjectV2.StatModTime)
case DeleteType:
modTime = time.Unix(0, version.DeleteMarker.ModTime)
case LegacyType:
modTime = version.ObjectV1.Stat.ModTime
default:
continue
}
if modTime.After(latestModTime) {
latestModTime = modTime
latestIndex = i
}
}
if len(z.Versions) >= 1 {
switch z.Versions[latestIndex].Type { switch z.Versions[latestIndex].Type {
case ObjectType: case ObjectType:
return z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path) fi, err = z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path)
fi.IsLatest = true
return fi, err
case DeleteType: case DeleteType:
return z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path) fi, err = z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path)
fi.IsLatest = true
return fi, err
case LegacyType: case LegacyType:
return z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path) fi, err = z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path)
fi.IsLatest = true
return fi, err
} }
} }
return FileInfo{}, errFileNotFound return FileInfo{}, errFileNotFound
@ -575,23 +577,22 @@ func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) {
switch version.Type { switch version.Type {
case ObjectType: case ObjectType:
if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) { if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) {
return version.ObjectV2.ToFileInfo(volume, path) fi, err = version.ObjectV2.ToFileInfo(volume, path)
fi.IsLatest = latestModTime.Equal(fi.ModTime)
return fi, err
} }
case LegacyType: case LegacyType:
if version.ObjectV1.VersionID == versionID { if version.ObjectV1.VersionID == versionID {
return version.ObjectV1.ToFileInfo(volume, path) fi, err = version.ObjectV1.ToFileInfo(volume, path)
fi.IsLatest = latestModTime.Equal(fi.ModTime)
return fi, err
} }
case DeleteType: case DeleteType:
if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) { if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) {
return version.DeleteMarker.ToFileInfo(volume, path) fi, err = version.DeleteMarker.ToFileInfo(volume, path)
} fi.IsLatest = latestModTime.Equal(fi.ModTime)
default: return fi, err
logger.LogIf(GlobalContext, fmt.Errorf("unknown version type: %v", version.Type))
if versionID == "" {
return FileInfo{}, errFileNotFound
} }
return FileInfo{}, errFileVersionNotFound
} }
} }

@ -606,15 +606,15 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
} }
} }
case "Size": case "Size":
z.StatSize, err = dc.ReadInt64() z.Size, err = dc.ReadInt64()
if err != nil { if err != nil {
err = msgp.WrapError(err, "StatSize") err = msgp.WrapError(err, "Size")
return return
} }
case "MTime": case "MTime":
z.StatModTime, err = dc.ReadInt64() z.ModTime, err = dc.ReadInt64()
if err != nil { if err != nil {
err = msgp.WrapError(err, "StatModTime") err = msgp.WrapError(err, "ModTime")
return return
} }
case "MetaSys": case "MetaSys":
@ -885,9 +885,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
if err != nil { if err != nil {
return return
} }
err = en.WriteInt64(z.StatSize) err = en.WriteInt64(z.Size)
if err != nil { if err != nil {
err = msgp.WrapError(err, "StatSize") err = msgp.WrapError(err, "Size")
return return
} }
// write "MTime" // write "MTime"
@ -895,9 +895,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
if err != nil { if err != nil {
return return
} }
err = en.WriteInt64(z.StatModTime) err = en.WriteInt64(z.ModTime)
if err != nil { if err != nil {
err = msgp.WrapError(err, "StatModTime") err = msgp.WrapError(err, "ModTime")
return return
} }
if (zb0001Mask & 0x8000) == 0 { // if not empty if (zb0001Mask & 0x8000) == 0 { // if not empty
@ -1032,10 +1032,10 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
} }
// string "Size" // string "Size"
o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
o = msgp.AppendInt64(o, z.StatSize) o = msgp.AppendInt64(o, z.Size)
// string "MTime" // string "MTime"
o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65)
o = msgp.AppendInt64(o, z.StatModTime) o = msgp.AppendInt64(o, z.ModTime)
if (zb0001Mask & 0x8000) == 0 { // if not empty if (zb0001Mask & 0x8000) == 0 { // if not empty
// string "MetaSys" // string "MetaSys"
o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73)
@ -1227,15 +1227,15 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
} }
} }
case "Size": case "Size":
z.StatSize, bts, err = msgp.ReadInt64Bytes(bts) z.Size, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil { if err != nil {
err = msgp.WrapError(err, "StatSize") err = msgp.WrapError(err, "Size")
return return
} }
case "MTime": case "MTime":
z.StatModTime, bts, err = msgp.ReadInt64Bytes(bts) z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
if err != nil { if err != nil {
err = msgp.WrapError(err, "StatModTime") err = msgp.WrapError(err, "ModTime")
return return
} }
case "MetaSys": case "MetaSys":

@ -106,7 +106,7 @@ require (
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94
github.com/tidwall/gjson v1.3.5 github.com/tidwall/gjson v1.3.5
github.com/tidwall/sjson v1.0.4 github.com/tidwall/sjson v1.0.4
github.com/tinylib/msgp v1.1.1 github.com/tinylib/msgp v1.1.2
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
github.com/ugorji/go v1.1.5-pre // indirect github.com/ugorji/go v1.1.5-pre // indirect
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a

@ -419,8 +419,8 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg= github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg=
github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
github.com/tinylib/msgp v1.1.1 h1:TnCZ3FIuKeaIy+F45+Cnp+caqdXGy4z74HvwXN+570Y= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=

Loading…
Cancel
Save