diff --git a/cmd/api-response.go b/cmd/api-response.go index ec3bfac45..d4e22782c 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -35,7 +35,7 @@ import ( const ( // RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision. - maxObjectList = 50000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. + maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. maxDeleteList = 10000 // Limit number of objects deleted in a delete call. maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse. maxPartsList = 10000 // Limit number of parts in a listPartsResponse. diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 91bd11438..f00ec154f 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -156,8 +156,8 @@ type xlMetaV2Object struct { PartETags []string `json:"PartETags" msg:"PartETags"` // Part ETags PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,omitempty"` // Part ActualSizes (compression) - StatSize int64 `json:"Size" msg:"Size"` // Object version size - StatModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time + Size int64 `json:"Size" msg:"Size"` // Object version size + ModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Object version internal metadata MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,omitempty"` // Object version metadata set by user } @@ -184,7 +184,7 @@ func (j xlMetaV2Version) Valid() bool { j.ObjectV2.ErasureAlgorithm.valid() && j.ObjectV2.BitrotChecksumAlgo.valid() && isXLMetaErasureInfoValid(j.ObjectV2.ErasureM, j.ObjectV2.ErasureN) && - j.ObjectV2.StatModTime > 0 + j.ObjectV2.ModTime > 0 case DeleteType: return j.DeleteMarker != nil && j.DeleteMarker.ModTime > 0 @@ -271,8 +271,8 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error { ObjectV2: &xlMetaV2Object{ VersionID: uv, DataDir: dd, - StatSize: fi.Size, - StatModTime: fi.ModTime.UnixNano(), + Size: fi.Size, + ModTime: fi.ModTime.UnixNano(), ErasureAlgorithm: ReedSolomon, ErasureM: fi.Erasure.DataBlocks, ErasureN: fi.Erasure.ParityBlocks, @@ -373,8 +373,8 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { fi := FileInfo{ Volume: volume, Name: path, - Size: j.StatSize, - ModTime: time.Unix(0, j.StatModTime).UTC(), + Size: j.Size, + ModTime: time.Unix(0, j.ModTime).UTC(), VersionID: versionID, } fi.Parts = make([]ObjectPartInfo, len(j.PartNumbers)) @@ -463,7 +463,7 @@ func (z xlMetaV2) TotalSize() int64 { for i := range z.Versions { switch z.Versions[i].Type { case ObjectType: - total += z.Versions[i].ObjectV2.StatSize + total += z.Versions[i].ObjectV2.Size case LegacyType: total += z.Versions[i].ObjectV1.Stat.Size } @@ -488,8 +488,6 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim fi, err = version.DeleteMarker.ToFileInfo(volume, path) case LegacyType: fi, err = version.ObjectV1.ToFileInfo(volume, path) - default: - continue } if err != nil { return nil, latestModTime, err @@ -498,12 +496,7 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim latestModTime = fi.ModTime latestVersionID = fi.VersionID } - switch version.Type { - case LegacyType: - fallthrough - case ObjectType, DeleteType: - versions = append(versions, fi) - } + versions = append(versions, fi) } // We didn't find the version in delete markers so latest version @@ -521,44 +514,53 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim // ToFileInfo converts xlMetaV2 into a common FileInfo datastructure // for consumption across callers. -func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) { +func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) { var uv uuid.UUID if versionID != "" { uv, _ = uuid.Parse(versionID) } + var latestModTime time.Time + var latestIndex int + for i, version := range z.Versions { + if !version.Valid() { + logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version)) + return FileInfo{}, errFileNotFound + } + var modTime time.Time + switch version.Type { + case ObjectType: + modTime = time.Unix(0, version.ObjectV2.ModTime) + case DeleteType: + modTime = time.Unix(0, version.DeleteMarker.ModTime) + case LegacyType: + modTime = version.ObjectV1.Stat.ModTime + } + if modTime.After(latestModTime) { + latestModTime = modTime + latestIndex = i + } + } + if versionID == "" { - var latestModTime time.Time - var latestIndex int - for i, version := range z.Versions { - if !version.Valid() { - logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version)) + if len(z.Versions) >= 1 { + if !z.Versions[latestIndex].Valid() { + logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", z.Versions[latestIndex])) return FileInfo{}, errFileNotFound } - var modTime time.Time - switch version.Type { - case ObjectType: - modTime = time.Unix(0, version.ObjectV2.StatModTime) - case DeleteType: - modTime = time.Unix(0, version.DeleteMarker.ModTime) - case LegacyType: - modTime = version.ObjectV1.Stat.ModTime - default: - continue - } - if modTime.After(latestModTime) { - latestModTime = modTime - latestIndex = i - } - } - if len(z.Versions) >= 1 { switch z.Versions[latestIndex].Type { case ObjectType: - return z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path) + fi, err = z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path) + fi.IsLatest = true + return fi, err case DeleteType: - return z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path) + fi, err = z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path) + fi.IsLatest = true + return fi, err case LegacyType: - return z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path) + fi, err = z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path) + fi.IsLatest = true + return fi, err } } return FileInfo{}, errFileNotFound @@ -575,23 +577,22 @@ func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) { switch version.Type { case ObjectType: if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) { - return version.ObjectV2.ToFileInfo(volume, path) + fi, err = version.ObjectV2.ToFileInfo(volume, path) + fi.IsLatest = latestModTime.Equal(fi.ModTime) + return fi, err } case LegacyType: if version.ObjectV1.VersionID == versionID { - return version.ObjectV1.ToFileInfo(volume, path) + fi, err = version.ObjectV1.ToFileInfo(volume, path) + fi.IsLatest = latestModTime.Equal(fi.ModTime) + return fi, err } case DeleteType: if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) { - return version.DeleteMarker.ToFileInfo(volume, path) - } - default: - logger.LogIf(GlobalContext, fmt.Errorf("unknown version type: %v", version.Type)) - if versionID == "" { - return FileInfo{}, errFileNotFound + fi, err = version.DeleteMarker.ToFileInfo(volume, path) + fi.IsLatest = latestModTime.Equal(fi.ModTime) + return fi, err } - - return FileInfo{}, errFileVersionNotFound } } diff --git a/cmd/xl-storage-format-v2_gen.go b/cmd/xl-storage-format-v2_gen.go index da111d395..bc604eb81 100644 --- a/cmd/xl-storage-format-v2_gen.go +++ b/cmd/xl-storage-format-v2_gen.go @@ -606,15 +606,15 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { } } case "Size": - z.StatSize, err = dc.ReadInt64() + z.Size, err = dc.ReadInt64() if err != nil { - err = msgp.WrapError(err, "StatSize") + err = msgp.WrapError(err, "Size") return } case "MTime": - z.StatModTime, err = dc.ReadInt64() + z.ModTime, err = dc.ReadInt64() if err != nil { - err = msgp.WrapError(err, "StatModTime") + err = msgp.WrapError(err, "ModTime") return } case "MetaSys": @@ -885,9 +885,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { if err != nil { return } - err = en.WriteInt64(z.StatSize) + err = en.WriteInt64(z.Size) if err != nil { - err = msgp.WrapError(err, "StatSize") + err = msgp.WrapError(err, "Size") return } // write "MTime" @@ -895,9 +895,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { if err != nil { return } - err = en.WriteInt64(z.StatModTime) + err = en.WriteInt64(z.ModTime) if err != nil { - err = msgp.WrapError(err, "StatModTime") + err = msgp.WrapError(err, "ModTime") return } if (zb0001Mask & 0x8000) == 0 { // if not empty @@ -1032,10 +1032,10 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { } // string "Size" o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) - o = msgp.AppendInt64(o, z.StatSize) + o = msgp.AppendInt64(o, z.Size) // string "MTime" o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) - o = msgp.AppendInt64(o, z.StatModTime) + o = msgp.AppendInt64(o, z.ModTime) if (zb0001Mask & 0x8000) == 0 { // if not empty // string "MetaSys" o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) @@ -1227,15 +1227,15 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) { } } case "Size": - z.StatSize, bts, err = msgp.ReadInt64Bytes(bts) + z.Size, bts, err = msgp.ReadInt64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "StatSize") + err = msgp.WrapError(err, "Size") return } case "MTime": - z.StatModTime, bts, err = msgp.ReadInt64Bytes(bts) + z.ModTime, bts, err = msgp.ReadInt64Bytes(bts) if err != nil { - err = msgp.WrapError(err, "StatModTime") + err = msgp.WrapError(err, "ModTime") return } case "MetaSys": diff --git a/go.mod b/go.mod index e9669b3d3..fc386f618 100644 --- a/go.mod +++ b/go.mod @@ -106,7 +106,7 @@ require ( github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 github.com/tidwall/gjson v1.3.5 github.com/tidwall/sjson v1.0.4 - github.com/tinylib/msgp v1.1.1 + github.com/tinylib/msgp v1.1.2 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/ugorji/go v1.1.5-pre // indirect github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a diff --git a/go.sum b/go.sum index dc15314c7..a73d9b20d 100644 --- a/go.sum +++ b/go.sum @@ -419,8 +419,8 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= -github.com/tinylib/msgp v1.1.1 h1:TnCZ3FIuKeaIy+F45+Cnp+caqdXGy4z74HvwXN+570Y= -github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= +github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=