re-implement data usage crawler to be more efficient (#9075)
Implementation overview: https://gist.github.com/klauspost/1801c858d5e0df391114436fdad6987bmaster
Klaus Post
5 years ago
committed by
GitHub
parent
7fdeb44372
commit
8d98662633
@ -0,0 +1 @@ |
||||
checks = ["all", "-ST1005", "-ST1000", "-SA4000", "-SA9004", "-SA1019", "-SA1008", "-U1000", "-ST1003", "-ST1018"] |
||||
@ -0,0 +1,521 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2020 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/binary" |
||||
"fmt" |
||||
"io" |
||||
"path" |
||||
"path/filepath" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/cespare/xxhash/v2" |
||||
"github.com/minio/minio/cmd/logger" |
||||
"github.com/minio/minio/pkg/color" |
||||
"github.com/minio/minio/pkg/hash" |
||||
"github.com/tinylib/msgp/msgp" |
||||
) |
||||
|
||||
const dataUsageHashLen = 8 |
||||
|
||||
//go:generate msgp -file $GOFILE -unexported
|
||||
|
||||
// dataUsageHash is the hash type used.
|
||||
type dataUsageHash uint64 |
||||
|
||||
// sizeHistogram is a size histogram.
|
||||
type sizeHistogram [dataUsageBucketLen]uint64 |
||||
|
||||
//msgp:tuple dataUsageEntry
|
||||
type dataUsageEntry struct { |
||||
// These fields do no include any children.
|
||||
Size int64 |
||||
Objects uint64 |
||||
ObjSizes sizeHistogram |
||||
|
||||
Children dataUsageHashMap |
||||
} |
||||
|
||||
//msgp:ignore dataUsageEntryInfo
|
||||
type dataUsageEntryInfo struct { |
||||
Name string |
||||
Parent string |
||||
Entry dataUsageEntry |
||||
} |
||||
|
||||
type dataUsageCacheInfo struct { |
||||
// Name of the bucket. Also root element.
|
||||
Name string |
||||
LastUpdate time.Time |
||||
NextCycle uint8 |
||||
} |
||||
|
||||
// merge other data usage entry into this, excluding children.
|
||||
func (e *dataUsageEntry) merge(other dataUsageEntry) { |
||||
e.Objects += other.Objects |
||||
e.Size += other.Size |
||||
for i, v := range other.ObjSizes[:] { |
||||
e.ObjSizes[i] += v |
||||
} |
||||
} |
||||
|
||||
// mod returns true if the hash mod cycles == cycle.
|
||||
func (h dataUsageHash) mod(cycle uint8, cycles uint8) bool { |
||||
return uint8(h)%cycles == cycle%cycles |
||||
} |
||||
|
||||
// addChildString will add a child based on its name.
|
||||
// If it already exists it will not be added again.
|
||||
func (e *dataUsageEntry) addChildString(name string) { |
||||
e.addChild(hashPath(name)) |
||||
} |
||||
|
||||
// addChild will add a child based on its hash.
|
||||
// If it already exists it will not be added again.
|
||||
func (e *dataUsageEntry) addChild(hash dataUsageHash) { |
||||
if _, ok := e.Children[hash]; ok { |
||||
return |
||||
} |
||||
if e.Children == nil { |
||||
e.Children = make(dataUsageHashMap, 1) |
||||
} |
||||
e.Children[hash] = struct{}{} |
||||
} |
||||
|
||||
// find a path in the cache.
|
||||
// Returns nil if not found.
|
||||
func (d *dataUsageCache) find(path string) *dataUsageEntry { |
||||
due, ok := d.Cache[hashPath(path)] |
||||
if !ok { |
||||
return nil |
||||
} |
||||
return &due |
||||
} |
||||
|
||||
// dui converts the flattened version of the path to DataUsageInfo.
|
||||
func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo { |
||||
e := d.find(path) |
||||
if e == nil { |
||||
return DataUsageInfo{LastUpdate: time.Now()} |
||||
} |
||||
flat := d.flatten(*e) |
||||
return DataUsageInfo{ |
||||
LastUpdate: d.Info.LastUpdate, |
||||
ObjectsCount: flat.Objects, |
||||
ObjectsTotalSize: uint64(flat.Size), |
||||
ObjectsSizesHistogram: flat.ObjSizes.asMap(), |
||||
BucketsCount: uint64(len(e.Children)), |
||||
BucketsSizes: d.pathSizes(buckets), |
||||
} |
||||
} |
||||
|
||||
// replace will add or replace an entry in the cache.
|
||||
// If a parent is specified it will be added to that if not already there.
|
||||
// If the parent does not exist, it will be added.
|
||||
func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) { |
||||
hash := hashPath(path) |
||||
if d.Cache == nil { |
||||
d.Cache = make(map[dataUsageHash]dataUsageEntry, 100) |
||||
} |
||||
d.Cache[hash] = e |
||||
if parent != "" { |
||||
phash := hashPath(parent) |
||||
p := d.Cache[phash] |
||||
p.addChild(hash) |
||||
d.Cache[phash] = p |
||||
} |
||||
} |
||||
|
||||
// replaceHashed add or replaces an entry to the cache based on its hash.
|
||||
// If a parent is specified it will be added to that if not already there.
|
||||
// If the parent does not exist, it will be added.
|
||||
func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash, e dataUsageEntry) { |
||||
if d.Cache == nil { |
||||
d.Cache = make(map[dataUsageHash]dataUsageEntry, 100) |
||||
} |
||||
d.Cache[hash] = e |
||||
if parent != nil { |
||||
p := d.Cache[*parent] |
||||
p.addChild(hash) |
||||
d.Cache[*parent] = p |
||||
} |
||||
} |
||||
|
||||
// StringAll returns a detailed string representation of all entries in the cache.
|
||||
func (d *dataUsageCache) StringAll() string { |
||||
s := fmt.Sprintf("info:%+v\n", d.Info) |
||||
for k, v := range d.Cache { |
||||
s += fmt.Sprintf("\t%v: %+v\n", k, v) |
||||
} |
||||
return strings.TrimSpace(s) |
||||
} |
||||
|
||||
// String returns a human readable representation of the string.
|
||||
func (h dataUsageHash) String() string { |
||||
return fmt.Sprintf("%x", uint64(h)) |
||||
} |
||||
|
||||
// flatten all children of the root into the root element and return it.
|
||||
func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry { |
||||
for id := range root.Children { |
||||
e := d.Cache[id] |
||||
if len(e.Children) > 0 { |
||||
e = d.flatten(e) |
||||
} |
||||
root.merge(e) |
||||
} |
||||
root.Children = nil |
||||
return root |
||||
} |
||||
|
||||
// add a size to the histogram.
|
||||
func (h *sizeHistogram) add(size int64) { |
||||
// Fetch the histogram interval corresponding
|
||||
// to the passed object size.
|
||||
for i, interval := range ObjectsHistogramIntervals { |
||||
if size >= interval.start && size <= interval.end { |
||||
h[i]++ |
||||
break |
||||
} |
||||
} |
||||
} |
||||
|
||||
// asMap returns the map as a map[string]uint64.
|
||||
func (h *sizeHistogram) asMap() map[string]uint64 { |
||||
res := make(map[string]uint64, 7) |
||||
for i, count := range h { |
||||
res[ObjectsHistogramIntervals[i].name] = count |
||||
} |
||||
return res |
||||
} |
||||
|
||||
// pathSizes returns the path sizes as a map.
|
||||
func (d *dataUsageCache) pathSizes(buckets []BucketInfo) map[string]uint64 { |
||||
var dst = make(map[string]uint64, len(buckets)) |
||||
for _, bucket := range buckets { |
||||
e := d.find(bucket.Name) |
||||
if e == nil { |
||||
if dataUsageDebug { |
||||
logger.Info(color.Green("data-usage:")+" Bucket not found in cache: %v", bucket.Name) |
||||
} |
||||
continue |
||||
} |
||||
flat := d.flatten(*e) |
||||
dst[bucket.Name] = uint64(flat.Size) |
||||
} |
||||
return dst |
||||
} |
||||
|
||||
// sizeRecursive returns the path as a flattened entry.
|
||||
func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry { |
||||
root := d.find(path) |
||||
if root == nil || len(root.Children) == 0 { |
||||
return root |
||||
} |
||||
flat := d.flatten(*root) |
||||
return &flat |
||||
} |
||||
|
||||
// dataUsageCache contains a cache of data usage entries.
|
||||
//msgp:ignore dataUsageCache
|
||||
type dataUsageCache struct { |
||||
Info dataUsageCacheInfo |
||||
Cache map[dataUsageHash]dataUsageEntry |
||||
} |
||||
|
||||
// root returns the root of the cache.
|
||||
func (d *dataUsageCache) root() *dataUsageEntry { |
||||
return d.find(d.Info.Name) |
||||
} |
||||
|
||||
// rootHash returns the root of the cache.
|
||||
func (d *dataUsageCache) rootHash() dataUsageHash { |
||||
return hashPath(d.Info.Name) |
||||
} |
||||
|
||||
// clone returns a copy of the cache with no references to the existing.
|
||||
func (d *dataUsageCache) clone() dataUsageCache { |
||||
clone := dataUsageCache{ |
||||
Info: d.Info, |
||||
Cache: make(map[dataUsageHash]dataUsageEntry, len(d.Cache)), |
||||
} |
||||
for k, v := range d.Cache { |
||||
clone.Cache[k] = v |
||||
} |
||||
return clone |
||||
} |
||||
|
||||
// merge root of other into d.
|
||||
// children of root will be flattened before being merged.
|
||||
// Last update time will be set to the last updated.
|
||||
func (d *dataUsageCache) merge(other dataUsageCache) { |
||||
existingRoot := d.root() |
||||
otherRoot := other.root() |
||||
if existingRoot == nil && otherRoot == nil { |
||||
return |
||||
} |
||||
if otherRoot == nil { |
||||
return |
||||
} |
||||
if existingRoot == nil { |
||||
*d = other.clone() |
||||
return |
||||
} |
||||
if other.Info.LastUpdate.After(d.Info.LastUpdate) { |
||||
d.Info.LastUpdate = other.Info.LastUpdate |
||||
} |
||||
existingRoot.merge(*otherRoot) |
||||
eHash := d.rootHash() |
||||
for key := range otherRoot.Children { |
||||
entry := other.Cache[key] |
||||
flat := other.flatten(entry) |
||||
existing := d.Cache[key] |
||||
// If not found, merging simply adds.
|
||||
existing.merge(flat) |
||||
d.replaceHashed(key, &eHash, existing) |
||||
} |
||||
} |
||||
|
||||
// load the cache content with name from minioMetaBackgroundOpsBucket.
|
||||
// Only backend errors are returned as errors.
|
||||
// If the object is not found or unable to deserialize d is cleared and nil error is returned.
|
||||
func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name string) error { |
||||
var buf bytes.Buffer |
||||
err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{}) |
||||
if err != nil { |
||||
if !isErrObjectNotFound(err) { |
||||
return toObjectErr(err, dataUsageBucket, name) |
||||
} |
||||
*d = dataUsageCache{} |
||||
return nil |
||||
} |
||||
err = d.deserialize(buf.Bytes()) |
||||
if err != nil { |
||||
*d = dataUsageCache{} |
||||
logger.LogIf(ctx, err) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
|
||||
func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name string) error { |
||||
b := d.serialize() |
||||
size := int64(len(b)) |
||||
r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
_, err = store.PutObject(ctx, |
||||
dataUsageBucket, |
||||
name, |
||||
NewPutObjReader(r, nil, nil), |
||||
ObjectOptions{}) |
||||
return err |
||||
} |
||||
|
||||
// dataUsageCacheVer indicates the cache version.
|
||||
// Bumping the cache version will drop data from previous versions
|
||||
// and write new data with the new version.
|
||||
const dataUsageCacheVer = 1 |
||||
|
||||
// serialize the contents of the cache.
|
||||
func (d *dataUsageCache) serialize() []byte { |
||||
// Alloc pessimistically
|
||||
// dataUsageCacheVer
|
||||
due := dataUsageEntry{} |
||||
msgLen := 1 |
||||
msgLen += d.Info.Msgsize() |
||||
// len(d.Cache)
|
||||
msgLen += binary.MaxVarintLen64 |
||||
// Hashes (one for key, assume 1 child/node)
|
||||
msgLen += len(d.Cache) * dataUsageHashLen * 2 |
||||
msgLen += len(d.Cache) * due.Msgsize() |
||||
|
||||
// Create destination buffer...
|
||||
dst := make([]byte, 0, msgLen) |
||||
|
||||
var n int |
||||
tmp := make([]byte, 1024) |
||||
// byte: version.
|
||||
dst = append(dst, dataUsageCacheVer) |
||||
// Info...
|
||||
dst, err := d.Info.MarshalMsg(dst) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
n = binary.PutUvarint(tmp, uint64(len(d.Cache))) |
||||
dst = append(dst, tmp[:n]...) |
||||
|
||||
for k, v := range d.Cache { |
||||
// Put key
|
||||
binary.LittleEndian.PutUint64(tmp[:dataUsageHashLen], uint64(k)) |
||||
dst = append(dst, tmp[:8]...) |
||||
tmp, err = v.MarshalMsg(tmp[:0]) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
// key, value pairs.
|
||||
dst = append(dst, tmp...) |
||||
|
||||
} |
||||
return dst |
||||
} |
||||
|
||||
// deserialize the supplied byte slice into the cache.
|
||||
func (d *dataUsageCache) deserialize(b []byte) error { |
||||
if len(b) < 1 { |
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
switch b[0] { |
||||
case 1: |
||||
default: |
||||
return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0])) |
||||
} |
||||
b = b[1:] |
||||
|
||||
// Info...
|
||||
b, err := d.Info.UnmarshalMsg(b) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
cacheLen, n := binary.Uvarint(b) |
||||
if n <= 0 { |
||||
return fmt.Errorf("dataUsageCache: reading cachelen, n <= 0 ") |
||||
} |
||||
b = b[n:] |
||||
d.Cache = make(map[dataUsageHash]dataUsageEntry, cacheLen) |
||||
|
||||
for i := 0; i < int(cacheLen); i++ { |
||||
if len(b) <= dataUsageHashLen { |
||||
return io.ErrUnexpectedEOF |
||||
} |
||||
k := binary.LittleEndian.Uint64(b[:dataUsageHashLen]) |
||||
b = b[dataUsageHashLen:] |
||||
var v dataUsageEntry |
||||
b, err = v.UnmarshalMsg(b) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
d.Cache[dataUsageHash(k)] = v |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// Trim this from start+end of hashes.
|
||||
var hashPathCutSet = dataUsageRoot |
||||
|
||||
func init() { |
||||
if dataUsageRoot != string(filepath.Separator) { |
||||
hashPathCutSet = dataUsageRoot + string(filepath.Separator) |
||||
} |
||||
} |
||||
|
||||
// hashPath calculates a hash of the provided string.
|
||||
func hashPath(data string) dataUsageHash { |
||||
if data != dataUsageRoot { |
||||
data = strings.Trim(data, hashPathCutSet) |
||||
} |
||||
data = path.Clean(data) |
||||
return dataUsageHash(xxhash.Sum64String(data)) |
||||
} |
||||
|
||||
//msgp:ignore dataUsageEntryInfo
|
||||
type dataUsageHashMap map[dataUsageHash]struct{} |
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (d dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) { |
||||
o = msgp.Require(b, d.Msgsize()) |
||||
|
||||
// Write bin header manually
|
||||
const mbin32 uint8 = 0xc6 |
||||
sz := uint32(len(d)) * dataUsageHashLen |
||||
o = append(o, mbin32, byte(sz>>24), byte(sz>>16), byte(sz>>8), byte(sz)) |
||||
|
||||
var tmp [dataUsageHashLen]byte |
||||
for k := range d { |
||||
binary.LittleEndian.PutUint64(tmp[:], uint64(k)) |
||||
o = append(o, tmp[:]...) |
||||
} |
||||
return |
||||
} |
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (d dataUsageHashMap) Msgsize() (s int) { |
||||
s = 5 + len(d)*dataUsageHashLen |
||||
return |
||||
} |
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (d *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) { |
||||
var hashes []byte |
||||
hashes, bts, err = msgp.ReadBytesZC(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "dataUsageHashMap") |
||||
return |
||||
} |
||||
|
||||
var dst = make(dataUsageHashMap, len(hashes)/dataUsageHashLen) |
||||
for len(hashes) >= dataUsageHashLen { |
||||
dst[dataUsageHash(binary.LittleEndian.Uint64(hashes[:dataUsageHashLen]))] = struct{}{} |
||||
hashes = hashes[dataUsageHashLen:] |
||||
} |
||||
*d = dst |
||||
o = bts |
||||
return |
||||
} |
||||
|
||||
func (d *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) { |
||||
var zb0001 uint32 |
||||
zb0001, err = dc.ReadBytesHeader() |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
var dst = make(dataUsageHashMap, zb0001) |
||||
var tmp [8]byte |
||||
for i := uint32(0); i < zb0001; i++ { |
||||
_, err = io.ReadFull(dc, tmp[:]) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "dataUsageHashMap") |
||||
return |
||||
} |
||||
dst[dataUsageHash(binary.LittleEndian.Uint64(tmp[:]))] = struct{}{} |
||||
} |
||||
return nil |
||||
} |
||||
func (d dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) { |
||||
err = en.WriteBytesHeader(uint32(len(d)) * dataUsageHashLen) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
var tmp [dataUsageHashLen]byte |
||||
for k := range d { |
||||
binary.LittleEndian.PutUint64(tmp[:], uint64(k)) |
||||
_, err = en.Write(tmp[:]) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,447 @@ |
||||
package cmd |
||||
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
import ( |
||||
"github.com/tinylib/msgp/msgp" |
||||
) |
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) { |
||||
var field []byte |
||||
_ = field |
||||
var zb0001 uint32 |
||||
zb0001, err = dc.ReadMapHeader() |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
for zb0001 > 0 { |
||||
zb0001-- |
||||
field, err = dc.ReadMapKeyPtr() |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
switch msgp.UnsafeString(field) { |
||||
case "Name": |
||||
z.Name, err = dc.ReadString() |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Name") |
||||
return |
||||
} |
||||
case "LastUpdate": |
||||
z.LastUpdate, err = dc.ReadTime() |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "LastUpdate") |
||||
return |
||||
} |
||||
case "NextCycle": |
||||
z.NextCycle, err = dc.ReadUint8() |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "NextCycle") |
||||
return |
||||
} |
||||
default: |
||||
err = dc.Skip() |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) { |
||||
// map header, size 3
|
||||
// write "Name"
|
||||
err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65) |
||||
if err != nil { |
||||
return |
||||
} |
||||
err = en.WriteString(z.Name) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Name") |
||||
return |
||||
} |
||||
// write "LastUpdate"
|
||||
err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65) |
||||
if err != nil { |
||||
return |
||||
} |
||||
err = en.WriteTime(z.LastUpdate) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "LastUpdate") |
||||
return |
||||
} |
||||
// write "NextCycle"
|
||||
err = en.Append(0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65) |
||||
if err != nil { |
||||
return |
||||
} |
||||
err = en.WriteUint8(z.NextCycle) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "NextCycle") |
||||
return |
||||
} |
||||
return |
||||
} |
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) { |
||||
o = msgp.Require(b, z.Msgsize()) |
||||
// map header, size 3
|
||||
// string "Name"
|
||||
o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65) |
||||
o = msgp.AppendString(o, z.Name) |
||||
// string "LastUpdate"
|
||||
o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65) |
||||
o = msgp.AppendTime(o, z.LastUpdate) |
||||
// string "NextCycle"
|
||||
o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65) |
||||
o = msgp.AppendUint8(o, z.NextCycle) |
||||
return |
||||
} |
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { |
||||
var field []byte |
||||
_ = field |
||||
var zb0001 uint32 |
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
for zb0001 > 0 { |
||||
zb0001-- |
||||
field, bts, err = msgp.ReadMapKeyZC(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
switch msgp.UnsafeString(field) { |
||||
case "Name": |
||||
z.Name, bts, err = msgp.ReadStringBytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Name") |
||||
return |
||||
} |
||||
case "LastUpdate": |
||||
z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "LastUpdate") |
||||
return |
||||
} |
||||
case "NextCycle": |
||||
z.NextCycle, bts, err = msgp.ReadUint8Bytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "NextCycle") |
||||
return |
||||
} |
||||
default: |
||||
bts, err = msgp.Skip(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
} |
||||
} |
||||
o = bts |
||||
return |
||||
} |
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z dataUsageCacheInfo) Msgsize() (s int) { |
||||
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint8Size |
||||
return |
||||
} |
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { |
||||
var zb0001 uint32 |
||||
zb0001, err = dc.ReadArrayHeader() |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
if zb0001 != 4 { |
||||
err = msgp.ArrayError{Wanted: 4, Got: zb0001} |
||||
return |
||||
} |
||||
z.Size, err = dc.ReadInt64() |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Size") |
||||
return |
||||
} |
||||
z.Objects, err = dc.ReadUint64() |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Objects") |
||||
return |
||||
} |
||||
var zb0002 uint32 |
||||
zb0002, err = dc.ReadArrayHeader() |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "ObjSizes") |
||||
return |
||||
} |
||||
if zb0002 != uint32(dataUsageBucketLen) { |
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} |
||||
return |
||||
} |
||||
for za0001 := range z.ObjSizes { |
||||
z.ObjSizes[za0001], err = dc.ReadUint64() |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "ObjSizes", za0001) |
||||
return |
||||
} |
||||
} |
||||
err = z.Children.DecodeMsg(dc) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Children") |
||||
return |
||||
} |
||||
return |
||||
} |
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { |
||||
// array header, size 4
|
||||
err = en.Append(0x94) |
||||
if err != nil { |
||||
return |
||||
} |
||||
err = en.WriteInt64(z.Size) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Size") |
||||
return |
||||
} |
||||
err = en.WriteUint64(z.Objects) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Objects") |
||||
return |
||||
} |
||||
err = en.WriteArrayHeader(uint32(dataUsageBucketLen)) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "ObjSizes") |
||||
return |
||||
} |
||||
for za0001 := range z.ObjSizes { |
||||
err = en.WriteUint64(z.ObjSizes[za0001]) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "ObjSizes", za0001) |
||||
return |
||||
} |
||||
} |
||||
err = z.Children.EncodeMsg(en) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Children") |
||||
return |
||||
} |
||||
return |
||||
} |
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { |
||||
o = msgp.Require(b, z.Msgsize()) |
||||
// array header, size 4
|
||||
o = append(o, 0x94) |
||||
o = msgp.AppendInt64(o, z.Size) |
||||
o = msgp.AppendUint64(o, z.Objects) |
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen)) |
||||
for za0001 := range z.ObjSizes { |
||||
o = msgp.AppendUint64(o, z.ObjSizes[za0001]) |
||||
} |
||||
o, err = z.Children.MarshalMsg(o) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Children") |
||||
return |
||||
} |
||||
return |
||||
} |
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { |
||||
var zb0001 uint32 |
||||
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
if zb0001 != 4 { |
||||
err = msgp.ArrayError{Wanted: 4, Got: zb0001} |
||||
return |
||||
} |
||||
z.Size, bts, err = msgp.ReadInt64Bytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Size") |
||||
return |
||||
} |
||||
z.Objects, bts, err = msgp.ReadUint64Bytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Objects") |
||||
return |
||||
} |
||||
var zb0002 uint32 |
||||
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "ObjSizes") |
||||
return |
||||
} |
||||
if zb0002 != uint32(dataUsageBucketLen) { |
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} |
||||
return |
||||
} |
||||
for za0001 := range z.ObjSizes { |
||||
z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "ObjSizes", za0001) |
||||
return |
||||
} |
||||
} |
||||
bts, err = z.Children.UnmarshalMsg(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, "Children") |
||||
return |
||||
} |
||||
o = bts |
||||
return |
||||
} |
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *dataUsageEntry) Msgsize() (s int) { |
||||
s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize() |
||||
return |
||||
} |
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) { |
||||
{ |
||||
var zb0001 uint64 |
||||
zb0001, err = dc.ReadUint64() |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
(*z) = dataUsageHash(zb0001) |
||||
} |
||||
return |
||||
} |
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) { |
||||
err = en.WriteUint64(uint64(z)) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
return |
||||
} |
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z dataUsageHash) MarshalMsg(b []byte) (o []byte, err error) { |
||||
o = msgp.Require(b, z.Msgsize()) |
||||
o = msgp.AppendUint64(o, uint64(z)) |
||||
return |
||||
} |
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) { |
||||
{ |
||||
var zb0001 uint64 |
||||
zb0001, bts, err = msgp.ReadUint64Bytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
(*z) = dataUsageHash(zb0001) |
||||
} |
||||
o = bts |
||||
return |
||||
} |
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z dataUsageHash) Msgsize() (s int) { |
||||
s = msgp.Uint64Size |
||||
return |
||||
} |
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *sizeHistogram) DecodeMsg(dc *msgp.Reader) (err error) { |
||||
var zb0001 uint32 |
||||
zb0001, err = dc.ReadArrayHeader() |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
if zb0001 != uint32(dataUsageBucketLen) { |
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001} |
||||
return |
||||
} |
||||
for za0001 := range z { |
||||
z[za0001], err = dc.ReadUint64() |
||||
if err != nil { |
||||
err = msgp.WrapError(err, za0001) |
||||
return |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *sizeHistogram) EncodeMsg(en *msgp.Writer) (err error) { |
||||
err = en.WriteArrayHeader(uint32(dataUsageBucketLen)) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
for za0001 := range z { |
||||
err = en.WriteUint64(z[za0001]) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, za0001) |
||||
return |
||||
} |
||||
} |
||||
return |
||||
} |
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *sizeHistogram) MarshalMsg(b []byte) (o []byte, err error) { |
||||
o = msgp.Require(b, z.Msgsize()) |
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen)) |
||||
for za0001 := range z { |
||||
o = msgp.AppendUint64(o, z[za0001]) |
||||
} |
||||
return |
||||
} |
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *sizeHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) { |
||||
var zb0001 uint32 |
||||
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err) |
||||
return |
||||
} |
||||
if zb0001 != uint32(dataUsageBucketLen) { |
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001} |
||||
return |
||||
} |
||||
for za0001 := range z { |
||||
z[za0001], bts, err = msgp.ReadUint64Bytes(bts) |
||||
if err != nil { |
||||
err = msgp.WrapError(err, za0001) |
||||
return |
||||
} |
||||
} |
||||
o = bts |
||||
return |
||||
} |
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *sizeHistogram) Msgsize() (s int) { |
||||
s = msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) |
||||
return |
||||
} |
||||
@ -0,0 +1,349 @@ |
||||
package cmd |
||||
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
import ( |
||||
"bytes" |
||||
"testing" |
||||
|
||||
"github.com/tinylib/msgp/msgp" |
||||
) |
||||
|
||||
func TestMarshalUnmarshaldataUsageCacheInfo(t *testing.T) { |
||||
v := dataUsageCacheInfo{} |
||||
bts, err := v.MarshalMsg(nil) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
left, err := v.UnmarshalMsg(bts) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if len(left) > 0 { |
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) |
||||
} |
||||
|
||||
left, err = msgp.Skip(bts) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if len(left) > 0 { |
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkMarshalMsgdataUsageCacheInfo(b *testing.B) { |
||||
v := dataUsageCacheInfo{} |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
v.MarshalMsg(nil) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkAppendMsgdataUsageCacheInfo(b *testing.B) { |
||||
v := dataUsageCacheInfo{} |
||||
bts := make([]byte, 0, v.Msgsize()) |
||||
bts, _ = v.MarshalMsg(bts[0:0]) |
||||
b.SetBytes(int64(len(bts))) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
bts, _ = v.MarshalMsg(bts[0:0]) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkUnmarshaldataUsageCacheInfo(b *testing.B) { |
||||
v := dataUsageCacheInfo{} |
||||
bts, _ := v.MarshalMsg(nil) |
||||
b.ReportAllocs() |
||||
b.SetBytes(int64(len(bts))) |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
_, err := v.UnmarshalMsg(bts) |
||||
if err != nil { |
||||
b.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestEncodeDecodedataUsageCacheInfo(t *testing.T) { |
||||
v := dataUsageCacheInfo{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
|
||||
m := v.Msgsize() |
||||
if buf.Len() > m { |
||||
t.Log("WARNING: TestEncodeDecodedataUsageCacheInfo Msgsize() is inaccurate") |
||||
} |
||||
|
||||
vn := dataUsageCacheInfo{} |
||||
err := msgp.Decode(&buf, &vn) |
||||
if err != nil { |
||||
t.Error(err) |
||||
} |
||||
|
||||
buf.Reset() |
||||
msgp.Encode(&buf, &v) |
||||
err = msgp.NewReader(&buf).Skip() |
||||
if err != nil { |
||||
t.Error(err) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkEncodedataUsageCacheInfo(b *testing.B) { |
||||
v := dataUsageCacheInfo{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
b.SetBytes(int64(buf.Len())) |
||||
en := msgp.NewWriter(msgp.Nowhere) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
v.EncodeMsg(en) |
||||
} |
||||
en.Flush() |
||||
} |
||||
|
||||
func BenchmarkDecodedataUsageCacheInfo(b *testing.B) { |
||||
v := dataUsageCacheInfo{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
b.SetBytes(int64(buf.Len())) |
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b) |
||||
dc := msgp.NewReader(rd) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
err := v.DecodeMsg(dc) |
||||
if err != nil { |
||||
b.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestMarshalUnmarshaldataUsageEntry(t *testing.T) { |
||||
v := dataUsageEntry{} |
||||
bts, err := v.MarshalMsg(nil) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
left, err := v.UnmarshalMsg(bts) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if len(left) > 0 { |
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) |
||||
} |
||||
|
||||
left, err = msgp.Skip(bts) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if len(left) > 0 { |
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkMarshalMsgdataUsageEntry(b *testing.B) { |
||||
v := dataUsageEntry{} |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
v.MarshalMsg(nil) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkAppendMsgdataUsageEntry(b *testing.B) { |
||||
v := dataUsageEntry{} |
||||
bts := make([]byte, 0, v.Msgsize()) |
||||
bts, _ = v.MarshalMsg(bts[0:0]) |
||||
b.SetBytes(int64(len(bts))) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
bts, _ = v.MarshalMsg(bts[0:0]) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkUnmarshaldataUsageEntry(b *testing.B) { |
||||
v := dataUsageEntry{} |
||||
bts, _ := v.MarshalMsg(nil) |
||||
b.ReportAllocs() |
||||
b.SetBytes(int64(len(bts))) |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
_, err := v.UnmarshalMsg(bts) |
||||
if err != nil { |
||||
b.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestEncodeDecodedataUsageEntry(t *testing.T) { |
||||
v := dataUsageEntry{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
|
||||
m := v.Msgsize() |
||||
if buf.Len() > m { |
||||
t.Log("WARNING: TestEncodeDecodedataUsageEntry Msgsize() is inaccurate") |
||||
} |
||||
|
||||
vn := dataUsageEntry{} |
||||
err := msgp.Decode(&buf, &vn) |
||||
if err != nil { |
||||
t.Error(err) |
||||
} |
||||
|
||||
buf.Reset() |
||||
msgp.Encode(&buf, &v) |
||||
err = msgp.NewReader(&buf).Skip() |
||||
if err != nil { |
||||
t.Error(err) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkEncodedataUsageEntry(b *testing.B) { |
||||
v := dataUsageEntry{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
b.SetBytes(int64(buf.Len())) |
||||
en := msgp.NewWriter(msgp.Nowhere) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
v.EncodeMsg(en) |
||||
} |
||||
en.Flush() |
||||
} |
||||
|
||||
func BenchmarkDecodedataUsageEntry(b *testing.B) { |
||||
v := dataUsageEntry{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
b.SetBytes(int64(buf.Len())) |
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b) |
||||
dc := msgp.NewReader(rd) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
err := v.DecodeMsg(dc) |
||||
if err != nil { |
||||
b.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestMarshalUnmarshalsizeHistogram(t *testing.T) { |
||||
v := sizeHistogram{} |
||||
bts, err := v.MarshalMsg(nil) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
left, err := v.UnmarshalMsg(bts) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if len(left) > 0 { |
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) |
||||
} |
||||
|
||||
left, err = msgp.Skip(bts) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if len(left) > 0 { |
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkMarshalMsgsizeHistogram(b *testing.B) { |
||||
v := sizeHistogram{} |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
v.MarshalMsg(nil) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkAppendMsgsizeHistogram(b *testing.B) { |
||||
v := sizeHistogram{} |
||||
bts := make([]byte, 0, v.Msgsize()) |
||||
bts, _ = v.MarshalMsg(bts[0:0]) |
||||
b.SetBytes(int64(len(bts))) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
bts, _ = v.MarshalMsg(bts[0:0]) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkUnmarshalsizeHistogram(b *testing.B) { |
||||
v := sizeHistogram{} |
||||
bts, _ := v.MarshalMsg(nil) |
||||
b.ReportAllocs() |
||||
b.SetBytes(int64(len(bts))) |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
_, err := v.UnmarshalMsg(bts) |
||||
if err != nil { |
||||
b.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestEncodeDecodesizeHistogram(t *testing.T) { |
||||
v := sizeHistogram{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
|
||||
m := v.Msgsize() |
||||
if buf.Len() > m { |
||||
t.Log("WARNING: TestEncodeDecodesizeHistogram Msgsize() is inaccurate") |
||||
} |
||||
|
||||
vn := sizeHistogram{} |
||||
err := msgp.Decode(&buf, &vn) |
||||
if err != nil { |
||||
t.Error(err) |
||||
} |
||||
|
||||
buf.Reset() |
||||
msgp.Encode(&buf, &v) |
||||
err = msgp.NewReader(&buf).Skip() |
||||
if err != nil { |
||||
t.Error(err) |
||||
} |
||||
} |
||||
|
||||
func BenchmarkEncodesizeHistogram(b *testing.B) { |
||||
v := sizeHistogram{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
b.SetBytes(int64(buf.Len())) |
||||
en := msgp.NewWriter(msgp.Nowhere) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
v.EncodeMsg(en) |
||||
} |
||||
en.Flush() |
||||
} |
||||
|
||||
func BenchmarkDecodesizeHistogram(b *testing.B) { |
||||
v := sizeHistogram{} |
||||
var buf bytes.Buffer |
||||
msgp.Encode(&buf, &v) |
||||
b.SetBytes(int64(buf.Len())) |
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b) |
||||
dc := msgp.NewReader(rd) |
||||
b.ReportAllocs() |
||||
b.ResetTimer() |
||||
for i := 0; i < b.N; i++ { |
||||
err := v.DecodeMsg(dc) |
||||
if err != nil { |
||||
b.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,664 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2020 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"os" |
||||
"path/filepath" |
||||
"testing" |
||||
) |
||||
|
||||
type usageTestFile struct { |
||||
name string |
||||
size int |
||||
} |
||||
|
||||
func Test_updateUsage(t *testing.T) { |
||||
base, err := ioutil.TempDir("", "Test_updateUsage") |
||||
if err != nil { |
||||
t.Skip(err) |
||||
} |
||||
defer os.RemoveAll(base) |
||||
var files = []usageTestFile{ |
||||
{name: "rootfile", size: 10000}, |
||||
{name: "rootfile2", size: 10000}, |
||||
{name: "dir1/d1file", size: 2000}, |
||||
{name: "dir2/d2file", size: 300}, |
||||
{name: "dir1/dira/dafile", size: 100000}, |
||||
{name: "dir1/dira/dbfile", size: 200000}, |
||||
{name: "dir1/dira/dirasub/dcfile", size: 1000000}, |
||||
{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10}, |
||||
} |
||||
createUsageTestFiles(t, base, files) |
||||
|
||||
getSize := func(item Item) (i int64, err error) { |
||||
if item.Typ&os.ModeDir == 0 { |
||||
s, err := os.Stat(item.Path) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
return s.Size(), nil |
||||
} |
||||
return 0, nil |
||||
} |
||||
got, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// Test dirs
|
||||
var want = []struct { |
||||
path string |
||||
isNil bool |
||||
size, objs int |
||||
flatten bool |
||||
oSizes sizeHistogram |
||||
}{ |
||||
{ |
||||
path: "/", |
||||
size: 1322310, |
||||
flatten: true, |
||||
objs: 8, |
||||
oSizes: sizeHistogram{0: 2, 1: 6}, |
||||
}, |
||||
{ |
||||
path: "/", |
||||
size: 20000, |
||||
objs: 2, |
||||
oSizes: sizeHistogram{1: 2}, |
||||
}, |
||||
{ |
||||
path: "/dir1", |
||||
size: 2000, |
||||
objs: 1, |
||||
oSizes: sizeHistogram{1: 1}, |
||||
}, |
||||
{ |
||||
path: "/dir1/dira", |
||||
flatten: true, |
||||
size: 1300010, |
||||
objs: 4, |
||||
oSizes: sizeHistogram{0: 1, 1: 3}, |
||||
}, |
||||
{ |
||||
path: "/dir1/dira/", |
||||
flatten: true, |
||||
size: 1300010, |
||||
objs: 4, |
||||
oSizes: sizeHistogram{0: 1, 1: 3}, |
||||
}, |
||||
{ |
||||
path: "/dir1/dira", |
||||
size: 300000, |
||||
objs: 2, |
||||
oSizes: sizeHistogram{0: 0, 1: 2}, |
||||
}, |
||||
{ |
||||
path: "/dir1/dira/", |
||||
size: 300000, |
||||
objs: 2, |
||||
oSizes: sizeHistogram{0: 0, 1: 2}, |
||||
}, |
||||
{ |
||||
path: "/nonexistying", |
||||
isNil: true, |
||||
}, |
||||
} |
||||
|
||||
for _, w := range want { |
||||
t.Run(w.path, func(t *testing.T) { |
||||
e := got.find(w.path) |
||||
if w.isNil { |
||||
if e != nil { |
||||
t.Error("want nil, got", e) |
||||
} |
||||
return |
||||
} |
||||
if e == nil { |
||||
t.Fatal("got nil result") |
||||
} |
||||
if w.flatten { |
||||
*e = got.flatten(*e) |
||||
} |
||||
if e.Size != int64(w.size) { |
||||
t.Error("got size", e.Size, "want", w.size) |
||||
} |
||||
if e.Objects != uint64(w.objs) { |
||||
t.Error("got objects", e.Objects, "want", w.objs) |
||||
} |
||||
if e.ObjSizes != w.oSizes { |
||||
t.Error("got histogram", e.ObjSizes, "want", w.oSizes) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
files = []usageTestFile{ |
||||
{ |
||||
name: "newfolder/afile", |
||||
size: 4, |
||||
}, |
||||
{ |
||||
name: "newfolder/anotherone", |
||||
size: 1, |
||||
}, |
||||
{ |
||||
name: "newfolder/anemptyone", |
||||
size: 0, |
||||
}, |
||||
{ |
||||
name: "dir1/fileindir1", |
||||
size: 20000, |
||||
}, |
||||
{ |
||||
name: "dir1/dirc/fileindirc", |
||||
size: 20000, |
||||
}, |
||||
{ |
||||
name: "rootfile3", |
||||
size: 1000, |
||||
}, |
||||
} |
||||
createUsageTestFiles(t, base, files) |
||||
got, err = updateUsage(context.Background(), base, got, func() {}, getSize) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
want = []struct { |
||||
path string |
||||
isNil bool |
||||
size, objs int |
||||
flatten bool |
||||
oSizes sizeHistogram |
||||
}{ |
||||
{ |
||||
path: "/", |
||||
size: 1363315, |
||||
flatten: true, |
||||
objs: 14, |
||||
oSizes: sizeHistogram{0: 6, 1: 8}, |
||||
}, |
||||
{ |
||||
path: "/", |
||||
size: 21000, |
||||
objs: 3, |
||||
oSizes: sizeHistogram{0: 1, 1: 2}, |
||||
}, |
||||
{ |
||||
path: "/newfolder", |
||||
size: 5, |
||||
objs: 3, |
||||
oSizes: sizeHistogram{0: 3}, |
||||
}, |
||||
{ |
||||
path: "/dir1/dira", |
||||
size: 1300010, |
||||
flatten: true, |
||||
objs: 4, |
||||
oSizes: sizeHistogram{0: 1, 1: 3}, |
||||
}, |
||||
{ |
||||
path: "/nonexistying", |
||||
isNil: true, |
||||
}, |
||||
} |
||||
|
||||
for _, w := range want { |
||||
t.Run(w.path, func(t *testing.T) { |
||||
e := got.find(w.path) |
||||
if w.isNil { |
||||
if e != nil { |
||||
t.Error("want nil, got", e) |
||||
} |
||||
return |
||||
} |
||||
if e == nil { |
||||
t.Fatal("got nil result") |
||||
} |
||||
if w.flatten { |
||||
*e = got.flatten(*e) |
||||
} |
||||
if e.Size != int64(w.size) { |
||||
t.Error("got size", e.Size, "want", w.size) |
||||
} |
||||
if e.Objects != uint64(w.objs) { |
||||
t.Error("got objects", e.Objects, "want", w.objs) |
||||
} |
||||
if e.ObjSizes != w.oSizes { |
||||
t.Error("got histogram", e.ObjSizes, "want", w.oSizes) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
files = []usageTestFile{ |
||||
{ |
||||
name: "dir1/dira/dirasub/fileindira2", |
||||
size: 200, |
||||
}, |
||||
} |
||||
|
||||
createUsageTestFiles(t, base, files) |
||||
err = os.RemoveAll(filepath.Join(base, "dir1/dira/dirasub/dcfile")) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
// Changed dir must be picked up in this many cycles.
|
||||
for i := 0; i < dataUsageUpdateDirCycles; i++ { |
||||
got, err = updateUsage(context.Background(), base, got, func() {}, getSize) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
|
||||
want = []struct { |
||||
path string |
||||
isNil bool |
||||
size, objs int |
||||
flatten bool |
||||
oSizes sizeHistogram |
||||
}{ |
||||
{ |
||||
path: "/", |
||||
size: 363515, |
||||
flatten: true, |
||||
objs: 14, |
||||
oSizes: sizeHistogram{0: 7, 1: 7}, |
||||
}, |
||||
{ |
||||
path: "/dir1/dira", |
||||
size: 300210, |
||||
objs: 4, |
||||
flatten: true, |
||||
oSizes: sizeHistogram{0: 2, 1: 2}, |
||||
}, |
||||
} |
||||
|
||||
for _, w := range want { |
||||
t.Run(w.path, func(t *testing.T) { |
||||
e := got.find(w.path) |
||||
if w.isNil { |
||||
if e != nil { |
||||
t.Error("want nil, got", e) |
||||
} |
||||
return |
||||
} |
||||
if e == nil { |
||||
t.Fatal("got nil result") |
||||
} |
||||
if w.flatten { |
||||
*e = got.flatten(*e) |
||||
} |
||||
if e.Size != int64(w.size) { |
||||
t.Error("got size", e.Size, "want", w.size) |
||||
} |
||||
if e.Objects != uint64(w.objs) { |
||||
t.Error("got objects", e.Objects, "want", w.objs) |
||||
} |
||||
if e.ObjSizes != w.oSizes { |
||||
t.Error("got histogram", e.ObjSizes, "want", w.oSizes) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
t.Log(got.StringAll()) |
||||
|
||||
t.Logf("Root, flat: %+v", got.flatten(*got.root())) |
||||
t.Logf("Root: %+v", *got.root()) |
||||
t.Logf("/dir1/dira: %+v", *got.find("/dir1/dira")) |
||||
|
||||
} |
||||
|
||||
func Test_updateUsagePrefix(t *testing.T) { |
||||
base, err := ioutil.TempDir("", "Test_updateUsagePrefix") |
||||
if err != nil { |
||||
t.Skip(err) |
||||
} |
||||
base = filepath.Join(base, "bucket") |
||||
defer os.RemoveAll(base) |
||||
var files = []usageTestFile{ |
||||
{name: "bucket/rootfile", size: 10000}, |
||||
{name: "bucket/rootfile2", size: 10000}, |
||||
{name: "bucket/dir1/d1file", size: 2000}, |
||||
{name: "bucket/dir2/d2file", size: 300}, |
||||
{name: "bucket/dir1/dira/dafile", size: 100000}, |
||||
{name: "bucket/dir1/dira/dbfile", size: 200000}, |
||||
{name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000}, |
||||
{name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10}, |
||||
} |
||||
createUsageTestFiles(t, base, files) |
||||
|
||||
getSize := func(item Item) (i int64, err error) { |
||||
if item.Typ&os.ModeDir == 0 { |
||||
s, err := os.Stat(item.Path) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
return s.Size(), nil |
||||
} |
||||
return 0, nil |
||||
} |
||||
got, err := updateUsage(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// Test dirs
|
||||
var want = []struct { |
||||
path string |
||||
isNil bool |
||||
size, objs int |
||||
oSizes sizeHistogram |
||||
}{ |
||||
{ |
||||
path: "flat", |
||||
size: 1322310, |
||||
objs: 8, |
||||
oSizes: sizeHistogram{0: 2, 1: 6}, |
||||
}, |
||||
{ |
||||
path: "bucket/", |
||||
size: 20000, |
||||
objs: 2, |
||||
oSizes: sizeHistogram{1: 2}, |
||||
}, |
||||
{ |
||||
path: "bucket/dir1", |
||||
size: 2000, |
||||
objs: 1, |
||||
oSizes: sizeHistogram{1: 1}, |
||||
}, |
||||
{ |
||||
path: "bucket/dir1/dira", |
||||
size: 1300010, |
||||
objs: 4, |
||||
oSizes: sizeHistogram{0: 1, 1: 3}, |
||||
}, |
||||
{ |
||||
path: "bucket/dir1/dira/", |
||||
size: 1300010, |
||||
objs: 4, |
||||
oSizes: sizeHistogram{0: 1, 1: 3}, |
||||
}, |
||||
{ |
||||
path: "bucket/nonexistying", |
||||
isNil: true, |
||||
}, |
||||
} |
||||
|
||||
for _, w := range want { |
||||
t.Run(w.path, func(t *testing.T) { |
||||
e := got.find(w.path) |
||||
if w.path == "flat" { |
||||
f := got.flatten(*got.root()) |
||||
e = &f |
||||
} |
||||
if w.isNil { |
||||
if e != nil { |
||||
t.Error("want nil, got", e) |
||||
} |
||||
return |
||||
} |
||||
if e == nil { |
||||
t.Fatal("got nil result") |
||||
} |
||||
if e.Size != int64(w.size) { |
||||
t.Error("got size", e.Size, "want", w.size) |
||||
} |
||||
if e.Objects != uint64(w.objs) { |
||||
t.Error("got objects", e.Objects, "want", w.objs) |
||||
} |
||||
if e.ObjSizes != w.oSizes { |
||||
t.Error("got histogram", e.ObjSizes, "want", w.oSizes) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
files = []usageTestFile{ |
||||
{ |
||||
name: "bucket/newfolder/afile", |
||||
size: 4, |
||||
}, |
||||
{ |
||||
name: "bucket/newfolder/anotherone", |
||||
size: 1, |
||||
}, |
||||
{ |
||||
name: "bucket/newfolder/anemptyone", |
||||
size: 0, |
||||
}, |
||||
{ |
||||
name: "bucket/dir1/fileindir1", |
||||
size: 20000, |
||||
}, |
||||
{ |
||||
name: "bucket/dir1/dirc/fileindirc", |
||||
size: 20000, |
||||
}, |
||||
{ |
||||
name: "bucket/rootfile3", |
||||
size: 1000, |
||||
}, |
||||
} |
||||
createUsageTestFiles(t, base, files) |
||||
got, err = updateUsage(context.Background(), base, got, func() {}, getSize) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
want = []struct { |
||||
path string |
||||
isNil bool |
||||
size, objs int |
||||
oSizes sizeHistogram |
||||
}{ |
||||
{ |
||||
path: "flat", |
||||
size: 1363315, |
||||
objs: 14, |
||||
oSizes: sizeHistogram{0: 6, 1: 8}, |
||||
}, |
||||
{ |
||||
path: "bucket/", |
||||
size: 21000, |
||||
objs: 3, |
||||
oSizes: sizeHistogram{0: 1, 1: 2}, |
||||
}, |
||||
{ |
||||
path: "bucket/newfolder", |
||||
size: 5, |
||||
objs: 3, |
||||
oSizes: sizeHistogram{0: 3}, |
||||
}, |
||||
{ |
||||
path: "bucket/dir1/dira", |
||||
size: 1300010, |
||||
objs: 4, |
||||
oSizes: sizeHistogram{0: 1, 1: 3}, |
||||
}, |
||||
{ |
||||
path: "bucket/nonexistying", |
||||
isNil: true, |
||||
}, |
||||
} |
||||
|
||||
for _, w := range want { |
||||
t.Run(w.path, func(t *testing.T) { |
||||
e := got.find(w.path) |
||||
if w.path == "flat" { |
||||
f := got.flatten(*got.root()) |
||||
e = &f |
||||
} |
||||
if w.isNil { |
||||
if e != nil { |
||||
t.Error("want nil, got", e) |
||||
} |
||||
return |
||||
} |
||||
if e == nil { |
||||
t.Fatal("got nil result") |
||||
} |
||||
if e.Size != int64(w.size) { |
||||
t.Error("got size", e.Size, "want", w.size) |
||||
} |
||||
if e.Objects != uint64(w.objs) { |
||||
t.Error("got objects", e.Objects, "want", w.objs) |
||||
} |
||||
if e.ObjSizes != w.oSizes { |
||||
t.Error("got histogram", e.ObjSizes, "want", w.oSizes) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
files = []usageTestFile{ |
||||
{ |
||||
name: "bucket/dir1/dira/dirasub/fileindira2", |
||||
size: 200, |
||||
}, |
||||
} |
||||
|
||||
createUsageTestFiles(t, base, files) |
||||
err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile")) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
// Changed dir must be picked up in this many cycles.
|
||||
for i := 0; i < dataUsageUpdateDirCycles; i++ { |
||||
got, err = updateUsage(context.Background(), base, got, func() {}, getSize) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
|
||||
want = []struct { |
||||
path string |
||||
isNil bool |
||||
size, objs int |
||||
oSizes sizeHistogram |
||||
}{ |
||||
{ |
||||
path: "flat", |
||||
size: 363515, |
||||
objs: 14, |
||||
oSizes: sizeHistogram{0: 7, 1: 7}, |
||||
}, |
||||
{ |
||||
path: "bucket/dir1/dira", |
||||
size: 300210, |
||||
objs: 4, |
||||
oSizes: sizeHistogram{0: 2, 1: 2}, |
||||
}, |
||||
} |
||||
|
||||
for _, w := range want { |
||||
t.Run(w.path, func(t *testing.T) { |
||||
e := got.find(w.path) |
||||
if w.path == "flat" { |
||||
f := got.flatten(*got.root()) |
||||
e = &f |
||||
} |
||||
if w.isNil { |
||||
if e != nil { |
||||
t.Error("want nil, got", e) |
||||
} |
||||
return |
||||
} |
||||
if e == nil { |
||||
t.Fatal("got nil result") |
||||
} |
||||
if e.Size != int64(w.size) { |
||||
t.Error("got size", e.Size, "want", w.size) |
||||
} |
||||
if e.Objects != uint64(w.objs) { |
||||
t.Error("got objects", e.Objects, "want", w.objs) |
||||
} |
||||
if e.ObjSizes != w.oSizes { |
||||
t.Error("got histogram", e.ObjSizes, "want", w.oSizes) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
t.Log(got.StringAll()) |
||||
|
||||
t.Logf("Root, flat: %+v", got.flatten(*got.root())) |
||||
t.Logf("Root: %+v", *got.root()) |
||||
t.Logf("bucket/dir1/dira: %+v", *got.find("bucket/dir1/dira")) |
||||
} |
||||
|
||||
func createUsageTestFiles(t *testing.T, base string, files []usageTestFile) { |
||||
for _, f := range files { |
||||
err := os.MkdirAll(filepath.Dir(filepath.Join(base, f.name)), os.ModePerm) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
err = ioutil.WriteFile(filepath.Join(base, f.name), make([]byte, f.size), os.ModePerm) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func Test_dataUsageCacheSerialize(t *testing.T) { |
||||
base, err := ioutil.TempDir("", "Test_dataUsageCacheSerialize") |
||||
if err != nil { |
||||
t.Skip(err) |
||||
} |
||||
defer os.RemoveAll(base) |
||||
var files = []usageTestFile{ |
||||
{name: "rootfile", size: 10000}, |
||||
{name: "rootfile2", size: 10000}, |
||||
{name: "dir1/d1file", size: 2000}, |
||||
{name: "dir2/d2file", size: 300}, |
||||
{name: "dir1/dira/dafile", size: 100000}, |
||||
{name: "dir1/dira/dbfile", size: 200000}, |
||||
{name: "dir1/dira/dirasub/dcfile", size: 1000000}, |
||||
{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10}, |
||||
} |
||||
createUsageTestFiles(t, base, files) |
||||
|
||||
getSize := func(item Item) (i int64, err error) { |
||||
if item.Typ&os.ModeDir == 0 { |
||||
s, err := os.Stat(item.Path) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
return s.Size(), nil |
||||
} |
||||
return 0, nil |
||||
} |
||||
want, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
b := want.serialize() |
||||
t.Log("serialize -> ", len(b), "bytes") |
||||
|
||||
var got dataUsageCache |
||||
err = got.deserialize(b) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if got.Info.LastUpdate.IsZero() { |
||||
t.Error("lastupdate not set") |
||||
} |
||||
|
||||
if fmt.Sprint(want) == fmt.Sprint(got) { |
||||
t.Fatalf("deserialize mismatch\nwant: %+v\ngot: %+v", want, got) |
||||
} |
||||
} |
||||
Loading…
Reference in new issue