From 12de98fb6238a4123b9452764e5c6bcd651ddd8e Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 29 Jun 2015 16:02:17 -0700 Subject: [PATCH] Rename memory driver as cache --- .../cache-multipart.go} | 150 +++++------ .../{memory/memory.go => cache/cache.go} | 237 +++++++++--------- .../memory_test.go => cache/cache_test.go} | 2 +- .../{multipart.go => donut-multipart.go} | 0 4 files changed, 192 insertions(+), 197 deletions(-) rename pkg/storage/drivers/{memory/memory_multipart.go => cache/cache-multipart.go} (73%) rename pkg/storage/drivers/{memory/memory.go => cache/cache.go} (68%) rename pkg/storage/drivers/{memory/memory_test.go => cache/cache_test.go} (98%) rename pkg/storage/drivers/donut/{multipart.go => donut-multipart.go} (100%) diff --git a/pkg/storage/drivers/memory/memory_multipart.go b/pkg/storage/drivers/cache/cache-multipart.go similarity index 73% rename from pkg/storage/drivers/memory/memory_multipart.go rename to pkg/storage/drivers/cache/cache-multipart.go index d3d6f8ba2..56c29bf24 100644 --- a/pkg/storage/drivers/memory/memory_multipart.go +++ b/pkg/storage/drivers/cache/cache-multipart.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package memory +package cache import ( "bytes" @@ -35,54 +35,54 @@ import ( "github.com/minio/minio/pkg/storage/drivers" ) -func (memory *memoryDriver) NewMultipartUpload(bucket, key, contentType string) (string, error) { - memory.lock.RLock() +func (cache *cacheDriver) NewMultipartUpload(bucket, key, contentType string) (string, error) { + cache.lock.RLock() if !drivers.IsValidBucket(bucket) { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) } if !drivers.IsValidObjectName(key) { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) } - if _, ok := memory.storedBuckets[bucket]; ok == false { - memory.lock.RUnlock() + if _, ok := cache.storedBuckets[bucket]; ok == false { + cache.lock.RUnlock() return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } - storedBucket := memory.storedBuckets[bucket] + storedBucket := cache.storedBuckets[bucket] objectKey := bucket + "/" + key if _, ok := storedBucket.objectMetadata[objectKey]; ok == true { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil) } - memory.lock.RUnlock() + cache.lock.RUnlock() - memory.lock.Lock() + cache.lock.Lock() id := []byte(strconv.FormatInt(rand.Int63(), 10) + bucket + key + time.Now().String()) uploadIDSum := sha512.Sum512(id) uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47] - memory.storedBuckets[bucket].multiPartSession[key] = multiPartSession{ + cache.storedBuckets[bucket].multiPartSession[key] = multiPartSession{ uploadID: uploadID, initiated: time.Now(), totalParts: 0, } - memory.lock.Unlock() + cache.lock.Unlock() return uploadID, nil } -func (memory *memoryDriver) AbortMultipartUpload(bucket, key, uploadID string) error { - memory.lock.RLock() - storedBucket := memory.storedBuckets[bucket] +func (cache *cacheDriver) AbortMultipartUpload(bucket, key, uploadID string) error { + cache.lock.RLock() + storedBucket := cache.storedBuckets[bucket] if storedBucket.multiPartSession[key].uploadID != uploadID { - memory.lock.RUnlock() + cache.lock.RUnlock() return iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil) } - memory.lock.RUnlock() + cache.lock.RUnlock() - memory.cleanupMultiparts(bucket, key, uploadID) - memory.cleanupMultipartSession(bucket, key, uploadID) + cache.cleanupMultiparts(bucket, key, uploadID) + cache.cleanupMultipartSession(bucket, key, uploadID) return nil } @@ -90,17 +90,17 @@ func getMultipartKey(key string, uploadID string, partNumber int) string { return key + "?uploadId=" + uploadID + "&partNumber=" + strconv.Itoa(partNumber) } -func (memory *memoryDriver) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { +func (cache *cacheDriver) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { // Verify upload id - memory.lock.RLock() - storedBucket := memory.storedBuckets[bucket] + cache.lock.RLock() + storedBucket := cache.storedBuckets[bucket] if storedBucket.multiPartSession[key].uploadID != uploadID { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil) } - memory.lock.RUnlock() + cache.lock.RUnlock() - etag, err := memory.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data) + etag, err := cache.createObjectPart(bucket, key, uploadID, partID, "", expectedMD5Sum, size, data) if err != nil { return "", iodine.New(err, nil) } @@ -109,29 +109,29 @@ func (memory *memoryDriver) CreateObjectPart(bucket, key, uploadID string, partI return etag, nil } -// createObject - PUT object to memory buffer -func (memory *memoryDriver) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { - memory.lock.RLock() +// createObject - PUT object to cache buffer +func (cache *cacheDriver) createObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { + cache.lock.RLock() if !drivers.IsValidBucket(bucket) { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) } if !drivers.IsValidObjectName(key) { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) } - if _, ok := memory.storedBuckets[bucket]; ok == false { - memory.lock.RUnlock() + if _, ok := cache.storedBuckets[bucket]; ok == false { + cache.lock.RUnlock() return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } - storedBucket := memory.storedBuckets[bucket] + storedBucket := cache.storedBuckets[bucket] // get object key partKey := bucket + "/" + getMultipartKey(key, uploadID, partID) if _, ok := storedBucket.partMetadata[partKey]; ok == true { - memory.lock.RUnlock() + cache.lock.RUnlock() return storedBucket.partMetadata[partKey].ETag, nil } - memory.lock.RUnlock() + cache.lock.RUnlock() if contentType == "" { contentType = "application/octet-stream" @@ -170,9 +170,9 @@ func (memory *memoryDriver) createObjectPart(bucket, key, uploadID string, partI md5SumBytes := hash.Sum(nil) totalLength := int64(len(readBytes)) - memory.lock.Lock() - memory.multiPartObjects.Set(partKey, readBytes) - memory.lock.Unlock() + cache.lock.Lock() + cache.multiPartObjects.Set(partKey, readBytes) + cache.lock.Unlock() // setting up for de-allocation readBytes = nil @@ -190,31 +190,31 @@ func (memory *memoryDriver) createObjectPart(bucket, key, uploadID string, partI Size: totalLength, } - memory.lock.Lock() + cache.lock.Lock() storedBucket.partMetadata[partKey] = newPart multiPartSession := storedBucket.multiPartSession[key] multiPartSession.totalParts++ storedBucket.multiPartSession[key] = multiPartSession - memory.storedBuckets[bucket] = storedBucket - memory.lock.Unlock() + cache.storedBuckets[bucket] = storedBucket + cache.lock.Unlock() return md5Sum, nil } -func (memory *memoryDriver) cleanupMultipartSession(bucket, key, uploadID string) { - memory.lock.Lock() - defer memory.lock.Unlock() - delete(memory.storedBuckets[bucket].multiPartSession, key) +func (cache *cacheDriver) cleanupMultipartSession(bucket, key, uploadID string) { + cache.lock.Lock() + defer cache.lock.Unlock() + delete(cache.storedBuckets[bucket].multiPartSession, key) } -func (memory *memoryDriver) cleanupMultiparts(bucket, key, uploadID string) { - for i := 1; i <= memory.storedBuckets[bucket].multiPartSession[key].totalParts; i++ { +func (cache *cacheDriver) cleanupMultiparts(bucket, key, uploadID string) { + for i := 1; i <= cache.storedBuckets[bucket].multiPartSession[key].totalParts; i++ { objectKey := bucket + "/" + getMultipartKey(key, uploadID, i) - memory.multiPartObjects.Delete(objectKey) + cache.multiPartObjects.Delete(objectKey) } } -func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (string, error) { +func (cache *cacheDriver) CompleteMultipartUpload(bucket, key, uploadID string, parts map[int]string) (string, error) { if !drivers.IsValidBucket(bucket) { return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) } @@ -222,26 +222,26 @@ func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) } // Verify upload id - memory.lock.RLock() - if _, ok := memory.storedBuckets[bucket]; ok == false { - memory.lock.RUnlock() + cache.lock.RLock() + if _, ok := cache.storedBuckets[bucket]; ok == false { + cache.lock.RUnlock() return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } - storedBucket := memory.storedBuckets[bucket] + storedBucket := cache.storedBuckets[bucket] if storedBucket.multiPartSession[key].uploadID != uploadID { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.InvalidUploadID{UploadID: uploadID}, nil) } - memory.lock.RUnlock() + cache.lock.RUnlock() - memory.lock.Lock() + cache.lock.Lock() var size int64 var fullObject bytes.Buffer for i := 1; i <= len(parts); i++ { recvMD5 := parts[i] - object, ok := memory.multiPartObjects.Get(bucket + "/" + getMultipartKey(key, uploadID, i)) + object, ok := cache.multiPartObjects.Get(bucket + "/" + getMultipartKey(key, uploadID, i)) if ok == false { - memory.lock.Unlock() + cache.lock.Unlock() return "", iodine.New(errors.New("missing part: "+strconv.Itoa(i)), nil) } size += int64(len(object)) @@ -261,20 +261,20 @@ func (memory *memoryDriver) CompleteMultipartUpload(bucket, key, uploadID string object = nil go debug.FreeOSMemory() } - memory.lock.Unlock() + cache.lock.Unlock() md5sumSlice := md5.Sum(fullObject.Bytes()) // this is needed for final verification inside CreateObject, do not convert this to hex md5sum := base64.StdEncoding.EncodeToString(md5sumSlice[:]) - etag, err := memory.CreateObject(bucket, key, "", md5sum, size, &fullObject) + etag, err := cache.CreateObject(bucket, key, "", md5sum, size, &fullObject) if err != nil { // No need to call internal cleanup functions here, caller will call AbortMultipartUpload() // which would in-turn cleanup properly in accordance with S3 Spec return "", iodine.New(err, nil) } fullObject.Reset() - memory.cleanupMultiparts(bucket, key, uploadID) - memory.cleanupMultipartSession(bucket, key, uploadID) + cache.cleanupMultiparts(bucket, key, uploadID) + cache.cleanupMultipartSession(bucket, key, uploadID) return etag, nil } @@ -285,14 +285,14 @@ func (a byKey) Len() int { return len(a) } func (a byKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byKey) Less(i, j int) bool { return a[i].Key < a[j].Key } -func (memory *memoryDriver) ListMultipartUploads(bucket string, resources drivers.BucketMultipartResourcesMetadata) (drivers.BucketMultipartResourcesMetadata, error) { +func (cache *cacheDriver) ListMultipartUploads(bucket string, resources drivers.BucketMultipartResourcesMetadata) (drivers.BucketMultipartResourcesMetadata, error) { // TODO handle delimiter - memory.lock.RLock() - defer memory.lock.RUnlock() - if _, ok := memory.storedBuckets[bucket]; ok == false { + cache.lock.RLock() + defer cache.lock.RUnlock() + if _, ok := cache.storedBuckets[bucket]; ok == false { return drivers.BucketMultipartResourcesMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } - storedBucket := memory.storedBuckets[bucket] + storedBucket := cache.storedBuckets[bucket] var uploads []*drivers.UploadMetadata for key, session := range storedBucket.multiPartSession { @@ -346,14 +346,14 @@ func (a partNumber) Len() int { return len(a) } func (a partNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber } -func (memory *memoryDriver) ListObjectParts(bucket, key string, resources drivers.ObjectResourcesMetadata) (drivers.ObjectResourcesMetadata, error) { +func (cache *cacheDriver) ListObjectParts(bucket, key string, resources drivers.ObjectResourcesMetadata) (drivers.ObjectResourcesMetadata, error) { // Verify upload id - memory.lock.RLock() - defer memory.lock.RUnlock() - if _, ok := memory.storedBuckets[bucket]; ok == false { + cache.lock.RLock() + defer cache.lock.RUnlock() + if _, ok := cache.storedBuckets[bucket]; ok == false { return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } - storedBucket := memory.storedBuckets[bucket] + storedBucket := cache.storedBuckets[bucket] if _, ok := storedBucket.multiPartSession[key]; ok == false { return drivers.ObjectResourcesMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: key}, nil) } @@ -390,10 +390,10 @@ func (memory *memoryDriver) ListObjectParts(bucket, key string, resources driver return objectResourcesMetadata, nil } -func (memory *memoryDriver) expiredPart(a ...interface{}) { +func (cache *cacheDriver) expiredPart(a ...interface{}) { key := a[0].(string) // loop through all buckets - for _, storedBucket := range memory.storedBuckets { + for _, storedBucket := range cache.storedBuckets { delete(storedBucket.partMetadata, key) } debug.FreeOSMemory() diff --git a/pkg/storage/drivers/memory/memory.go b/pkg/storage/drivers/cache/cache.go similarity index 68% rename from pkg/storage/drivers/memory/memory.go rename to pkg/storage/drivers/cache/cache.go index ca2a88130..527babced 100644 --- a/pkg/storage/drivers/memory/memory.go +++ b/pkg/storage/drivers/cache/cache.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package memory +package cache import ( "bufio" @@ -37,8 +37,8 @@ import ( "github.com/minio/minio/pkg/storage/trove" ) -// memoryDriver - local variables -type memoryDriver struct { +// cacheDriver - local variables +type cacheDriver struct { storedBuckets map[string]storedBucket lock *sync.RWMutex objects *trove.Cache @@ -64,65 +64,65 @@ const ( totalBuckets = 100 ) -// NewDriver instantiate a new memory driver +// NewDriver instantiate a new cache driver func NewDriver(maxSize uint64, expiration time.Duration) (drivers.Driver, error) { - memory := new(memoryDriver) - memory.storedBuckets = make(map[string]storedBucket) - memory.objects = trove.NewCache(maxSize, expiration) - memory.maxSize = maxSize - memory.expiration = expiration - memory.multiPartObjects = trove.NewCache(0, time.Duration(0)) - memory.lock = new(sync.RWMutex) - - memory.objects.OnExpired = memory.expiredObject - memory.multiPartObjects.OnExpired = memory.expiredPart - - // set up memory expiration - memory.objects.ExpireObjects(time.Second * 5) - return memory, nil + cache := new(cacheDriver) + cache.storedBuckets = make(map[string]storedBucket) + cache.objects = trove.NewCache(maxSize, expiration) + cache.maxSize = maxSize + cache.expiration = expiration + cache.multiPartObjects = trove.NewCache(0, time.Duration(0)) + cache.lock = new(sync.RWMutex) + + cache.objects.OnExpired = cache.expiredObject + cache.multiPartObjects.OnExpired = cache.expiredPart + + // set up cache expiration + cache.objects.ExpireObjects(time.Second * 5) + return cache, nil } -// GetObject - GET object from memory buffer -func (memory *memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) { - memory.lock.RLock() +// GetObject - GET object from cache buffer +func (cache *cacheDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) { + cache.lock.RLock() if !drivers.IsValidBucket(bucket) { - memory.lock.RUnlock() + cache.lock.RUnlock() return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) } if !drivers.IsValidObjectName(object) { - memory.lock.RUnlock() + cache.lock.RUnlock() return 0, iodine.New(drivers.ObjectNameInvalid{Object: object}, nil) } - if _, ok := memory.storedBuckets[bucket]; ok == false { - memory.lock.RUnlock() + if _, ok := cache.storedBuckets[bucket]; ok == false { + cache.lock.RUnlock() return 0, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } objectKey := bucket + "/" + object - data, ok := memory.objects.Get(objectKey) + data, ok := cache.objects.Get(objectKey) if !ok { - memory.lock.RUnlock() + cache.lock.RUnlock() return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, nil) } written, err := io.Copy(w, bytes.NewBuffer(data)) - memory.lock.RUnlock() + cache.lock.RUnlock() return written, iodine.New(err, nil) } -// GetPartialObject - GET object from memory buffer range -func (memory *memoryDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) { +// GetPartialObject - GET object from cache buffer range +func (cache *cacheDriver) GetPartialObject(w io.Writer, bucket, object string, start, length int64) (int64, error) { errParams := map[string]string{ "bucket": bucket, "object": object, "start": strconv.FormatInt(start, 10), "length": strconv.FormatInt(length, 10), } - memory.lock.RLock() + cache.lock.RLock() if !drivers.IsValidBucket(bucket) { - memory.lock.RUnlock() + cache.lock.RUnlock() return 0, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, errParams) } if !drivers.IsValidObjectName(object) { - memory.lock.RUnlock() + cache.lock.RUnlock() return 0, iodine.New(drivers.ObjectNameInvalid{Object: object}, errParams) } if start < 0 { @@ -132,49 +132,49 @@ func (memory *memoryDriver) GetPartialObject(w io.Writer, bucket, object string, }, errParams) } objectKey := bucket + "/" + object - data, ok := memory.objects.Get(objectKey) + data, ok := cache.objects.Get(objectKey) if !ok { - memory.lock.RUnlock() + cache.lock.RUnlock() return 0, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: object}, errParams) } written, err := io.CopyN(w, bytes.NewBuffer(data[start:]), length) - memory.lock.RUnlock() + cache.lock.RUnlock() return written, iodine.New(err, nil) } // GetBucketMetadata - -func (memory *memoryDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) { - memory.lock.RLock() - defer memory.lock.RUnlock() +func (cache *cacheDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) { + cache.lock.RLock() + defer cache.lock.RUnlock() if !drivers.IsValidBucket(bucket) { return drivers.BucketMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) } - if _, ok := memory.storedBuckets[bucket]; ok == false { + if _, ok := cache.storedBuckets[bucket]; ok == false { return drivers.BucketMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } - return memory.storedBuckets[bucket].bucketMetadata, nil + return cache.storedBuckets[bucket].bucketMetadata, nil } // SetBucketMetadata - -func (memory *memoryDriver) SetBucketMetadata(bucket, acl string) error { - memory.lock.RLock() +func (cache *cacheDriver) SetBucketMetadata(bucket, acl string) error { + cache.lock.RLock() if !drivers.IsValidBucket(bucket) { - memory.lock.RUnlock() + cache.lock.RUnlock() return iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) } - if _, ok := memory.storedBuckets[bucket]; ok == false { - memory.lock.RUnlock() + if _, ok := cache.storedBuckets[bucket]; ok == false { + cache.lock.RUnlock() return iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } if strings.TrimSpace(acl) == "" { acl = "private" } - memory.lock.RUnlock() - memory.lock.Lock() - defer memory.lock.Unlock() - storedBucket := memory.storedBuckets[bucket] + cache.lock.RUnlock() + cache.lock.Lock() + defer cache.lock.Unlock() + storedBucket := cache.storedBuckets[bucket] storedBucket.bucketMetadata.ACL = drivers.BucketACL(acl) - memory.storedBuckets[bucket] = storedBucket + cache.storedBuckets[bucket] = storedBucket return nil } @@ -197,44 +197,44 @@ func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) error { return iodine.New(errors.New("invalid argument"), nil) } -func (memory *memoryDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { - if size > int64(memory.maxSize) { +func (cache *cacheDriver) CreateObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { + if size > int64(cache.maxSize) { generic := drivers.GenericObjectError{Bucket: bucket, Object: key} return "", iodine.New(drivers.EntityTooLarge{ GenericObjectError: generic, Size: strconv.FormatInt(size, 10), - MaxSize: strconv.FormatUint(memory.maxSize, 10), + MaxSize: strconv.FormatUint(cache.maxSize, 10), }, nil) } - md5sum, err := memory.createObject(bucket, key, contentType, expectedMD5Sum, size, data) + md5sum, err := cache.createObject(bucket, key, contentType, expectedMD5Sum, size, data) // free debug.FreeOSMemory() return md5sum, iodine.New(err, nil) } -// createObject - PUT object to memory buffer -func (memory *memoryDriver) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { - memory.lock.RLock() +// createObject - PUT object to cache buffer +func (cache *cacheDriver) createObject(bucket, key, contentType, expectedMD5Sum string, size int64, data io.Reader) (string, error) { + cache.lock.RLock() if !drivers.IsValidBucket(bucket) { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) } if !drivers.IsValidObjectName(key) { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) } - if _, ok := memory.storedBuckets[bucket]; ok == false { - memory.lock.RUnlock() + if _, ok := cache.storedBuckets[bucket]; ok == false { + cache.lock.RUnlock() return "", iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } - storedBucket := memory.storedBuckets[bucket] + storedBucket := cache.storedBuckets[bucket] // get object key objectKey := bucket + "/" + key if _, ok := storedBucket.objectMetadata[objectKey]; ok == true { - memory.lock.RUnlock() + cache.lock.RUnlock() return "", iodine.New(drivers.ObjectExists{Bucket: bucket, Object: key}, nil) } - memory.lock.RUnlock() + cache.lock.RUnlock() if contentType == "" { contentType = "application/octet-stream" @@ -272,12 +272,12 @@ func (memory *memoryDriver) createObject(bucket, key, contentType, expectedMD5Su md5SumBytes := hash.Sum(nil) totalLength := len(readBytes) - memory.lock.Lock() - ok := memory.objects.Set(objectKey, readBytes) + cache.lock.Lock() + ok := cache.objects.Set(objectKey, readBytes) // setting up for de-allocation readBytes = nil go debug.FreeOSMemory() - memory.lock.Unlock() + cache.lock.Unlock() if !ok { return "", iodine.New(drivers.InternalError{}, nil) } @@ -300,33 +300,33 @@ func (memory *memoryDriver) createObject(bucket, key, contentType, expectedMD5Su Size: int64(totalLength), } - memory.lock.Lock() + cache.lock.Lock() storedBucket.objectMetadata[objectKey] = newObject - memory.storedBuckets[bucket] = storedBucket - memory.lock.Unlock() + cache.storedBuckets[bucket] = storedBucket + cache.lock.Unlock() return newObject.Md5, nil } -// CreateBucket - create bucket in memory -func (memory *memoryDriver) CreateBucket(bucketName, acl string) error { - memory.lock.RLock() - if len(memory.storedBuckets) == totalBuckets { - memory.lock.RUnlock() +// CreateBucket - create bucket in cache +func (cache *cacheDriver) CreateBucket(bucketName, acl string) error { + cache.lock.RLock() + if len(cache.storedBuckets) == totalBuckets { + cache.lock.RUnlock() return iodine.New(drivers.TooManyBuckets{Bucket: bucketName}, nil) } if !drivers.IsValidBucket(bucketName) { - memory.lock.RUnlock() + cache.lock.RUnlock() return iodine.New(drivers.BucketNameInvalid{Bucket: bucketName}, nil) } if !drivers.IsValidBucketACL(acl) { - memory.lock.RUnlock() + cache.lock.RUnlock() return iodine.New(drivers.InvalidACL{ACL: acl}, nil) } - if _, ok := memory.storedBuckets[bucketName]; ok == true { - memory.lock.RUnlock() + if _, ok := cache.storedBuckets[bucketName]; ok == true { + cache.lock.RUnlock() return iodine.New(drivers.BucketExists{Bucket: bucketName}, nil) } - memory.lock.RUnlock() + cache.lock.RUnlock() if strings.TrimSpace(acl) == "" { // default is private @@ -340,9 +340,9 @@ func (memory *memoryDriver) CreateBucket(bucketName, acl string) error { newBucket.bucketMetadata.Name = bucketName newBucket.bucketMetadata.Created = time.Now().UTC() newBucket.bucketMetadata.ACL = drivers.BucketACL(acl) - memory.lock.Lock() - defer memory.lock.Unlock() - memory.storedBuckets[bucketName] = newBucket + cache.lock.Lock() + cache.storedBuckets[bucketName] = newBucket + cache.lock.Unlock() return nil } @@ -364,7 +364,7 @@ func appendUniq(slice []string, i string) []string { return append(slice, i) } -func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delim string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) { +func (cache *cacheDriver) filterDelimiterPrefix(keys []string, key, delim string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) { switch true { case key == r.Prefix: keys = appendUniq(keys, key) @@ -377,7 +377,7 @@ func (memory *memoryDriver) filterDelimiterPrefix(keys []string, key, delim stri return keys, r } -func (memory *memoryDriver) listObjects(keys []string, key string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) { +func (cache *cacheDriver) listObjects(keys []string, key string, r drivers.BucketResourcesMetadata) ([]string, drivers.BucketResourcesMetadata) { switch true { // Prefix absent, delimit object key based on delimiter case r.IsDelimiterSet(): @@ -393,7 +393,7 @@ func (memory *memoryDriver) listObjects(keys []string, key string, r drivers.Buc if strings.HasPrefix(key, r.Prefix) { trimmedName := strings.TrimPrefix(key, r.Prefix) delim := delimiter(trimmedName, r.Delimiter) - keys, r = memory.filterDelimiterPrefix(keys, key, delim, r) + keys, r = cache.filterDelimiterPrefix(keys, key, delim, r) } // Prefix present, nothing to delimit case r.IsPrefixSet(): @@ -405,26 +405,26 @@ func (memory *memoryDriver) listObjects(keys []string, key string, r drivers.Buc return keys, r } -// ListObjects - list objects from memory -func (memory *memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) { - memory.lock.RLock() - defer memory.lock.RUnlock() +// ListObjects - list objects from cache +func (cache *cacheDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) { + cache.lock.RLock() + defer cache.lock.RUnlock() if !drivers.IsValidBucket(bucket) { return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) } if !drivers.IsValidObjectName(resources.Prefix) { return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.ObjectNameInvalid{Object: resources.Prefix}, nil) } - if _, ok := memory.storedBuckets[bucket]; ok == false { + if _, ok := cache.storedBuckets[bucket]; ok == false { return nil, drivers.BucketResourcesMetadata{IsTruncated: false}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } var results []drivers.ObjectMetadata var keys []string - storedBucket := memory.storedBuckets[bucket] + storedBucket := cache.storedBuckets[bucket] for key := range storedBucket.objectMetadata { if strings.HasPrefix(key, bucket+"/") { key = key[len(bucket)+1:] - keys, resources = memory.listObjects(keys, key, resources) + keys, resources = cache.listObjects(keys, key, resources) } } var newKeys []string @@ -453,34 +453,29 @@ func (memory *memoryDriver) ListObjects(bucket string, resources drivers.BucketR return results, resources, nil } -// ByBucketName is a type for sorting bucket metadata by bucket name -type ByBucketName []drivers.BucketMetadata +// byBucketName is a type for sorting bucket metadata by bucket name +type byBucketName []drivers.BucketMetadata -// Len of bucket name -func (b ByBucketName) Len() int { return len(b) } +func (b byBucketName) Len() int { return len(b) } +func (b byBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name } -// Swap bucket i, j -func (b ByBucketName) Swap(i, j int) { b[i], b[j] = b[j], b[i] } - -// Less -func (b ByBucketName) Less(i, j int) bool { return b[i].Name < b[j].Name } - -// ListBuckets - List buckets from memory -func (memory *memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) { - memory.lock.RLock() - defer memory.lock.RUnlock() +// ListBuckets - List buckets from cache +func (cache *cacheDriver) ListBuckets() ([]drivers.BucketMetadata, error) { + cache.lock.RLock() + defer cache.lock.RUnlock() var results []drivers.BucketMetadata - for _, bucket := range memory.storedBuckets { + for _, bucket := range cache.storedBuckets { results = append(results, bucket.bucketMetadata) } - sort.Sort(ByBucketName(results)) + sort.Sort(byBucketName(results)) return results, nil } -// GetObjectMetadata - get object metadata from memory -func (memory *memoryDriver) GetObjectMetadata(bucket, key string) (drivers.ObjectMetadata, error) { - memory.lock.RLock() - defer memory.lock.RUnlock() +// GetObjectMetadata - get object metadata from cache +func (cache *cacheDriver) GetObjectMetadata(bucket, key string) (drivers.ObjectMetadata, error) { + cache.lock.RLock() + defer cache.lock.RUnlock() // check if bucket exists if !drivers.IsValidBucket(bucket) { return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNameInvalid{Bucket: bucket}, nil) @@ -488,10 +483,10 @@ func (memory *memoryDriver) GetObjectMetadata(bucket, key string) (drivers.Objec if !drivers.IsValidObjectName(key) { return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNameInvalid{Object: key}, nil) } - if _, ok := memory.storedBuckets[bucket]; ok == false { + if _, ok := cache.storedBuckets[bucket]; ok == false { return drivers.ObjectMetadata{}, iodine.New(drivers.BucketNotFound{Bucket: bucket}, nil) } - storedBucket := memory.storedBuckets[bucket] + storedBucket := cache.storedBuckets[bucket] objectKey := bucket + "/" + key if object, ok := storedBucket.objectMetadata[objectKey]; ok == true { return object, nil @@ -499,18 +494,18 @@ func (memory *memoryDriver) GetObjectMetadata(bucket, key string) (drivers.Objec return drivers.ObjectMetadata{}, iodine.New(drivers.ObjectNotFound{Bucket: bucket, Object: key}, nil) } -func (memory *memoryDriver) expiredObject(a ...interface{}) { - cacheStats := memory.objects.Stats() +func (cache *cacheDriver) expiredObject(a ...interface{}) { + cacheStats := cache.objects.Stats() log.Printf("CurrentSize: %d, CurrentItems: %d, TotalExpirations: %d", cacheStats.Bytes, cacheStats.Items, cacheStats.Expired) key := a[0].(string) // loop through all buckets - for bucket, storedBucket := range memory.storedBuckets { + for bucket, storedBucket := range cache.storedBuckets { delete(storedBucket.objectMetadata, key) // remove bucket if no objects found anymore if len(storedBucket.objectMetadata) == 0 { - if time.Since(memory.storedBuckets[bucket].bucketMetadata.Created) > memory.expiration { - delete(memory.storedBuckets, bucket) + if time.Since(cache.storedBuckets[bucket].bucketMetadata.Created) > cache.expiration { + delete(cache.storedBuckets, bucket) } } } diff --git a/pkg/storage/drivers/memory/memory_test.go b/pkg/storage/drivers/cache/cache_test.go similarity index 98% rename from pkg/storage/drivers/memory/memory_test.go rename to pkg/storage/drivers/cache/cache_test.go index c20fd51db..9a9ba25eb 100644 --- a/pkg/storage/drivers/memory/memory_test.go +++ b/pkg/storage/drivers/cache/cache_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package memory +package cache import ( "testing" diff --git a/pkg/storage/drivers/donut/multipart.go b/pkg/storage/drivers/donut/donut-multipart.go similarity index 100% rename from pkg/storage/drivers/donut/multipart.go rename to pkg/storage/drivers/donut/donut-multipart.go