From 59a81a73c6977246a8288468cfbbbf97fa1f1c4b Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Wed, 15 Apr 2015 12:35:23 -0700 Subject: [PATCH] Adding lru to memory driver, not wired to command line opts --- Godeps/Godeps.json | 6 +- .../github.com/golang/groupcache/lru/lru.go | 121 +++++++++++++++++ .../golang/groupcache/lru/lru_test.go | 73 +++++++++++ pkg/api/api_test.go | 2 +- pkg/server/server.go | 2 +- pkg/storage/drivers/memory/memory.go | 124 ++++++++++-------- pkg/storage/drivers/memory/memory_test.go | 2 +- 7 files changed, 273 insertions(+), 57 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go create mode 100644 Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ba12af788..ea896c339 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,6 @@ { "ImportPath": "github.com/minio-io/minio", - "GoVersion": "go1.4", + "GoVersion": "go1.4.2", "Packages": [ "./..." ], @@ -9,6 +9,10 @@ "ImportPath": "github.com/dustin/go-humanize", "Rev": "8cc1aaa2d955ee82833337cfb10babc42be6bce6" }, + { + "ImportPath": "github.com/golang/groupcache/lru", + "Rev": "604ed5785183e59ae2789449d89e73f3a2a77987" + }, { "ImportPath": "github.com/gorilla/context", "Rev": "50c25fb3b2b3b3cc724e9b6ac75fb44b3bccd0da" diff --git a/Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go b/Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go new file mode 100644 index 000000000..cdfe2991f --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go @@ -0,0 +1,121 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package lru implements an LRU cache. +package lru + +import "container/list" + +// Cache is an LRU cache. It is not safe for concurrent access. +type Cache struct { + // MaxEntries is the maximum number of cache entries before + // an item is evicted. Zero means no limit. + MaxEntries int + + // OnEvicted optionally specificies a callback function to be + // executed when an entry is purged from the cache. + OnEvicted func(key Key, value interface{}) + + ll *list.List + cache map[interface{}]*list.Element +} + +// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators +type Key interface{} + +type entry struct { + key Key + value interface{} +} + +// New creates a new Cache. +// If maxEntries is zero, the cache has no limit and it's assumed +// that eviction is done by the caller. +func New(maxEntries int) *Cache { + return &Cache{ + MaxEntries: maxEntries, + ll: list.New(), + cache: make(map[interface{}]*list.Element), + } +} + +// Add adds a value to the cache. +func (c *Cache) Add(key Key, value interface{}) { + if c.cache == nil { + c.cache = make(map[interface{}]*list.Element) + c.ll = list.New() + } + if ee, ok := c.cache[key]; ok { + c.ll.MoveToFront(ee) + ee.Value.(*entry).value = value + return + } + ele := c.ll.PushFront(&entry{key, value}) + c.cache[key] = ele + if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { + c.RemoveOldest() + } +} + +// Get looks up a key's value from the cache. +func (c *Cache) Get(key Key) (value interface{}, ok bool) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.ll.MoveToFront(ele) + return ele.Value.(*entry).value, true + } + return +} + +// Remove removes the provided key from the cache. +func (c *Cache) Remove(key Key) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.removeElement(ele) + } +} + +// RemoveOldest removes the oldest item from the cache. +func (c *Cache) RemoveOldest() { + if c.cache == nil { + return + } + ele := c.ll.Back() + if ele != nil { + c.removeElement(ele) + } +} + +func (c *Cache) removeElement(e *list.Element) { + c.ll.Remove(e) + kv := e.Value.(*entry) + delete(c.cache, kv.key) + if c.OnEvicted != nil { + c.OnEvicted(kv.key, kv.value) + } +} + +// Len returns the number of items in the cache. +func (c *Cache) Len() int { + if c.cache == nil { + return 0 + } + return c.ll.Len() +} diff --git a/Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go b/Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go new file mode 100644 index 000000000..98a2656e8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go @@ -0,0 +1,73 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lru + +import ( + "testing" +) + +type simpleStruct struct { + int + string +} + +type complexStruct struct { + int + simpleStruct +} + +var getTests = []struct { + name string + keyToAdd interface{} + keyToGet interface{} + expectedOk bool +}{ + {"string_hit", "myKey", "myKey", true}, + {"string_miss", "myKey", "nonsense", false}, + {"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true}, + {"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false}, + {"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}}, + complexStruct{1, simpleStruct{2, "three"}}, true}, +} + +func TestGet(t *testing.T) { + for _, tt := range getTests { + lru := New(0) + lru.Add(tt.keyToAdd, 1234) + val, ok := lru.Get(tt.keyToGet) + if ok != tt.expectedOk { + t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok) + } else if ok && val != 1234 { + t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val) + } + } +} + +func TestRemove(t *testing.T) { + lru := New(0) + lru.Add("myKey", 1234) + if val, ok := lru.Get("myKey"); !ok { + t.Fatal("TestRemove returned no match") + } else if val != 1234 { + t.Fatalf("TestRemove failed. Expected %d, got %v", 1234, val) + } + + lru.Remove("myKey") + if _, ok := lru.Get("myKey"); ok { + t.Fatal("TestRemove returned a removed entry") + } +} diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 179ee60db..8c74e0d21 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -60,7 +60,7 @@ var _ = Suite(&MySuite{ var _ = Suite(&MySuite{ initDriver: func() (drivers.Driver, string) { - _, _, driver := memory.Start() + _, _, driver := memory.Start(1000) return driver, "" }, }) diff --git a/pkg/server/server.go b/pkg/server/server.go index faa54f333..971121e01 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -136,7 +136,7 @@ func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, status switch { case driverType == Memory: { - ctrlChan, statusChan, driver = memory.Start() + ctrlChan, statusChan, driver = memory.Start(1000) ctrlChans = append(ctrlChans, ctrlChan) statusChans = append(statusChans, statusChan) } diff --git a/pkg/storage/drivers/memory/memory.go b/pkg/storage/drivers/memory/memory.go index 199839645..d48aed75c 100644 --- a/pkg/storage/drivers/memory/memory.go +++ b/pkg/storage/drivers/memory/memory.go @@ -30,14 +30,16 @@ import ( "crypto/md5" "encoding/hex" + "github.com/golang/groupcache/lru" "io/ioutil" ) // memoryDriver - local variables type memoryDriver struct { - bucketdata map[string]storedBucket - objectdata map[string]storedObject - lock *sync.RWMutex + bucketMetadata map[string]storedBucket + objectMetadata map[string]storedObject + objects *lru.Cache + lock *sync.RWMutex } type storedBucket struct { @@ -48,19 +50,21 @@ type storedBucket struct { type storedObject struct { metadata drivers.ObjectMetadata - data []byte } // Start memory object server -func Start() (chan<- string, <-chan error, drivers.Driver) { +func Start(maxObjects int) (chan<- string, <-chan error, drivers.Driver) { ctrlChannel := make(chan string) errorChannel := make(chan error) memory := new(memoryDriver) - memory.bucketdata = make(map[string]storedBucket) - memory.objectdata = make(map[string]storedObject) + memory.bucketMetadata = make(map[string]storedBucket) + memory.objectMetadata = make(map[string]storedObject) + memory.objects = lru.New(maxObjects) memory.lock = new(sync.RWMutex) + memory.objects.OnEvicted = memory.evictObject + go start(ctrlChannel, errorChannel) return ctrlChannel, errorChannel, memory } @@ -73,15 +77,18 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error) { func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) { memory.lock.RLock() defer memory.lock.RUnlock() - if _, ok := memory.bucketdata[bucket]; ok == false { + if _, ok := memory.bucketMetadata[bucket]; ok == false { return 0, drivers.BucketNotFound{Bucket: bucket} } // get object - key := object - if val, ok := memory.objectdata[key]; ok { - objectBuffer := bytes.NewBuffer(val.data) - written, err := io.Copy(w, objectBuffer) - return written, err + objectKey := bucket + "/" + object + if _, ok := memory.objectMetadata[objectKey]; ok { + if data, ok := memory.objects.Get(objectKey); ok { + dataSlice := data.([]byte) + objectBuffer := bytes.NewBuffer(dataSlice) + written, err := io.Copy(w, objectBuffer) + return written, err + } } return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object} } @@ -104,10 +111,10 @@ func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string, func (memory memoryDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) { memory.lock.RLock() defer memory.lock.RUnlock() - if _, ok := memory.bucketdata[bucket]; ok == false { + if _, ok := memory.bucketMetadata[bucket]; ok == false { return drivers.BucketMetadata{}, drivers.BucketNotFound{Bucket: bucket} } - return memory.bucketdata[bucket].metadata, nil + return memory.bucketMetadata[bucket].metadata, nil } // CreateBucketPolicy - Not implemented @@ -124,12 +131,14 @@ func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy, func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error { memory.lock.RLock() - if _, ok := memory.bucketdata[bucket]; ok == false { + if _, ok := memory.bucketMetadata[bucket]; ok == false { memory.lock.RUnlock() return drivers.BucketNotFound{Bucket: bucket} } - if _, ok := memory.objectdata[key]; ok == true { + objectKey := bucket + "/" + key + + if _, ok := memory.objectMetadata[objectKey]; ok == true { memory.lock.RUnlock() return drivers.ObjectExists{Bucket: bucket, Object: key} } @@ -143,6 +152,7 @@ func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, var bytesBuffer bytes.Buffer var newObject = storedObject{} + var dataSlice []byte if _, ok := io.Copy(&bytesBuffer, data); ok == nil { size := bytesBuffer.Len() md5SumBytes := md5.Sum(bytesBuffer.Bytes()) @@ -156,19 +166,15 @@ func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, Md5: md5Sum, Size: int64(size), } - newObject.data = bytesBuffer.Bytes() + dataSlice = bytesBuffer.Bytes() } memory.lock.Lock() - if _, ok := memory.bucketdata[bucket]; ok == false { - memory.lock.Unlock() - return drivers.BucketNotFound{Bucket: bucket} - } - - if _, ok := memory.objectdata[key]; ok == true { + if _, ok := memory.objectMetadata[objectKey]; ok == true { memory.lock.Unlock() return drivers.ObjectExists{Bucket: bucket, Object: key} } - memory.objectdata[key] = newObject + memory.objectMetadata[objectKey] = newObject + memory.objects.Add(objectKey, dataSlice) memory.lock.Unlock() return nil } @@ -181,7 +187,7 @@ func (memory memoryDriver) CreateBucket(bucketName string) error { return drivers.BucketNameInvalid{Bucket: bucketName} } - if _, ok := memory.bucketdata[bucketName]; ok == true { + if _, ok := memory.bucketMetadata[bucketName]; ok == true { memory.lock.RUnlock() return drivers.BucketExists{Bucket: bucketName} } @@ -193,7 +199,7 @@ func (memory memoryDriver) CreateBucket(bucketName string) error { newBucket.metadata.Created = time.Now() memory.lock.Lock() defer memory.lock.Unlock() - memory.bucketdata[bucketName] = newBucket + memory.bucketMetadata[bucketName] = newBucket return nil } @@ -233,35 +239,38 @@ func (memory memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedNa func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) { memory.lock.RLock() defer memory.lock.RUnlock() - if _, ok := memory.bucketdata[bucket]; ok == false { + if _, ok := memory.bucketMetadata[bucket]; ok == false { return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket} } var results []drivers.ObjectMetadata var keys []string - for key := range memory.objectdata { - switch true { - // Prefix absent, delimit object key based on delimiter - case resources.IsDelimiterSet(): - delimitedName := delimiter(key, resources.Delimiter) + for key := range memory.objectMetadata { + if strings.HasPrefix(key, bucket+"/") { + key = key[len(bucket)+1:] switch true { - case delimitedName == "" || delimitedName == key: + // Prefix absent, delimit object key based on delimiter + case resources.IsDelimiterSet(): + delimitedName := delimiter(key, resources.Delimiter) + switch true { + case delimitedName == "" || delimitedName == key: + keys = appendUniq(keys, key) + case delimitedName != "": + resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName) + } + // Prefix present, delimit object key with prefix key based on delimiter + case resources.IsDelimiterPrefixSet(): + if strings.HasPrefix(key, resources.Prefix) { + trimmedName := strings.TrimPrefix(key, resources.Prefix) + delimitedName := delimiter(trimmedName, resources.Delimiter) + resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources) + } + // Prefix present, nothing to delimit + case resources.IsPrefixSet(): + keys = appendUniq(keys, key) + // Prefix and delimiter absent + case resources.IsDefault(): keys = appendUniq(keys, key) - case delimitedName != "": - resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName) - } - // Prefix present, delimit object key with prefix key based on delimiter - case resources.IsDelimiterPrefixSet(): - if strings.HasPrefix(key, resources.Prefix) { - trimmedName := strings.TrimPrefix(key, resources.Prefix) - delimitedName := delimiter(trimmedName, resources.Delimiter) - resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources) } - // Prefix present, nothing to delimit - case resources.IsPrefixSet(): - keys = appendUniq(keys, key) - // Prefix and delimiter absent - case resources.IsDefault(): - keys = appendUniq(keys, key) } } sort.Strings(keys) @@ -269,7 +278,7 @@ func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketRe if len(results) == resources.Maxkeys { return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil } - object := memory.objectdata[key] + object := memory.objectMetadata[bucket+"/"+key] if bucket == object.metadata.Bucket { results = append(results, object.metadata) } @@ -294,7 +303,7 @@ func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) { memory.lock.RLock() defer memory.lock.RUnlock() var results []drivers.BucketMetadata - for _, bucket := range memory.bucketdata { + for _, bucket := range memory.bucketMetadata { results = append(results, bucket.metadata) } sort.Sort(ByBucketName(results)) @@ -306,11 +315,20 @@ func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (driver memory.lock.RLock() defer memory.lock.RUnlock() // check if bucket exists - if _, ok := memory.bucketdata[bucket]; ok == false { + if _, ok := memory.bucketMetadata[bucket]; ok == false { return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket} } - if object, ok := memory.objectdata[key]; ok == true { + + objectKey := bucket + "/" + key + + if object, ok := memory.objectMetadata[objectKey]; ok == true { return object.metadata, nil } return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key} } + +func (memory memoryDriver) evictObject(key lru.Key, value interface{}) { + k := key.(string) + + delete(memory.objectMetadata, k) +} diff --git a/pkg/storage/drivers/memory/memory_test.go b/pkg/storage/drivers/memory/memory_test.go index b0b8f177d..bd79e0dd9 100644 --- a/pkg/storage/drivers/memory/memory_test.go +++ b/pkg/storage/drivers/memory/memory_test.go @@ -31,7 +31,7 @@ var _ = Suite(&MySuite{}) func (s *MySuite) TestAPISuite(c *C) { create := func() drivers.Driver { - _, _, store := Start() + _, _, store := Start(1000) return store } drivers.APITestSuite(c, create)