Merge pull request #468 from fkautz/pr_out_adding_lru_to_memory_driver_not_wired_to_command_line_opts

master
Frederick F. Kautz IV 9 years ago
commit 7d01300d82
  1. 6
      Godeps/Godeps.json
  2. 121
      Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go
  3. 73
      Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go
  4. 2
      pkg/api/api_test.go
  5. 2
      pkg/server/server.go
  6. 124
      pkg/storage/drivers/memory/memory.go
  7. 2
      pkg/storage/drivers/memory/memory_test.go

6
Godeps/Godeps.json generated vendored

@ -1,6 +1,6 @@
{
"ImportPath": "github.com/minio-io/minio",
"GoVersion": "go1.4",
"GoVersion": "go1.4.2",
"Packages": [
"./..."
],
@ -9,6 +9,10 @@
"ImportPath": "github.com/dustin/go-humanize",
"Rev": "8cc1aaa2d955ee82833337cfb10babc42be6bce6"
},
{
"ImportPath": "github.com/golang/groupcache/lru",
"Rev": "604ed5785183e59ae2789449d89e73f3a2a77987"
},
{
"ImportPath": "github.com/gorilla/context",
"Rev": "50c25fb3b2b3b3cc724e9b6ac75fb44b3bccd0da"

@ -0,0 +1,121 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package lru implements an LRU cache.
package lru
import "container/list"
// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {
// MaxEntries is the maximum number of cache entries before
// an item is evicted. Zero means no limit.
MaxEntries int
// OnEvicted optionally specificies a callback function to be
// executed when an entry is purged from the cache.
OnEvicted func(key Key, value interface{})
ll *list.List
cache map[interface{}]*list.Element
}
// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
type Key interface{}
type entry struct {
key Key
value interface{}
}
// New creates a new Cache.
// If maxEntries is zero, the cache has no limit and it's assumed
// that eviction is done by the caller.
func New(maxEntries int) *Cache {
return &Cache{
MaxEntries: maxEntries,
ll: list.New(),
cache: make(map[interface{}]*list.Element),
}
}
// Add adds a value to the cache.
func (c *Cache) Add(key Key, value interface{}) {
if c.cache == nil {
c.cache = make(map[interface{}]*list.Element)
c.ll = list.New()
}
if ee, ok := c.cache[key]; ok {
c.ll.MoveToFront(ee)
ee.Value.(*entry).value = value
return
}
ele := c.ll.PushFront(&entry{key, value})
c.cache[key] = ele
if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
c.RemoveOldest()
}
}
// Get looks up a key's value from the cache.
func (c *Cache) Get(key Key) (value interface{}, ok bool) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.ll.MoveToFront(ele)
return ele.Value.(*entry).value, true
}
return
}
// Remove removes the provided key from the cache.
func (c *Cache) Remove(key Key) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.removeElement(ele)
}
}
// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
if c.cache == nil {
return
}
ele := c.ll.Back()
if ele != nil {
c.removeElement(ele)
}
}
func (c *Cache) removeElement(e *list.Element) {
c.ll.Remove(e)
kv := e.Value.(*entry)
delete(c.cache, kv.key)
if c.OnEvicted != nil {
c.OnEvicted(kv.key, kv.value)
}
}
// Len returns the number of items in the cache.
func (c *Cache) Len() int {
if c.cache == nil {
return 0
}
return c.ll.Len()
}

@ -0,0 +1,73 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package lru
import (
"testing"
)
type simpleStruct struct {
int
string
}
type complexStruct struct {
int
simpleStruct
}
var getTests = []struct {
name string
keyToAdd interface{}
keyToGet interface{}
expectedOk bool
}{
{"string_hit", "myKey", "myKey", true},
{"string_miss", "myKey", "nonsense", false},
{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
complexStruct{1, simpleStruct{2, "three"}}, true},
}
func TestGet(t *testing.T) {
for _, tt := range getTests {
lru := New(0)
lru.Add(tt.keyToAdd, 1234)
val, ok := lru.Get(tt.keyToGet)
if ok != tt.expectedOk {
t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
} else if ok && val != 1234 {
t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
}
}
}
func TestRemove(t *testing.T) {
lru := New(0)
lru.Add("myKey", 1234)
if val, ok := lru.Get("myKey"); !ok {
t.Fatal("TestRemove returned no match")
} else if val != 1234 {
t.Fatalf("TestRemove failed. Expected %d, got %v", 1234, val)
}
lru.Remove("myKey")
if _, ok := lru.Get("myKey"); ok {
t.Fatal("TestRemove returned a removed entry")
}
}

@ -60,7 +60,7 @@ var _ = Suite(&MySuite{
var _ = Suite(&MySuite{
initDriver: func() (drivers.Driver, string) {
_, _, driver := memory.Start()
_, _, driver := memory.Start(1000)
return driver, ""
},
})

@ -136,7 +136,7 @@ func getDriverChannels(driverType DriverType) (ctrlChans []chan<- string, status
switch {
case driverType == Memory:
{
ctrlChan, statusChan, driver = memory.Start()
ctrlChan, statusChan, driver = memory.Start(1000)
ctrlChans = append(ctrlChans, ctrlChan)
statusChans = append(statusChans, statusChan)
}

@ -30,14 +30,16 @@ import (
"crypto/md5"
"encoding/hex"
"github.com/golang/groupcache/lru"
"io/ioutil"
)
// memoryDriver - local variables
type memoryDriver struct {
bucketdata map[string]storedBucket
objectdata map[string]storedObject
lock *sync.RWMutex
bucketMetadata map[string]storedBucket
objectMetadata map[string]storedObject
objects *lru.Cache
lock *sync.RWMutex
}
type storedBucket struct {
@ -48,19 +50,21 @@ type storedBucket struct {
type storedObject struct {
metadata drivers.ObjectMetadata
data []byte
}
// Start memory object server
func Start() (chan<- string, <-chan error, drivers.Driver) {
func Start(maxObjects int) (chan<- string, <-chan error, drivers.Driver) {
ctrlChannel := make(chan string)
errorChannel := make(chan error)
memory := new(memoryDriver)
memory.bucketdata = make(map[string]storedBucket)
memory.objectdata = make(map[string]storedObject)
memory.bucketMetadata = make(map[string]storedBucket)
memory.objectMetadata = make(map[string]storedObject)
memory.objects = lru.New(maxObjects)
memory.lock = new(sync.RWMutex)
memory.objects.OnEvicted = memory.evictObject
go start(ctrlChannel, errorChannel)
return ctrlChannel, errorChannel, memory
}
@ -73,15 +77,18 @@ func start(ctrlChannel <-chan string, errorChannel chan<- error) {
func (memory memoryDriver) GetObject(w io.Writer, bucket string, object string) (int64, error) {
memory.lock.RLock()
defer memory.lock.RUnlock()
if _, ok := memory.bucketdata[bucket]; ok == false {
if _, ok := memory.bucketMetadata[bucket]; ok == false {
return 0, drivers.BucketNotFound{Bucket: bucket}
}
// get object
key := object
if val, ok := memory.objectdata[key]; ok {
objectBuffer := bytes.NewBuffer(val.data)
written, err := io.Copy(w, objectBuffer)
return written, err
objectKey := bucket + "/" + object
if _, ok := memory.objectMetadata[objectKey]; ok {
if data, ok := memory.objects.Get(objectKey); ok {
dataSlice := data.([]byte)
objectBuffer := bytes.NewBuffer(dataSlice)
written, err := io.Copy(w, objectBuffer)
return written, err
}
}
return 0, drivers.ObjectNotFound{Bucket: bucket, Object: object}
}
@ -104,10 +111,10 @@ func (memory memoryDriver) GetPartialObject(w io.Writer, bucket, object string,
func (memory memoryDriver) GetBucketMetadata(bucket string) (drivers.BucketMetadata, error) {
memory.lock.RLock()
defer memory.lock.RUnlock()
if _, ok := memory.bucketdata[bucket]; ok == false {
if _, ok := memory.bucketMetadata[bucket]; ok == false {
return drivers.BucketMetadata{}, drivers.BucketNotFound{Bucket: bucket}
}
return memory.bucketdata[bucket].metadata, nil
return memory.bucketMetadata[bucket].metadata, nil
}
// CreateBucketPolicy - Not implemented
@ -124,12 +131,14 @@ func (memory memoryDriver) GetBucketPolicy(bucket string) (drivers.BucketPolicy,
func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string, data io.Reader) error {
memory.lock.RLock()
if _, ok := memory.bucketdata[bucket]; ok == false {
if _, ok := memory.bucketMetadata[bucket]; ok == false {
memory.lock.RUnlock()
return drivers.BucketNotFound{Bucket: bucket}
}
if _, ok := memory.objectdata[key]; ok == true {
objectKey := bucket + "/" + key
if _, ok := memory.objectMetadata[objectKey]; ok == true {
memory.lock.RUnlock()
return drivers.ObjectExists{Bucket: bucket, Object: key}
}
@ -143,6 +152,7 @@ func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string,
var bytesBuffer bytes.Buffer
var newObject = storedObject{}
var dataSlice []byte
if _, ok := io.Copy(&bytesBuffer, data); ok == nil {
size := bytesBuffer.Len()
md5SumBytes := md5.Sum(bytesBuffer.Bytes())
@ -156,19 +166,15 @@ func (memory memoryDriver) CreateObject(bucket, key, contentType, md5sum string,
Md5: md5Sum,
Size: int64(size),
}
newObject.data = bytesBuffer.Bytes()
dataSlice = bytesBuffer.Bytes()
}
memory.lock.Lock()
if _, ok := memory.bucketdata[bucket]; ok == false {
memory.lock.Unlock()
return drivers.BucketNotFound{Bucket: bucket}
}
if _, ok := memory.objectdata[key]; ok == true {
if _, ok := memory.objectMetadata[objectKey]; ok == true {
memory.lock.Unlock()
return drivers.ObjectExists{Bucket: bucket, Object: key}
}
memory.objectdata[key] = newObject
memory.objectMetadata[objectKey] = newObject
memory.objects.Add(objectKey, dataSlice)
memory.lock.Unlock()
return nil
}
@ -181,7 +187,7 @@ func (memory memoryDriver) CreateBucket(bucketName string) error {
return drivers.BucketNameInvalid{Bucket: bucketName}
}
if _, ok := memory.bucketdata[bucketName]; ok == true {
if _, ok := memory.bucketMetadata[bucketName]; ok == true {
memory.lock.RUnlock()
return drivers.BucketExists{Bucket: bucketName}
}
@ -193,7 +199,7 @@ func (memory memoryDriver) CreateBucket(bucketName string) error {
newBucket.metadata.Created = time.Now()
memory.lock.Lock()
defer memory.lock.Unlock()
memory.bucketdata[bucketName] = newBucket
memory.bucketMetadata[bucketName] = newBucket
return nil
}
@ -233,35 +239,38 @@ func (memory memoryDriver) filterDelimiterPrefix(keys []string, key, delimitedNa
func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketResourcesMetadata) ([]drivers.ObjectMetadata, drivers.BucketResourcesMetadata, error) {
memory.lock.RLock()
defer memory.lock.RUnlock()
if _, ok := memory.bucketdata[bucket]; ok == false {
if _, ok := memory.bucketMetadata[bucket]; ok == false {
return []drivers.ObjectMetadata{}, drivers.BucketResourcesMetadata{IsTruncated: false}, drivers.BucketNotFound{Bucket: bucket}
}
var results []drivers.ObjectMetadata
var keys []string
for key := range memory.objectdata {
switch true {
// Prefix absent, delimit object key based on delimiter
case resources.IsDelimiterSet():
delimitedName := delimiter(key, resources.Delimiter)
for key := range memory.objectMetadata {
if strings.HasPrefix(key, bucket+"/") {
key = key[len(bucket)+1:]
switch true {
case delimitedName == "" || delimitedName == key:
// Prefix absent, delimit object key based on delimiter
case resources.IsDelimiterSet():
delimitedName := delimiter(key, resources.Delimiter)
switch true {
case delimitedName == "" || delimitedName == key:
keys = appendUniq(keys, key)
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
// Prefix present, delimit object key with prefix key based on delimiter
case resources.IsDelimiterPrefixSet():
if strings.HasPrefix(key, resources.Prefix) {
trimmedName := strings.TrimPrefix(key, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
}
// Prefix present, nothing to delimit
case resources.IsPrefixSet():
keys = appendUniq(keys, key)
// Prefix and delimiter absent
case resources.IsDefault():
keys = appendUniq(keys, key)
case delimitedName != "":
resources.CommonPrefixes = appendUniq(resources.CommonPrefixes, delimitedName)
}
// Prefix present, delimit object key with prefix key based on delimiter
case resources.IsDelimiterPrefixSet():
if strings.HasPrefix(key, resources.Prefix) {
trimmedName := strings.TrimPrefix(key, resources.Prefix)
delimitedName := delimiter(trimmedName, resources.Delimiter)
resources, keys = memory.filterDelimiterPrefix(keys, key, delimitedName, resources)
}
// Prefix present, nothing to delimit
case resources.IsPrefixSet():
keys = appendUniq(keys, key)
// Prefix and delimiter absent
case resources.IsDefault():
keys = appendUniq(keys, key)
}
}
sort.Strings(keys)
@ -269,7 +278,7 @@ func (memory memoryDriver) ListObjects(bucket string, resources drivers.BucketRe
if len(results) == resources.Maxkeys {
return results, drivers.BucketResourcesMetadata{IsTruncated: true}, nil
}
object := memory.objectdata[key]
object := memory.objectMetadata[bucket+"/"+key]
if bucket == object.metadata.Bucket {
results = append(results, object.metadata)
}
@ -294,7 +303,7 @@ func (memory memoryDriver) ListBuckets() ([]drivers.BucketMetadata, error) {
memory.lock.RLock()
defer memory.lock.RUnlock()
var results []drivers.BucketMetadata
for _, bucket := range memory.bucketdata {
for _, bucket := range memory.bucketMetadata {
results = append(results, bucket.metadata)
}
sort.Sort(ByBucketName(results))
@ -306,11 +315,20 @@ func (memory memoryDriver) GetObjectMetadata(bucket, key, prefix string) (driver
memory.lock.RLock()
defer memory.lock.RUnlock()
// check if bucket exists
if _, ok := memory.bucketdata[bucket]; ok == false {
if _, ok := memory.bucketMetadata[bucket]; ok == false {
return drivers.ObjectMetadata{}, drivers.BucketNotFound{Bucket: bucket}
}
if object, ok := memory.objectdata[key]; ok == true {
objectKey := bucket + "/" + key
if object, ok := memory.objectMetadata[objectKey]; ok == true {
return object.metadata, nil
}
return drivers.ObjectMetadata{}, drivers.ObjectNotFound{Bucket: bucket, Object: key}
}
func (memory memoryDriver) evictObject(key lru.Key, value interface{}) {
k := key.(string)
delete(memory.objectMetadata, k)
}

@ -31,7 +31,7 @@ var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
create := func() drivers.Driver {
_, _, store := Start()
_, _, store := Start(1000)
return store
}
drivers.APITestSuite(c, create)

Loading…
Cancel
Save