oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #535 from fkautz/pr_out_adding_lru

Browse Source
master
Frederick F. Kautz IV 10 years ago
parent b6daf99dcd 5fc5f79ab7
commit c8db3e1c3b
3 changed files with 272 additions and 20 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 150
      pkg/storage/drivers/memory/lru/lru.go
  2. 110
      pkg/storage/drivers/memory/lru/lru_test.go
  3. 32
      pkg/storage/drivers/memory/memory.go

150
pkg/storage/drivers/memory/lru/lru.go
Unescape View File

@ -0,0 +1,150 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
---
Modifications from Minio under the following license:
Minimalist Object Storage, (C) 2015 Minio, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package lru implements an LRU cache.
package lru
import (
"container/list"
)
// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {
// MaxEntries is the maximum number of cache entries before
// an item is evicted. Zero means no limit.
MaxEntries int
// OnEvicted optionally specificies a callback function to be
// executed when an entry is purged from the cache.
OnEvicted func(key Key, value interface{})
ll *list.List
cache map[interface{}]*list.Element
}
// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
type Key interface{}
type entry struct {
key Key
value interface{}
}
// New creates a new Cache.
// If maxEntries is zero, the cache has no limit and it's assumed
// that eviction is done by the caller.
func New(maxEntries int) *Cache {
return &Cache{
MaxEntries: maxEntries,
ll: list.New(),
cache: make(map[interface{}]*list.Element),
}
}
// Add adds a value to the cache.
func (c *Cache) Add(key Key, value interface{}) {
if c.cache == nil {
c.cache = make(map[interface{}]*list.Element)
c.ll = list.New()
}
if ee, ok := c.cache[key]; ok {
c.ll.MoveToFront(ee)
ee.Value.(*entry).value = value
return
}
ele := c.ll.PushFront(&entry{key, value})
c.cache[key] = ele
if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
c.RemoveOldest()
}
}
// Get looks up a key's value from the cache.
func (c *Cache) Get(key Key) (value interface{}, ok bool) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.ll.MoveToFront(ele)
return ele.Value.(*entry).value, true
}
return
}
// Remove removes the provided key from the cache.
func (c *Cache) Remove(key Key) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.removeElement(ele)
}
}
// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
if c.cache == nil {
return
}
ele := c.ll.Back()
if ele != nil {
c.removeElement(ele)
}
}
// GetOldest returns the oldest key, value, ok without modifying the lru
func (c *Cache) GetOldest() (key Key, value interface{}, ok bool) {
if c.cache == nil {
return nil, nil, false
}
ele := c.ll.Back()
if ele != nil {
return ele.Value.(*entry).key, ele.Value.(*entry).value, true
}
return nil, nil, false
}
func (c *Cache) removeElement(e *list.Element) {
c.ll.Remove(e)
kv := e.Value.(*entry)
delete(c.cache, kv.key)
if c.OnEvicted != nil {
c.OnEvicted(kv.key, kv.value)
}
}
// Len returns the number of items in the cache.
func (c *Cache) Len() int {
if c.cache == nil {
return 0
}
return c.ll.Len()
}

110
pkg/storage/drivers/memory/lru/lru_test.go
Unescape View File

@ -0,0 +1,110 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
---
Modifications from Minio under the following license:
Minimalist Object Storage, (C) 2015 Minio, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package lru
import (
"testing"
)
type simpleStruct struct {
int
string
}
type complexStruct struct {
int
simpleStruct
}
var getTests = []struct {
name string
keyToAdd interface{}
keyToGet interface{}
expectedOk bool
}{
{"string_hit", "myKey", "myKey", true},
{"string_miss", "myKey", "nonsense", false},
{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
complexStruct{1, simpleStruct{2, "three"}}, true},
}
func TestGet(t *testing.T) {
for _, tt := range getTests {
lru := New(0)
lru.Add(tt.keyToAdd, 1234)
val, ok := lru.Get(tt.keyToGet)
if ok != tt.expectedOk {
t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
} else if ok && val != 1234 {
t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
}
}
}
func TestRemove(t *testing.T) {
lru := New(0)
lru.Add("myKey", 1234)
if val, ok := lru.Get("myKey"); !ok {
t.Fatal("TestRemove returned no match")
} else if val != 1234 {
t.Fatalf("TestRemove failed. Expected %d, got %v", 1234, val)
}
lru.Remove("myKey")
if _, ok := lru.Get("myKey"); ok {
t.Fatal("TestRemove returned a removed entry")
}
}
func TestOldest(t *testing.T) {
lru := New(0)
lru.Add("a", 1)
lru.Add("b", 2)
lru.Add("c", 3)
key, val, ok := lru.GetOldest()
if ok != true {
t.Fatalf("%s expected get to return 1 but got %v", "a", val)
}
if key != "a" && val != 1 {
t.Fatalf("%s expected get to return 1 but got %v", "a", val)
}
lru.RemoveOldest()
key, val, ok = lru.GetOldest()
if ok != true {
t.Fatalf("%s expected get to return 1 but got %v", "a", val)
}
if key != "b" && val != 2 {
t.Fatalf("%s expected get to return 1 but got %v", "a", val)
}
}

32
pkg/storage/drivers/memory/memory.go
Unescape View File

@ -33,9 +33,9 @@ import (
"io/ioutil"
"github.com/golang/groupcache/lru"
"github.com/minio-io/minio/pkg/iodine"
"github.com/minio-io/minio/pkg/storage/drivers"
"github.com/minio-io/minio/pkg/storage/drivers/memory/lru"
"github.com/minio-io/minio/pkg/utils/log"
"github.com/minio-io/minio/pkg/utils/split"
)
@ -59,7 +59,8 @@ type storedBucket struct {
}
type storedObject struct {
metadata drivers.ObjectMetadata
metadata drivers.ObjectMetadata
lastAccessed time.Time
}
const (
@ -483,29 +484,20 @@ func (memory *memoryDriver) expireObjects() {
if memory.shutdown {
return
}
var keysToRemove []string
if len(memory.objectMetadata) > 0 {
memory.lock.RLock()
var earliest time.Time
for key, object := range memory.objectMetadata {
if time.Now().Add(-memory.expiration).After(object.metadata.Created) {
keysToRemove = append(keysToRemove, key)
var sleepDuration time.Duration
memory.lock.Lock()
if k, _, ok := memory.objects.GetOldest(); ok {
key := k.(string)
object := memory.objectMetadata[key]
if time.Now().Sub(object.lastAccessed) > memory.expiration {
memory.objects.RemoveOldest()
} else {
if object.metadata.Created.Before(earliest) {
earliest = object.metadata.Created
}
sleepDuration = memory.expiration - time.Now().Sub(object.lastAccessed)
}
}
memory.lock.RUnlock()
memory.lock.Lock()
for _, key := range keysToRemove {
memory.objects.Remove(key)
}
memory.lock.Unlock()
sleepFor := earliest.Sub(time.Now())
if sleepFor > 0 {
time.Sleep(sleepFor)
}
time.Sleep(sleepDuration)
} else {
time.Sleep(memory.expiration)
}

Write Preview
Loading…
Cancel
Save