XL/PutObject: Add single putObject and multipart caching. (#2115)
- Additionally adds test cases as well for object cache. - Adds auto-expiry with expiration and cleanup time interval. Fixes #2080 Fixes #2091master
parent
b0c180b77c
commit
c0c8a8430e
@ -0,0 +1,39 @@ |
||||
## Object caching |
||||
|
||||
Object caching by turned on by default with following settings |
||||
|
||||
- Default cache size 8GB, can be changed from environment variable |
||||
``MINIO_CACHE_SIZE`` supports both SI and ISO IEC standard forms |
||||
for input size parameters. |
||||
|
||||
- Default expiration of entries is 72 hours, can be changed from |
||||
environment variable ``MINIO_CACHE_EXPIRY`` supportings Go |
||||
``time.Duration`` with valid units "ns", "us" (or "µs"), |
||||
"ms", "s", "m", "h". |
||||
|
||||
- Default expiry interval is 1/4th of the expiration hours, so |
||||
expiration sweep happens across the cache every 1/4th the time |
||||
duration of the set entry expiration duration. |
||||
|
||||
### Tricks |
||||
|
||||
Setting MINIO_CACHE_SIZE=0 will turn off caching entirely. |
||||
Setting MINIO_CACHE_EXPIRY=0s will turn off cache garbage collections, |
||||
all cached objects will never expire. |
||||
|
||||
### Behavior |
||||
|
||||
Caching happens for both GET and PUT. |
||||
|
||||
- GET caches new objects for entries not found in cache, |
||||
otherwise serves from the cache. |
||||
|
||||
- PUT/POST caches all successfully uploaded objects. |
||||
|
||||
NOTE: Cache is not populated if there are any errors |
||||
while reading from the disk. |
||||
|
||||
Expiration happens automatically based on the configured |
||||
interval as explained above, frequently accessed objects |
||||
stay alive for significantly longer time due to the fact |
||||
that expiration time is reset for every cache hit. |
@ -0,0 +1,62 @@ |
||||
``` |
||||
PACKAGE DOCUMENTATION |
||||
|
||||
package objcache |
||||
import "github.com/minio/minio/pkg/objcache" |
||||
|
||||
Package objcache implements in memory caching methods. |
||||
|
||||
VARIABLES |
||||
|
||||
var DefaultExpiry = time.Duration(72 * time.Hour) // 72hrs. |
||||
|
||||
DefaultExpiry represents default time duration value when individual |
||||
entries will be expired. |
||||
|
||||
var ErrCacheFull = errors.New("Not enough space in cache") |
||||
ErrCacheFull - cache is full. |
||||
|
||||
var ErrKeyNotFoundInCache = errors.New("Key not found in cache") |
||||
ErrKeyNotFoundInCache - key not found in cache. |
||||
|
||||
var NoExpiry = time.Duration(0) |
||||
NoExpiry represents caches to be permanent and can only be deleted. |
||||
|
||||
TYPES |
||||
|
||||
type Cache struct { |
||||
|
||||
// OnEviction - callback function for eviction |
||||
OnEviction func(key string) |
||||
// contains filtered or unexported fields |
||||
} |
||||
Cache holds the required variables to compose an in memory cache system |
||||
which also provides expiring key mechanism and also maxSize. |
||||
|
||||
func New(maxSize uint64, expiry time.Duration) *Cache |
||||
New - Return a new cache with a given default expiry duration. If the |
||||
expiry duration is less than one (or NoExpiry), the items in the cache |
||||
never expire (by default), and must be deleted manually. |
||||
|
||||
func (c *Cache) Create(key string, size int64) (w io.WriteCloser, err error) |
||||
Create - validates if object size fits with in cache size limit and |
||||
returns a io.WriteCloser to which object contents can be written and |
||||
finally Close()'d. During Close() we checks if the amount of data |
||||
written is equal to the size of the object, in which case it saves the |
||||
contents to object cache. |
||||
|
||||
func (c *Cache) Delete(key string) |
||||
Delete - delete deletes an entry from the cache. |
||||
|
||||
func (c *Cache) DeleteExpired() |
||||
DeleteExpired - deletes all the expired entries from the cache. |
||||
|
||||
func (c *Cache) Open(key string) (io.ReadSeeker, error) |
||||
Open - open the in-memory file, returns an in memory read seeker. |
||||
returns an error ErrNotFoundInCache, if the key does not exist. |
||||
|
||||
func (c *Cache) StopExpiry() |
||||
StopExpiry sends a message to the expiry routine to stop expiring cached |
||||
entries. NOTE: once this is called, cached entries will not be expired |
||||
if the consume has called this. |
||||
``` |
@ -0,0 +1,172 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
* |
||||
*/ |
||||
|
||||
package objcache |
||||
|
||||
import ( |
||||
"bytes" |
||||
"io" |
||||
"io/ioutil" |
||||
"testing" |
||||
"time" |
||||
) |
||||
|
||||
// TestObjCache - tests various cases for object cache behavior.
|
||||
func TestObjCache(t *testing.T) { |
||||
// Non exhaustive list of all object cache behavior cases.
|
||||
testCases := []struct { |
||||
expiry time.Duration |
||||
cacheSize uint64 |
||||
err error |
||||
closeErr error |
||||
}{ |
||||
// Validate if a key is not found in cache and Open fails.
|
||||
{ |
||||
expiry: NoExpiry, |
||||
cacheSize: 1024, |
||||
err: ErrKeyNotFoundInCache, |
||||
}, |
||||
// Validate if cache indicates that it is full and Create fails.
|
||||
{ |
||||
expiry: NoExpiry, |
||||
cacheSize: 1, |
||||
err: ErrCacheFull, |
||||
}, |
||||
// Validate if Create succeeds but Close fails to write to buffer.
|
||||
{ |
||||
expiry: NoExpiry, |
||||
cacheSize: 2, |
||||
closeErr: io.ErrShortBuffer, |
||||
}, |
||||
// Validate that Create and Close succeed, making sure to update the cache.
|
||||
{ |
||||
expiry: NoExpiry, |
||||
cacheSize: 1024, |
||||
}, |
||||
// Validate that Delete succeeds and Open fails with key not found in cache.
|
||||
{ |
||||
expiry: NoExpiry, |
||||
cacheSize: 1024, |
||||
err: ErrKeyNotFoundInCache, |
||||
}, |
||||
// Validate OnEviction function is called upon entry delete.
|
||||
{ |
||||
expiry: NoExpiry, |
||||
cacheSize: 1024, |
||||
}, |
||||
} |
||||
|
||||
// Test 1 validating Open failure.
|
||||
testCase := testCases[0] |
||||
cache := New(testCase.cacheSize, testCase.expiry) |
||||
_, err := cache.Open("test") |
||||
if testCase.err != err { |
||||
t.Errorf("Test case 2 expected to pass, failed instead %s", err) |
||||
} |
||||
|
||||
// Test 2 validating Create failure.
|
||||
testCase = testCases[1] |
||||
cache = New(testCase.cacheSize, testCase.expiry) |
||||
_, err = cache.Create("test", 2) |
||||
if testCase.err != err { |
||||
t.Errorf("Test case 2 expected to pass, failed instead %s", err) |
||||
} |
||||
|
||||
// Test 3 validating Create succeeds and returns a writer.
|
||||
// Subsequently we Close() without writing any data, to receive
|
||||
// `io.ErrShortBuffer`
|
||||
testCase = testCases[2] |
||||
cache = New(testCase.cacheSize, testCase.expiry) |
||||
w, err := cache.Create("test", 1) |
||||
if testCase.err != err { |
||||
t.Errorf("Test case 3 expected to pass, failed instead %s", err) |
||||
} |
||||
if err = w.Close(); err != testCase.closeErr { |
||||
t.Errorf("Test case 3 expected to pass, failed instead %s", err) |
||||
} |
||||
|
||||
// Test 4 validates Create and Close succeeds successfully caching
|
||||
// the writes.
|
||||
testCase = testCases[3] |
||||
cache = New(testCase.cacheSize, testCase.expiry) |
||||
w, err = cache.Create("test", 5) |
||||
if testCase.err != err { |
||||
t.Errorf("Test case 4 expected to pass, failed instead %s", err) |
||||
} |
||||
// Write '5' bytes.
|
||||
w.Write([]byte("Hello")) |
||||
// Close to successfully save into cache.
|
||||
if err = w.Close(); err != nil { |
||||
t.Errorf("Test case 4 expected to pass, failed instead %s", err) |
||||
} |
||||
r, err := cache.Open("test") |
||||
if err != nil { |
||||
t.Errorf("Test case 4 expected to pass, failed instead %s", err) |
||||
} |
||||
// Reads everything stored for key "test".
|
||||
cbytes, err := ioutil.ReadAll(r) |
||||
if err != nil { |
||||
t.Errorf("Test case 4 expected to pass, failed instead %s", err) |
||||
} |
||||
// Validate if read bytes match.
|
||||
if !bytes.Equal(cbytes, []byte("Hello")) { |
||||
t.Errorf("Test case 4 expected to pass. wanted \"Hello\", got %s", string(cbytes)) |
||||
} |
||||
|
||||
// Test 5 validates Delete succeeds and Open fails with err
|
||||
testCase = testCases[4] |
||||
cache = New(testCase.cacheSize, testCase.expiry) |
||||
w, err = cache.Create("test", 5) |
||||
if err != nil { |
||||
t.Errorf("Test case 5 expected to pass, failed instead %s", err) |
||||
} |
||||
// Write '5' bytes.
|
||||
w.Write([]byte("Hello")) |
||||
// Close to successfully save into cache.
|
||||
if err = w.Close(); err != nil { |
||||
t.Errorf("Test case 5 expected to pass, failed instead %s", err) |
||||
} |
||||
// Delete the cache entry.
|
||||
cache.Delete("test") |
||||
_, err = cache.Open("test") |
||||
if testCase.err != err { |
||||
t.Errorf("Test case 5 expected to pass, failed instead %s", err) |
||||
} |
||||
|
||||
// Test 6 validates OnEviction being called upon Delete is being invoked.
|
||||
testCase = testCases[5] |
||||
cache = New(testCase.cacheSize, testCase.expiry) |
||||
w, err = cache.Create("test", 5) |
||||
if err != nil { |
||||
t.Errorf("Test case 6 expected to pass, failed instead %s", err) |
||||
} |
||||
// Write '5' bytes.
|
||||
w.Write([]byte("Hello")) |
||||
// Close to successfully save into cache.
|
||||
if err = w.Close(); err != nil { |
||||
t.Errorf("Test case 6 expected to pass, failed instead %s", err) |
||||
} |
||||
var deleteKey string |
||||
cache.OnEviction = func(key string) { |
||||
deleteKey = key |
||||
} |
||||
// Delete the cache entry.
|
||||
cache.Delete("test") |
||||
if deleteKey != "test" { |
||||
t.Errorf("Test case 6 expected to pass, wanted \"test\", got %s", deleteKey) |
||||
} |
||||
} |
Loading…
Reference in new issue