Allow caching only in gateway mode. (#8232)

This PR changes cache on PUT behavior to background fill the cache
after PutObject completes. This will avoid concurrency issues as in #8219.

Added cleanup of partially filled cache to prevent cache corruption
- Fixes #8208
master
poornas 5 years ago committed by kannappanr
parent 208efb843b
commit 76df027264
  1. 36
      cmd/disk-cache-backend.go
  2. 60
      cmd/disk-cache.go
  3. 16
      cmd/server-main.go
  4. 13
      docs/disk-caching/DESIGN.md
  5. 29
      docs/disk-caching/README.md

@ -333,28 +333,6 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
return err return err
} }
// updates ETag in cache metadata file to the backend ETag.
func (c *diskCache) updateETag(ctx context.Context, bucket, object string, etag string) error {
metaPath := path.Join(getCacheSHADir(c.dir, bucket, object), cacheMetaJSONFile)
f, err := os.OpenFile(metaPath, os.O_RDWR, 0)
if err != nil {
return err
}
defer f.Close()
meta := &cacheMeta{Version: cacheMetaVersion}
if err := jsonLoad(f, meta); err != nil {
return err
}
meta.Meta["etag"] = etag
jsonData, err := json.Marshal(meta)
if err != nil {
return err
}
_, err = f.Write(jsonData)
return err
}
// Backend metadata could have changed through server side copy - reset cache metadata if that is the case // Backend metadata could have changed through server side copy - reset cache metadata if that is the case
func (c *diskCache) updateMetadataIfChanged(ctx context.Context, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error { func (c *diskCache) updateMetadataIfChanged(ctx context.Context, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error {
@ -418,7 +396,7 @@ func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, re
bufp := c.pool.Get().(*[]byte) bufp := c.pool.Get().(*[]byte)
defer c.pool.Put(bufp) defer c.pool.Put(bufp)
var n int var n, n2 int
for { for {
n, err = io.ReadFull(reader, *bufp) n, err = io.ReadFull(reader, *bufp)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
@ -437,10 +415,10 @@ func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, re
if _, err = f.Write(hashBytes); err != nil { if _, err = f.Write(hashBytes); err != nil {
return 0, err return 0, err
} }
if _, err = f.Write((*bufp)[:n]); err != nil { if n2, err = f.Write((*bufp)[:n]); err != nil {
return 0, err return 0, err
} }
bytesWritten += int64(n) bytesWritten += int64(n2)
if eof { if eof {
break break
} }
@ -521,9 +499,11 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
c.setOnline(false) c.setOnline(false)
} }
if err != nil { if err != nil {
removeAll(cachePath)
return err return err
} }
if actualSize != uint64(n) { if actualSize != uint64(n) {
removeAll(cachePath)
return IncompleteBody{} return IncompleteBody{}
} }
return c.saveMetadata(ctx, bucket, object, metadata, n) return c.saveMetadata(ctx, bucket, object, metadata, n)
@ -648,7 +628,11 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
filePath := path.Join(cacheObjPath, cacheDataFile) filePath := path.Join(cacheObjPath, cacheDataFile)
pr, pw := io.Pipe() pr, pw := io.Pipe()
go func() { go func() {
pw.CloseWithError(c.bitrotReadFromCache(ctx, filePath, off, length, pw)) err := c.bitrotReadFromCache(ctx, filePath, off, length, pw)
if err != nil {
removeAll(cacheObjPath)
}
pw.CloseWithError(err)
}() }()
// Cleanup function to cause the go routine above to exit, in // Cleanup function to cause the go routine above to exit, in
// case of incomplete read. // case of incomplete read.

@ -14,7 +14,6 @@ import (
"github.com/djherbis/atime" "github.com/djherbis/atime"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/hash"
"github.com/minio/minio/pkg/wildcard" "github.com/minio/minio/pkg/wildcard"
) )
@ -489,7 +488,6 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
// PutObject - caches the uploaded object for single Put operations // PutObject - caches the uploaded object for single Put operations
func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) { func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
putObjectFn := c.PutObjectFn putObjectFn := c.PutObjectFn
data := r.rawReader
dcache, err := c.getCacheToLoc(ctx, bucket, object) dcache, err := c.getCacheToLoc(ctx, bucket, object)
if err != nil { if err != nil {
// disk cache could not be located,execute backend call. // disk cache could not be located,execute backend call.
@ -513,52 +511,24 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
dcache.Delete(ctx, bucket, object) dcache.Delete(ctx, bucket, object)
return putObjectFn(ctx, bucket, object, r, opts) return putObjectFn(ctx, bucket, object, r, opts)
} }
// Initialize pipe to stream data to backend
pipeReader, pipeWriter := io.Pipe()
hashReader, err := hash.NewReader(pipeReader, size, "", "", data.ActualSize(), globalCLIContext.StrictS3Compat)
if err != nil {
return ObjectInfo{}, err
}
// Initialize pipe to stream data to cache
rPipe, wPipe := io.Pipe()
oinfoCh := make(chan ObjectInfo) objInfo, err = putObjectFn(ctx, bucket, object, r, opts)
errCh := make(chan error)
go func() {
oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), opts)
if perr != nil {
pipeWriter.CloseWithError(perr)
wPipe.CloseWithError(perr)
close(oinfoCh)
errCh <- perr
return
}
close(errCh)
oinfoCh <- oinfo
}()
// get a namespace lock on cache until cache is filled.
cLock := c.nsMutex.NewNSLock(ctx, bucket, object)
if err := cLock.GetLock(globalObjectTimeout); err != nil {
return ObjectInfo{}, err
}
defer cLock.Unlock()
go func() {
if err = dcache.Put(ctx, bucket, object, rPipe, data.Size(), opts); err != nil {
wPipe.CloseWithError(err)
return
}
}()
mwriter := io.MultiWriter(pipeWriter, wPipe) if err == nil {
_, err = io.Copy(mwriter, data) go func() {
if err != nil { // fill cache in the background
err = <-errCh bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
return objInfo, err if bErr != nil {
return
}
defer bReader.Close()
oi, err := c.stat(ctx, dcache, bucket, object)
// avoid cache overwrite if another background routine filled cache
if err != nil || oi.ETag != bReader.ObjInfo.ETag {
c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
}
}()
} }
pipeWriter.Close()
wPipe.Close()
objInfo = <-oinfoCh
dcache.updateETag(ctx, bucket, object, objInfo.ETag)
return objInfo, err return objInfo, err

@ -120,14 +120,7 @@ EXAMPLES:
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage
{{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32} {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32}
6. Start minio server with edge caching enabled. 6. Start minio server with KMS enabled.
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
{{.Prompt}} {{.HelpName}} /home/shared
7. Start minio server with KMS enabled.
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_ID{{.AssignmentOperator}}9b56cc08-8258-45d5-24a3-679876769126 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_ID{{.AssignmentOperator}}9b56cc08-8258-45d5-24a3-679876769126
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_SECRET{{.AssignmentOperator}}4e30c52f-13e4-a6f5-0763-d50e8cb4321f {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_SECRET{{.AssignmentOperator}}4e30c52f-13e4-a6f5-0763-d50e8cb4321f
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_ENDPOINT{{.AssignmentOperator}}https://vault-endpoint-ip:8200 {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_ENDPOINT{{.AssignmentOperator}}https://vault-endpoint-ip:8200
@ -339,13 +332,6 @@ func serverMain(ctx *cli.Context) {
// Load logger subsystem // Load logger subsystem
loadLoggers() loadLoggers()
var cacheConfig = globalServerConfig.GetCacheConfig()
if len(cacheConfig.Drives) > 0 {
// initialize the new disk cache objects.
globalCacheObjectAPI, err = newServerCacheObjects(context.Background(), cacheConfig)
logger.FatalIf(err, "Unable to initialize disk caching")
}
// Create new IAM system. // Create new IAM system.
globalIAMSys = NewIAMSys() globalIAMSys = NewIAMSys()
if err = globalIAMSys.Init(newObject); err != nil { if err = globalIAMSys.Init(newObject); err != nil {

@ -5,7 +5,7 @@ This document explains some basic assumptions and design approach, limits of the
## Command-line ## Command-line
``` ```
minio server -h minio gateway <name> -h
... ...
... ...
CACHE: CACHE:
@ -16,17 +16,18 @@ minio server -h
... ...
... ...
7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24', 7. Start MinIO gateway to s3 with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24',
exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension
with expiry up to 40 days. with expiry up to 40 days.
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/export{1..24}" $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/export{1..24}"
$ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf" $ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf"
$ export MINIO_CACHE_EXPIRY=40 $ export MINIO_CACHE_EXPIRY=40
$ export MINIO_CACHE_MAXUSE=80 $ export MINIO_CACHE_MAXUSE=80
$ minio server /home/shared $ minio gateway s3
``` ```
## Assumptions ## Assumptions
- Disk cache size defaults to 80% of your drive capacity. - Disk cache size defaults to 80% of your drive capacity.
- The cache drives are required to be a filesystem mount point with [`atime`](http://kerolasa.github.io/filetimes.html) support to be enabled on the drive. Alternatively writable directories with atime support can be specified in MINIO_CACHE_DRIVES - The cache drives are required to be a filesystem mount point with [`atime`](http://kerolasa.github.io/filetimes.html) support to be enabled on the drive. Alternatively writable directories with atime support can be specified in MINIO_CACHE_DRIVES
- Expiration of each cached entry takes user provided expiry as a hint, and defaults to 90 days if not provided. - Expiration of each cached entry takes user provided expiry as a hint, and defaults to 90 days if not provided.
@ -34,6 +35,7 @@ minio server -h
- An object is only cached when drive has sufficient disk space. - An object is only cached when drive has sufficient disk space.
## Behavior ## Behavior
Disk caching caches objects for **downloaded** objects i.e Disk caching caches objects for **downloaded** objects i.e
- Caches new objects for entries not found in cache while downloading. Otherwise serves from the cache. - Caches new objects for entries not found in cache while downloading. Otherwise serves from the cache.
@ -50,9 +52,10 @@ master key to automatically encrypt all cached content.
> NOTE: Expiration happens automatically based on the configured interval as explained above, frequently accessed objects stay alive in cache for a significantly longer time. > NOTE: Expiration happens automatically based on the configured interval as explained above, frequently accessed objects stay alive in cache for a significantly longer time.
### Crash Recovery ### Crash Recovery
Upon restart of minio server after a running minio process is killed or crashes, disk caching resumes automatically. The garbage collection cycle resumes and any previously cached entries are served from cache.
Upon restart of minio gateway after a running minio process is killed or crashes, disk caching resumes automatically. The garbage collection cycle resumes and any previously cached entries are served from cache.
## Limits ## Limits
- Bucket policies are not cached, so anonymous operations are not supported when backend is offline. - Bucket policies are not cached, so anonymous operations are not supported when backend is offline.
- Objects are distributed using deterministic hashing among the list of configured cache drives. If one or more drives go offline, or cache drive configuration is altered in any way, performance may degrade to a linear lookup time depending on the number of disks in cache. - Objects are distributed using deterministic hashing among the list of configured cache drives. If one or more drives go offline, or cache drive configuration is altered in any way, performance may degrade to a linear lookup time depending on the number of disks in cache.

@ -8,42 +8,29 @@ Disk caching feature here refers to the use of caching disks to store content cl
## Get started ## Get started
### 1. Prerequisites ### 1. Prerequisites
Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide). Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide).
### 2. Run MinIO with cache ### 2. Run MinIO gateway with cache
Disk caching can be enabled by updating the `cache` config settings for MinIO server. Config `cache` settings takes the mounted drive(s) or directory paths, cache expiry duration (in days) and any wildcard patterns to exclude from being cached.
```json Disk caching can be enabled by setting the `cache` environment variables for MinIO gateway . `cache` environment variables takes the mounted drive(s) or directory paths, cache expiry duration (in days) and any wildcard patterns to exclude from being cached.
"cache": {
"drives": ["/mnt/drive1", "/mnt/drive2", "/mnt/drive3"],
"expiry": 90,
"exclude": ["*.pdf","mybucket/*"],
"maxuse" : 70,
},
```
To update the configuration, use `mc admin config get` command to get the current configuration file for the minio cluster in json format, and save it locally. Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry up to 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a s3 gateway setup. Cache max usage is restricted to 80% of disk capacity in this example.
```sh
$ mc admin config get myminio/ > /tmp/myconfig
```
After updating the cache configuration in /tmp/myconfig , use `mc admin config set` command to update the configuration for the cluster.Restart the MinIO server to put the changes into effect.
```sh
$ mc admin config set myminio < /tmp/myconfig
```
The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for MinIO server. Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry up to 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup. Cache max usage is restricted to 80% of disk capacity in this example.
```bash ```bash
export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}" export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}"
export MINIO_CACHE_EXPIRY=90 export MINIO_CACHE_EXPIRY=90
export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*" export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*"
export MINIO_CACHE_MAXUSE=80 export MINIO_CACHE_MAXUSE=80
minio server /export{1...24} minio gateway s3
``` ```
### 3. Test your setup ### 3. Test your setup
To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/docs/minio-client-quickstart-guide). You’ll see the uploaded files are accessible from the all the MinIO endpoints.
To test this setup, access the MinIO gateway via browser or [`mc`](https://docs.min.io/docs/minio-client-quickstart-guide). You’ll see the uploaded files are accessible from all the MinIO endpoints.
# Explore Further # Explore Further
- [Disk cache design](https://github.com/minio/minio/blob/master/docs/disk-caching/DESIGN.md) - [Disk cache design](https://github.com/minio/minio/blob/master/docs/disk-caching/DESIGN.md)
- [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide) - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
- [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio) - [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)

Loading…
Cancel
Save