cache: allow ellipse style entries for MINIO_CACHE_DRIVES (#6088)

Fixes #5863
master
poornas 6 years ago committed by Harshavardhana
parent e40a5e05e1
commit 1da362538b
  1. 35
      cmd/disk-cache-config.go
  2. 20
      cmd/disk-cache-config_test.go
  3. 1
      docs/config/README.md
  4. 9
      docs/config/config.sample.json
  5. 4
      docs/disk-caching/DESIGN.md
  6. 7
      docs/disk-caching/README.md

@ -19,6 +19,9 @@ package cmd
import (
"encoding/json"
"path/filepath"
"strings"
"github.com/minio/minio/pkg/ellipses"
)
// CacheConfig represents cache config settings
@ -52,12 +55,42 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
// Parses given cacheDrivesEnv and returns a list of cache drives.
func parseCacheDrives(drives []string) ([]string, error) {
if len(drives) == 0 {
return drives, nil
}
var endpoints []string
for _, d := range drives {
if ellipses.HasEllipses(d) {
s, err := parseCacheDrivePaths(d)
if err != nil {
return nil, err
}
endpoints = append(endpoints, s...)
} else {
endpoints = append(endpoints, d)
}
}
for _, d := range endpoints {
if !filepath.IsAbs(d) {
return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
}
}
return drives, nil
return endpoints, nil
}
// Parses all arguments and returns a slice of drive paths following the ellipses pattern.
func parseCacheDrivePaths(arg string) (ep []string, err error) {
patterns, perr := ellipses.FindEllipsesPatterns(arg)
if perr != nil {
return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
}
for _, lbls := range patterns.Expand() {
ep = append(ep, strings.Join(lbls, ""))
}
return ep, nil
}
// Parses given cacheExcludesEnv and returns a list of cache exclude patterns.

@ -41,12 +41,32 @@ func TestParseCacheDrives(t *testing.T) {
expectedPatterns []string
success bool
}{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"C:/home/drive{1..3}", []string{}, false})
} else {
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
testCases = append(testCases, struct {
driveStr string
expectedPatterns []string
success bool
}{"/home/drive{1..3}", []string{}, false})
}
for i, testCase := range testCases {
drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))

@ -110,6 +110,7 @@ By default, parity for objects with standard storage class is set to `N/2`, and
|``drives``| _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled|
|``exclude`` | _[]string_ | List of wildcard patterns for prefixes to exclude from cache |
|``expiry`` | _int_ | Days to cache expiry |
|``maxuse`` | _int_ | Percentage of disk available to cache |
#### Notify
|Field|Type|Description|

@ -1,11 +1,12 @@
{
"version": "24",
"version": "26",
"credential": {
"accessKey": "USWUXHGYZQYFYFFIT3RE",
"secretKey": "MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03"
},
"region": "us-east-1",
"browser": "on",
"worm": "off",
"domain": "",
"storageclass": {
"standard": "",
@ -14,11 +15,12 @@
"cache": {
"drives": [],
"expiry": 90,
"exclude": []
"exclude": [],
"maxuse": 80
},
"usage": {
"interval": "3h"
}
},
"notify": {
"amqp": {
"1": {
@ -124,3 +126,4 @@
}
}
}

@ -16,10 +16,10 @@ minio server -h
...
...
7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/drive3',
7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24',
exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension
with expiry upto 40 days.
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/export{1..24}"
$ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf"
$ export MINIO_CACHE_EXPIRY=40
$ export MINIO_CACHE_MAXUSE=80

@ -17,14 +17,15 @@ Disk caching can be enabled by updating the `cache` config settings for Minio se
"cache": {
"drives": ["/mnt/drive1", "/mnt/drive2", "/mnt/drive3"],
"expiry": 90,
"exclude": ["*.pdf","mybucket/*"]
"exclude": ["*.pdf","mybucket/*"],
"maxuse" : 70,
},
```
The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` and `/mnt/drive3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup.
The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup. Cache max usage is restricted to 80% of disk capacity in this example.
```bash
export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}"
export MINIO_CACHE_EXPIRY=90
export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*"
export MINIO_CACHE_MAXUSE=80

Loading…
Cancel
Save