From 1da362538bd704978662afcf373050a6a5457bec Mon Sep 17 00:00:00 2001 From: poornas Date: Tue, 3 Jul 2018 16:54:10 -0700 Subject: [PATCH] cache: allow ellipse style entries for MINIO_CACHE_DRIVES (#6088) Fixes #5863 --- cmd/disk-cache-config.go | 35 +++++++++++++++++++++++++++++++++- cmd/disk-cache-config_test.go | 20 +++++++++++++++++++ docs/config/README.md | 1 + docs/config/config.sample.json | 9 ++++++--- docs/disk-caching/DESIGN.md | 4 ++-- docs/disk-caching/README.md | 7 ++++--- 6 files changed, 67 insertions(+), 9 deletions(-) diff --git a/cmd/disk-cache-config.go b/cmd/disk-cache-config.go index c7ebbd26f..60ab7673f 100644 --- a/cmd/disk-cache-config.go +++ b/cmd/disk-cache-config.go @@ -19,6 +19,9 @@ package cmd import ( "encoding/json" "path/filepath" + "strings" + + "github.com/minio/minio/pkg/ellipses" ) // CacheConfig represents cache config settings @@ -52,12 +55,42 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) { // Parses given cacheDrivesEnv and returns a list of cache drives. func parseCacheDrives(drives []string) ([]string, error) { + if len(drives) == 0 { + return drives, nil + } + var endpoints []string for _, d := range drives { + if ellipses.HasEllipses(d) { + s, err := parseCacheDrivePaths(d) + if err != nil { + return nil, err + } + endpoints = append(endpoints, s...) + } else { + endpoints = append(endpoints, d) + } + } + + for _, d := range endpoints { if !filepath.IsAbs(d) { return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d) } } - return drives, nil + return endpoints, nil +} + +// Parses all arguments and returns a slice of drive paths following the ellipses pattern. +func parseCacheDrivePaths(arg string) (ep []string, err error) { + patterns, perr := ellipses.FindEllipsesPatterns(arg) + if perr != nil { + return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error()) + } + + for _, lbls := range patterns.Expand() { + ep = append(ep, strings.Join(lbls, "")) + } + + return ep, nil } // Parses given cacheExcludesEnv and returns a list of cache exclude patterns. diff --git a/cmd/disk-cache-config_test.go b/cmd/disk-cache-config_test.go index 43d461ca9..15aa58319 100644 --- a/cmd/disk-cache-config_test.go +++ b/cmd/disk-cache-config_test.go @@ -41,12 +41,32 @@ func TestParseCacheDrives(t *testing.T) { expectedPatterns []string success bool }{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true}) + testCases = append(testCases, struct { + driveStr string + expectedPatterns []string + success bool + }{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true}) + testCases = append(testCases, struct { + driveStr string + expectedPatterns []string + success bool + }{"C:/home/drive{1..3}", []string{}, false}) } else { testCases = append(testCases, struct { driveStr string expectedPatterns []string success bool }{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true}) + testCases = append(testCases, struct { + driveStr string + expectedPatterns []string + success bool + }{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true}) + testCases = append(testCases, struct { + driveStr string + expectedPatterns []string + success bool + }{"/home/drive{1..3}", []string{}, false}) } for i, testCase := range testCases { drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter)) diff --git a/docs/config/README.md b/docs/config/README.md index 3e4870644..f39a425cf 100644 --- a/docs/config/README.md +++ b/docs/config/README.md @@ -110,6 +110,7 @@ By default, parity for objects with standard storage class is set to `N/2`, and |``drives``| _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled| |``exclude`` | _[]string_ | List of wildcard patterns for prefixes to exclude from cache | |``expiry`` | _int_ | Days to cache expiry | +|``maxuse`` | _int_ | Percentage of disk available to cache | #### Notify |Field|Type|Description| diff --git a/docs/config/config.sample.json b/docs/config/config.sample.json index 3b0c96911..8ae35a4e1 100644 --- a/docs/config/config.sample.json +++ b/docs/config/config.sample.json @@ -1,11 +1,12 @@ { - "version": "24", + "version": "26", "credential": { "accessKey": "USWUXHGYZQYFYFFIT3RE", "secretKey": "MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03" }, "region": "us-east-1", "browser": "on", + "worm": "off", "domain": "", "storageclass": { "standard": "", @@ -14,11 +15,12 @@ "cache": { "drives": [], "expiry": 90, - "exclude": [] + "exclude": [], + "maxuse": 80 }, "usage": { "interval": "3h" - } + }, "notify": { "amqp": { "1": { @@ -124,3 +126,4 @@ } } } + diff --git a/docs/disk-caching/DESIGN.md b/docs/disk-caching/DESIGN.md index 2e7b9f3ff..80ecfeaa6 100644 --- a/docs/disk-caching/DESIGN.md +++ b/docs/disk-caching/DESIGN.md @@ -16,10 +16,10 @@ minio server -h ... ... - 7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/drive3', + 7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24', exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension with expiry upto 40 days. - $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3" + $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/export{1..24}" $ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf" $ export MINIO_CACHE_EXPIRY=40 $ export MINIO_CACHE_MAXUSE=80 diff --git a/docs/disk-caching/README.md b/docs/disk-caching/README.md index 778105734..c348c049f 100644 --- a/docs/disk-caching/README.md +++ b/docs/disk-caching/README.md @@ -17,14 +17,15 @@ Disk caching can be enabled by updating the `cache` config settings for Minio se "cache": { "drives": ["/mnt/drive1", "/mnt/drive2", "/mnt/drive3"], "expiry": 90, - "exclude": ["*.pdf","mybucket/*"] + "exclude": ["*.pdf","mybucket/*"], + "maxuse" : 70, }, ``` -The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` and `/mnt/drive3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup. +The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup. Cache max usage is restricted to 80% of disk capacity in this example. ```bash -export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3" +export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}" export MINIO_CACHE_EXPIRY=90 export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*" export MINIO_CACHE_MAXUSE=80