From 179d2d7dac2fd7017b572562a408b0eb43641973 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 18 Oct 2015 14:47:56 -0700 Subject: [PATCH] Add initial cut of auto expiry of objects --- main.go | 6 ++-- pkg/fs/auto-expiry.go | 41 +++++++++++++++++++++++++++ routers.go | 3 ++ server-main.go | 66 +++++++++++++++++++++++++++++++------------ 4 files changed, 96 insertions(+), 20 deletions(-) create mode 100644 pkg/fs/auto-expiry.go diff --git a/main.go b/main.go index e6fd8cd68..a4dc84ffa 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,7 @@ import ( "os/user" "runtime" "strconv" + "time" "github.com/dustin/go-humanize" "github.com/minio/cli" @@ -34,8 +35,9 @@ type serverConfig struct { Anonymous bool // No signature turn off /// FS options - Path string // Path to export for cloud storage - MinFreeDisk int64 // Minimum free disk space for filesystem + Path string // Path to export for cloud storage + MinFreeDisk int64 // Minimum free disk space for filesystem + Expiry time.Duration // Set auto expiry for filesystem // TLS service TLS bool // TLS on when certs are specified diff --git a/pkg/fs/auto-expiry.go b/pkg/fs/auto-expiry.go new file mode 100644 index 000000000..081c18ad9 --- /dev/null +++ b/pkg/fs/auto-expiry.go @@ -0,0 +1,41 @@ +package fs + +import ( + "os" + "time" +) + +// AutoExpiryThread - auto expiry thread +func (fs Filesystem) AutoExpiryThread(expiry time.Duration) { + expireFiles := func(fp string, fl os.FileInfo, err error) error { + if fp == fs.path { + return nil + } + if fl.Mode().IsRegular() || fl.Mode()&os.ModeSymlink == os.ModeSymlink { + if time.Now().Sub(fl.ModTime()) > expiry { + if err := os.Remove(fp); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + } + return ErrDirNotEmpty + } + return nil + } + ticker := time.NewTicker(3 * time.Hour) + for { + select { + // TODO - add a way to stop the timer thread + case <-ticker.C: + err := WalkUnsorted(fs.path, expireFiles) + if err != nil { + if !os.IsNotExist(err) && err != ErrDirNotEmpty { + ticker.Stop() + return + } + } + } + } +} diff --git a/routers.go b/routers.go index e1987b292..5caec2f2b 100644 --- a/routers.go +++ b/routers.go @@ -59,6 +59,9 @@ func getNewCloudStorageAPI(conf serverConfig) CloudStorageAPI { fs.SetRootPath(conf.Path) fs.SetMinFreeDisk(conf.MinFreeDisk) + if conf.Expiry > 0 { + go fs.AutoExpiryThread(conf.Expiry) + } return CloudStorageAPI{ Filesystem: fs, Anonymous: conf.Anonymous, diff --git a/server-main.go b/server-main.go index eee03404b..1291aa81f 100644 --- a/server-main.go +++ b/server-main.go @@ -26,6 +26,7 @@ import ( "runtime" "strconv" "strings" + "time" "github.com/fatih/color" "github.com/minio/cli" @@ -55,6 +56,10 @@ EXAMPLES: 4. Start minio server with minimum free disk threshold to 5% $ minio {{.Name}} min-free-disk 5% /home/shared/Pictures + + 5. Start minio server with minimum free disk threshold to 15% with auto expiration set to 1h + $ minio {{.Name}} min-free-disk 15% expiry 1h /home/shared/Documents + `, } @@ -219,7 +224,7 @@ func checkServerSyntax(c *cli.Context) { if !c.Args().Present() || c.Args().First() == "help" { cli.ShowCommandHelpAndExit(c, "server", 1) } - if len(c.Args()) > 3 { + if len(c.Args()) > 5 { fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘mc server help’", nil) } path := strings.TrimSpace(c.Args().Last()) @@ -231,41 +236,66 @@ func checkServerSyntax(c *cli.Context) { func serverMain(c *cli.Context) { checkServerSyntax(c) - err := fetchAuth() - fatalIf(err.Trace(), "Failed to generate keys for minio.", nil) + perr := fetchAuth() + fatalIf(perr.Trace(), "Failed to generate keys for minio.", nil) - path := strings.TrimSpace(c.Args().Last()) - // Last argument is always path - if _, err := os.Stat(path); err != nil { - fatalIf(probe.NewError(err), "Unable to validate the path", nil) - } certFile := c.GlobalString("cert") keyFile := c.GlobalString("key") if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") { fatalIf(probe.NewError(errInvalidArgument), "Both certificate and key are required to enable https.", nil) } + var minFreeDisk int64 - // Only if args are greater than or equal to 2 verify if the proper variables are passed. - if len(c.Args()) >= 2 { - if c.Args().Get(0) == "min-free-disk" { - minFreeDisk, err = parsePercentToInt(c.Args().Get(1), 64) - fatalIf(err.Trace(c.Args().Get(2)), "Unable to parse minimum free disk parameter.", nil) - } - if c.Args().Get(0) != "min-free-disk" { - fatalIf(probe.NewError(errInvalidArgument), "Invalid arguments passed. ‘"+strings.Join(c.Args(), " ")+"’", nil) + minFreeDiskSet := false + + var expiration time.Duration + expirationSet := false + + args := c.Args() + for len(args) >= 2 { + switch args.First() { + case "min-free-disk": + if minFreeDiskSet { + fatalIf(probe.NewError(errInvalidArgument), "Minimum free disk should be set only once.", nil) + } + args = args.Tail() + var err *probe.Error + minFreeDisk, err = parsePercentToInt(args.First(), 64) + fatalIf(err.Trace(args.First()), "Invalid minium free disk size "+args.First()+" passed.", nil) + args = args.Tail() + minFreeDiskSet = true + case "expiry": + if expirationSet { + fatalIf(probe.NewError(errInvalidArgument), "Expiration should be set only once.", nil) + } + args = args.Tail() + var err error + expiration, err = time.ParseDuration(args.First()) + fatalIf(probe.NewError(err), "Invalid expiration time "+args.First()+" passed.", nil) + args = args.Tail() + expirationSet = true + default: + cli.ShowCommandHelpAndExit(c, "server", 1) // last argument is exit code } } + + path := strings.TrimSpace(c.Args().Last()) + // Last argument is always path + if _, err := os.Stat(path); err != nil { + fatalIf(probe.NewError(err), "Unable to validate the path", nil) + } tls := (certFile != "" && keyFile != "") apiServerConfig := serverConfig{ Address: c.GlobalString("address"), Anonymous: c.GlobalBool("anonymous"), Path: path, MinFreeDisk: minFreeDisk, + Expiry: expiration, TLS: tls, CertFile: certFile, KeyFile: keyFile, RateLimit: c.GlobalInt("ratelimit"), } - err = startServer(apiServerConfig) - errorIf(err.Trace(), "Failed to start the minio server.", nil) + perr = startServer(apiServerConfig) + errorIf(perr.Trace(), "Failed to start the minio server.", nil) }