Merge pull request #923 from harshavardhana/auto-expiry

Add initial cut of auto expiry of objects
master
Harshavardhana 9 years ago
commit d5e5b8777f
  1. 2
      main.go
  2. 41
      pkg/fs/auto-expiry.go
  3. 3
      routers.go
  4. 64
      server-main.go

@ -22,6 +22,7 @@ import (
"os/user" "os/user"
"runtime" "runtime"
"strconv" "strconv"
"time"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"github.com/minio/cli" "github.com/minio/cli"
@ -36,6 +37,7 @@ type serverConfig struct {
/// FS options /// FS options
Path string // Path to export for cloud storage Path string // Path to export for cloud storage
MinFreeDisk int64 // Minimum free disk space for filesystem MinFreeDisk int64 // Minimum free disk space for filesystem
Expiry time.Duration // Set auto expiry for filesystem
// TLS service // TLS service
TLS bool // TLS on when certs are specified TLS bool // TLS on when certs are specified

@ -0,0 +1,41 @@
package fs
import (
"os"
"time"
)
// AutoExpiryThread - auto expiry thread
func (fs Filesystem) AutoExpiryThread(expiry time.Duration) {
expireFiles := func(fp string, fl os.FileInfo, err error) error {
if fp == fs.path {
return nil
}
if fl.Mode().IsRegular() || fl.Mode()&os.ModeSymlink == os.ModeSymlink {
if time.Now().Sub(fl.ModTime()) > expiry {
if err := os.Remove(fp); err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
}
return ErrDirNotEmpty
}
return nil
}
ticker := time.NewTicker(3 * time.Hour)
for {
select {
// TODO - add a way to stop the timer thread
case <-ticker.C:
err := WalkUnsorted(fs.path, expireFiles)
if err != nil {
if !os.IsNotExist(err) && err != ErrDirNotEmpty {
ticker.Stop()
return
}
}
}
}
}

@ -59,6 +59,9 @@ func getNewCloudStorageAPI(conf serverConfig) CloudStorageAPI {
fs.SetRootPath(conf.Path) fs.SetRootPath(conf.Path)
fs.SetMinFreeDisk(conf.MinFreeDisk) fs.SetMinFreeDisk(conf.MinFreeDisk)
if conf.Expiry > 0 {
go fs.AutoExpiryThread(conf.Expiry)
}
return CloudStorageAPI{ return CloudStorageAPI{
Filesystem: fs, Filesystem: fs,
Anonymous: conf.Anonymous, Anonymous: conf.Anonymous,

@ -26,6 +26,7 @@ import (
"runtime" "runtime"
"strconv" "strconv"
"strings" "strings"
"time"
"github.com/fatih/color" "github.com/fatih/color"
"github.com/minio/cli" "github.com/minio/cli"
@ -55,6 +56,10 @@ EXAMPLES:
4. Start minio server with minimum free disk threshold to 5% 4. Start minio server with minimum free disk threshold to 5%
$ minio {{.Name}} min-free-disk 5% /home/shared/Pictures $ minio {{.Name}} min-free-disk 5% /home/shared/Pictures
5. Start minio server with minimum free disk threshold to 15% with auto expiration set to 1h
$ minio {{.Name}} min-free-disk 15% expiry 1h /home/shared/Documents
`, `,
} }
@ -219,7 +224,7 @@ func checkServerSyntax(c *cli.Context) {
if !c.Args().Present() || c.Args().First() == "help" { if !c.Args().Present() || c.Args().First() == "help" {
cli.ShowCommandHelpAndExit(c, "server", 1) cli.ShowCommandHelpAndExit(c, "server", 1)
} }
if len(c.Args()) > 3 { if len(c.Args()) > 5 {
fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘mc server help’", nil) fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘mc server help’", nil)
} }
path := strings.TrimSpace(c.Args().Last()) path := strings.TrimSpace(c.Args().Last())
@ -231,29 +236,53 @@ func checkServerSyntax(c *cli.Context) {
func serverMain(c *cli.Context) { func serverMain(c *cli.Context) {
checkServerSyntax(c) checkServerSyntax(c)
err := fetchAuth() perr := fetchAuth()
fatalIf(err.Trace(), "Failed to generate keys for minio.", nil) fatalIf(perr.Trace(), "Failed to generate keys for minio.", nil)
path := strings.TrimSpace(c.Args().Last())
// Last argument is always path
if _, err := os.Stat(path); err != nil {
fatalIf(probe.NewError(err), "Unable to validate the path", nil)
}
certFile := c.GlobalString("cert") certFile := c.GlobalString("cert")
keyFile := c.GlobalString("key") keyFile := c.GlobalString("key")
if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") { if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
fatalIf(probe.NewError(errInvalidArgument), "Both certificate and key are required to enable https.", nil) fatalIf(probe.NewError(errInvalidArgument), "Both certificate and key are required to enable https.", nil)
} }
var minFreeDisk int64 var minFreeDisk int64
// Only if args are greater than or equal to 2 verify if the proper variables are passed. minFreeDiskSet := false
if len(c.Args()) >= 2 {
if c.Args().Get(0) == "min-free-disk" { var expiration time.Duration
minFreeDisk, err = parsePercentToInt(c.Args().Get(1), 64) expirationSet := false
fatalIf(err.Trace(c.Args().Get(2)), "Unable to parse minimum free disk parameter.", nil)
args := c.Args()
for len(args) >= 2 {
switch args.First() {
case "min-free-disk":
if minFreeDiskSet {
fatalIf(probe.NewError(errInvalidArgument), "Minimum free disk should be set only once.", nil)
}
args = args.Tail()
var err *probe.Error
minFreeDisk, err = parsePercentToInt(args.First(), 64)
fatalIf(err.Trace(args.First()), "Invalid minium free disk size "+args.First()+" passed.", nil)
args = args.Tail()
minFreeDiskSet = true
case "expiry":
if expirationSet {
fatalIf(probe.NewError(errInvalidArgument), "Expiration should be set only once.", nil)
}
args = args.Tail()
var err error
expiration, err = time.ParseDuration(args.First())
fatalIf(probe.NewError(err), "Invalid expiration time "+args.First()+" passed.", nil)
args = args.Tail()
expirationSet = true
default:
cli.ShowCommandHelpAndExit(c, "server", 1) // last argument is exit code
} }
if c.Args().Get(0) != "min-free-disk" {
fatalIf(probe.NewError(errInvalidArgument), "Invalid arguments passed. ‘"+strings.Join(c.Args(), " ")+"’", nil)
} }
path := strings.TrimSpace(c.Args().Last())
// Last argument is always path
if _, err := os.Stat(path); err != nil {
fatalIf(probe.NewError(err), "Unable to validate the path", nil)
} }
tls := (certFile != "" && keyFile != "") tls := (certFile != "" && keyFile != "")
apiServerConfig := serverConfig{ apiServerConfig := serverConfig{
@ -261,11 +290,12 @@ func serverMain(c *cli.Context) {
Anonymous: c.GlobalBool("anonymous"), Anonymous: c.GlobalBool("anonymous"),
Path: path, Path: path,
MinFreeDisk: minFreeDisk, MinFreeDisk: minFreeDisk,
Expiry: expiration,
TLS: tls, TLS: tls,
CertFile: certFile, CertFile: certFile,
KeyFile: keyFile, KeyFile: keyFile,
RateLimit: c.GlobalInt("ratelimit"), RateLimit: c.GlobalInt("ratelimit"),
} }
err = startServer(apiServerConfig) perr = startServer(apiServerConfig)
errorIf(err.Trace(), "Failed to start the minio server.", nil) errorIf(perr.Trace(), "Failed to start the minio server.", nil)
} }

Loading…
Cancel
Save