Merge pull request #998 from harshavardhana/config-folder

minio: Add config-folder option.

config-logger-main.go

@@ -106,10 +106,6 @@ func mainConfigLogger(ctx *cli.Context) {
 		}
 	}
 	if ctx.Args().Get(0) == "list" {
-		if globalJSONFlag {
-			Println(conf.JSON())
-			return
-		}
 		Println(conf)
 	}
 }

config-version-main.go

@@ -16,12 +16,7 @@
 
 package main
 
-import (
-	"encoding/json"
-
-	"github.com/minio/cli"
-	"github.com/minio/minio-xl/pkg/probe"
-)
+import "github.com/minio/cli"
 
 // Print config version.
 var configVersionCmd = cli.Command{
@@ -48,15 +43,5 @@ func mainConfigVersion(ctx *cli.Context) {
 	// convert interface{} back to its original struct
 	newConf := config
 	type Version string
-	if globalJSONFlag {
-		tB, e := json.Marshal(
-			struct {
-				Version Version `json:"version"`
-			}{Version: Version(newConf.Version)},
-		)
-		fatalIf(probe.NewError(e), "Unable to construct version string.", nil)
-		Println(string(tB))
-		return
-	}
 	Println(newConf.Version)
 }

flags.go

@@ -22,6 +22,12 @@ import "github.com/minio/cli"
 var flags = []cli.Flag{}
 
 var (
+	configFolderFlag = cli.StringFlag{
+		Name:  "config-folder, C",
+		Value: mustGetConfigPath(),
+		Usage: "Path to configuration folder.",
+	}
+
 	addressFlag = cli.StringFlag{
 		Name:  "address",
 		Value: ":9000",

globals.go

@@ -1,21 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-var (
-	globalJSONFlag = false // Json flag set via command line
-)

main.go

@@ -19,6 +19,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"os/user"
 	"runtime"
 	"strconv"
 
@@ -55,9 +56,9 @@ func init() {
 	checkGolangRuntimeVersion()
 
 	// Check for the environment early on and gracefuly report.
-	_, err := userCurrent()
+	_, err := user.Current()
 	if err != nil {
-		Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
+
 	}
 
 	if os.Getenv("DOCKERIMAGE") == "1" {
@@ -118,6 +119,7 @@ func registerApp() *cli.App {
 	registerCommand(updateCmd)
 
 	// register all flags
+	registerFlag(configFolderFlag)
 	registerFlag(addressFlag)
 	registerFlag(accessLogFlag)
 	registerFlag(rateLimitFlag)
@@ -146,10 +148,19 @@ func registerApp() *cli.App {
 		}
 		Fatalln(msg)
 	}
-
 	return app
 }
 
+func checkMainSyntax(c *cli.Context) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
+	}
+	if configPath == "" {
+		Fatalf("Config folder cannot be empty, please specify --config-folder <foldername>.")
+	}
+}
+
 func main() {
 	probe.Init() // Set project's root source path.
 	probe.SetAppInfo("Release-Tag", minioReleaseTag)
@@ -160,8 +171,15 @@ func main() {
 
 	app := registerApp()
 	app.Before = func(c *cli.Context) error {
-		globalJSONFlag = c.GlobalBool("json")
+		// Sets new config folder.
+		setGlobalConfigPath(c.GlobalString("config-folder"))
+
+		// Valid input arguments to main.
+		checkMainSyntax(c)
+
+		// Migrate any old version of config / state files to newer format.
 		migrate()
+
 		return nil
 	}
 	app.ExtraInfo = func() map[string]string {

pkg/contentdb/contentdb.go

@@ -102,7 +102,7 @@ func Init() error {
 // Lookup returns matching content-type for known types of file extensions.
 func Lookup(extension string) (contentType string, e error) {
 	if !isInitialized {
-		return "", errors.New("contentdb is not initialized.")
+		return "", errors.New("contentdb is not initialized")
 	}
 
 	return extDB[extension], e

pkg/contentdb/db.go

@@ -184,8 +184,8 @@ type bintree struct {
 }
 
 var _bintree = &bintree{nil, map[string]*bintree{
-	"db": &bintree{nil, map[string]*bintree{
-		"db.json": &bintree{dbDbJSON, map[string]*bintree{}},
+	"db": {nil, map[string]*bintree{
+		"db.json": {dbDbJSON, map[string]*bintree{}},
 	}},
 }}
 

pkg/fs/config.go

@@ -17,52 +17,22 @@
 package fs
 
 import (
-	"os"
-	"os/user"
 	"path/filepath"
-	"runtime"
-	"strconv"
 
 	"github.com/minio/minio-xl/pkg/probe"
 	"github.com/minio/minio-xl/pkg/quick"
+	"github.com/minio/minio/pkg/user"
 )
 
-// Workaround for docker images with fully static binary and 32bit linux operating systems.
-// For static binaries NSS library will not be a part of the static binary hence user.Current() fails.
-// For 32bit linux CGO is not enabled so it will not provide linux specific codebase.
-func userCurrent() (*user.User, *probe.Error) {
-	if os.Getenv("DOCKERIMAGE") == "1" {
-		wd, err := os.Getwd()
-		if err != nil {
-			return nil, probe.NewError(err)
-		}
-		return &user.User{Uid: "0", Gid: "0", Username: "root", Name: "root", HomeDir: wd}, nil
-	}
-	if runtime.GOARCH == "386" && runtime.GOOS == "linux" {
-		return &user.User{
-			Uid:      strconv.Itoa(os.Getuid()),
-			Gid:      strconv.Itoa(os.Getgid()),
-			Username: os.Getenv("USER"),
-			Name:     os.Getenv("USER"),
-			HomeDir:  os.Getenv("HOME"),
-		}, nil
-	}
-	user, err := user.Current()
-	if err != nil {
-		return nil, probe.NewError(err)
-	}
-	return user, nil
-}
-
 func getFSBucketsConfigPath() (string, *probe.Error) {
 	if customBucketsConfigPath != "" {
 		return customBucketsConfigPath, nil
 	}
-	u, err := userCurrent()
-	if err != nil {
-		return "", err.Trace()
+	homeDir, e := user.HomeDir()
+	if e != nil {
+		return "", probe.NewError(e)
 	}
-	fsBucketsConfigPath := filepath.Join(u.HomeDir, ".minio", "buckets.json")
+	fsBucketsConfigPath := filepath.Join(homeDir, ".minio", "$buckets.json")
 	return fsBucketsConfigPath, nil
 }
 
@@ -70,29 +40,29 @@ func getFSMultipartsSessionConfigPath() (string, *probe.Error) {
 	if customMultipartsConfigPath != "" {
 		return customMultipartsConfigPath, nil
 	}
-	u, err := userCurrent()
-	if err != nil {
-		return "", err.Trace()
+	homeDir, e := user.HomeDir()
+	if e != nil {
+		return "", probe.NewError(e)
 	}
-	fsMultipartsConfigPath := filepath.Join(u.HomeDir, ".minio", "multiparts-session.json")
+	fsMultipartsConfigPath := filepath.Join(homeDir, ".minio", "$multiparts-session.json")
 	return fsMultipartsConfigPath, nil
 }
 
 // internal variable only accessed via get/set methods
 var customMultipartsConfigPath, customBucketsConfigPath string
 
-// SetFSBucketsConfigPath - set custom fs buckets config path
-func SetFSBucketsConfigPath(configPath string) {
+// setFSBucketsConfigPath - set custom fs buckets config path
+func setFSBucketsConfigPath(configPath string) {
 	customBucketsConfigPath = configPath
 }
 
 // SetFSMultipartsConfigPath - set custom multiparts session config path
-func SetFSMultipartsConfigPath(configPath string) {
+func setFSMultipartsConfigPath(configPath string) {
 	customMultipartsConfigPath = configPath
 }
 
-// SaveMultipartsSession - save multiparts
-func SaveMultipartsSession(multiparts *Multiparts) *probe.Error {
+// saveMultipartsSession - save multiparts
+func saveMultipartsSession(multiparts *Multiparts) *probe.Error {
 	fsMultipartsConfigPath, err := getFSMultipartsSessionConfigPath()
 	if err != nil {
 		return err.Trace()
@@ -107,8 +77,8 @@ func SaveMultipartsSession(multiparts *Multiparts) *probe.Error {
 	return nil
 }
 
-// SaveBucketsMetadata - save metadata of all buckets
-func SaveBucketsMetadata(buckets *Buckets) *probe.Error {
+// saveBucketsMetadata - save metadata of all buckets
+func saveBucketsMetadata(buckets *Buckets) *probe.Error {
 	fsBucketsConfigPath, err := getFSBucketsConfigPath()
 	if err != nil {
 		return err.Trace()

pkg/fs/fs-bucket.go

@@ -54,7 +54,7 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
 		return probe.NewError(err)
 	}
 	delete(fs.buckets.Metadata, bucket)
-	if err := SaveBucketsMetadata(fs.buckets); err != nil {
+	if err := saveBucketsMetadata(fs.buckets); err != nil {
 		return err.Trace(bucket)
 	}
 	return nil
@@ -147,7 +147,7 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
 	bucketMetadata.Created = fi.ModTime()
 	bucketMetadata.ACL = BucketACL(acl)
 	fs.buckets.Metadata[bucket] = bucketMetadata
-	if err := SaveBucketsMetadata(fs.buckets); err != nil {
+	if err := saveBucketsMetadata(fs.buckets); err != nil {
 		return err.Trace(bucket)
 	}
 	return nil
@@ -211,7 +211,7 @@ func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string
 	}
 	bucketMetadata.ACL = BucketACL(acl)
 	fs.buckets.Metadata[bucket] = bucketMetadata
-	if err := SaveBucketsMetadata(fs.buckets); err != nil {
+	if err := saveBucketsMetadata(fs.buckets); err != nil {
 		return err.Trace(bucket)
 	}
 	return nil

pkg/fs/fs-multipart.go

@@ -201,7 +201,7 @@ func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.E
 	if err != nil {
 		return "", probe.NewError(err)
 	}
-	if err := SaveMultipartsSession(fs.multiparts); err != nil {
+	if err := saveMultipartsSession(fs.multiparts); err != nil {
 		return "", err.Trace()
 	}
 	return uploadID, nil
@@ -419,7 +419,7 @@ func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, da
 		file.CloseAndPurge()
 		return ObjectMetadata{}, probe.NewError(err)
 	}
-	if err := SaveMultipartsSession(fs.multiparts); err != nil {
+	if err := saveMultipartsSession(fs.multiparts); err != nil {
 		file.CloseAndPurge()
 		return ObjectMetadata{}, err.Trace()
 	}

pkg/fs/fs.go

@@ -18,6 +18,7 @@ package fs
 
 import (
 	"os"
+	"path/filepath"
 	"sync"
 	"time"
 
@@ -54,7 +55,10 @@ type Multiparts struct {
 }
 
 // New instantiate a new donut
-func New() (Filesystem, *probe.Error) {
+func New(rootPath string) (Filesystem, *probe.Error) {
+	setFSBucketsConfigPath(filepath.Join(rootPath, "$buckets.json"))
+	setFSMultipartsConfigPath(filepath.Join(rootPath, "$multiparts-session.json"))
+
 	var err *probe.Error
 	// load multiparts session from disk
 	var multiparts *Multiparts
@@ -65,7 +69,7 @@ func New() (Filesystem, *probe.Error) {
 				Version:       "1",
 				ActiveSession: make(map[string]*MultipartSession),
 			}
-			if err := SaveMultipartsSession(multiparts); err != nil {
+			if err := saveMultipartsSession(multiparts); err != nil {
 				return Filesystem{}, err.Trace()
 			}
 		} else {
@@ -80,7 +84,7 @@ func New() (Filesystem, *probe.Error) {
 				Version:  "1",
 				Metadata: make(map[string]*BucketMetadata),
 			}
-			if err := SaveBucketsMetadata(buckets); err != nil {
+			if err := saveBucketsMetadata(buckets); err != nil {
 				return Filesystem{}, err.Trace()
 			}
 		} else {
@@ -88,18 +92,12 @@ func New() (Filesystem, *probe.Error) {
 		}
 	}
 	a := Filesystem{lock: new(sync.Mutex)}
+	a.path = rootPath
 	a.multiparts = multiparts
 	a.buckets = buckets
 	return a, nil
 }
 
-// SetRootPath - set root path
-func (fs *Filesystem) SetRootPath(path string) {
-	fs.lock.Lock()
-	defer fs.lock.Unlock()
-	fs.path = path
-}
-
 // SetMinFreeDisk - set min free disk
 func (fs *Filesystem) SetMinFreeDisk(minFreeDisk int64) {
 	fs.lock.Lock()

pkg/fs/fs_test.go

@@ -19,7 +19,6 @@ package fs
 import (
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"testing"
 
 	. "gopkg.in/check.v1"
@@ -34,15 +33,10 @@ var _ = Suite(&MySuite{})
 func (s *MySuite) TestAPISuite(c *C) {
 	var storageList []string
 	create := func() Filesystem {
-		configPath, err := ioutil.TempDir(os.TempDir(), "minio-")
-		c.Check(err, IsNil)
 		path, err := ioutil.TempDir(os.TempDir(), "minio-")
 		c.Check(err, IsNil)
-		SetFSMultipartsConfigPath(filepath.Join(configPath, "multiparts-session.json"))
-		SetFSBucketsConfigPath(filepath.Join(configPath, "buckets.json"))
 		storageList = append(storageList, path)
-		store, perr := New()
-		store.SetRootPath(path)
+		store, perr := New(path)
 		store.SetMinFreeDisk(0)
 		c.Check(perr, IsNil)
 		return store

pkg/user/user.go

@@ -0,0 +1,49 @@
+package user
+
+import (
+	"os"
+	"os/user"
+	"runtime"
+	"strconv"
+)
+
+// Current is a portable implementation to determine the current user.
+// Golang's user.Current does not work reliably under docker or 32bit linux
+//
+// Two issues this code handles :-
+//
+//   Docker Container - For static binaries NSS library will not be a part of the static binary hence user.Current() fails.
+//   Linux Intel 32 bit - CGO is not enabled so it will not link with NSS library.
+//
+func Current() (*user.User, error) {
+	if os.Getenv("DOCKERIMAGE") == "1" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		return &user.User{Uid: "0", Gid: "0", Username: "root", Name: "root", HomeDir: wd}, nil
+	}
+	if runtime.GOARCH == "386" && runtime.GOOS == "linux" {
+		return &user.User{
+			Uid:      strconv.Itoa(os.Getuid()),
+			Gid:      strconv.Itoa(os.Getgid()),
+			Username: os.Getenv("USER"),
+			Name:     os.Getenv("USER"),
+			HomeDir:  os.Getenv("HOME"),
+		}, nil
+	}
+	user, e := user.Current()
+	if e != nil {
+		return nil, e
+	}
+	return user, nil
+}
+
+// HomeDir - return current home directory.
+func HomeDir() (string, error) {
+	user, err := Current()
+	if err != nil {
+		return "", err
+	}
+	return user.HomeDir, nil
+}

routers.go

@@ -64,10 +64,9 @@ func registerCloudStorageAPI(mux *router.Router, a CloudStorageAPI) {
 
 // getNewCloudStorageAPI instantiate a new CloudStorageAPI
 func getNewCloudStorageAPI(conf cloudServerConfig) CloudStorageAPI {
-	fs, err := fs.New()
+	fs, err := fs.New(conf.Path)
 	fatalIf(err.Trace(), "Initializing filesystem failed.", nil)
 
-	fs.SetRootPath(conf.Path)
 	fs.SetMinFreeDisk(conf.MinFreeDisk)
 	if conf.Expiry > 0 {
 		go fs.AutoExpiryThread(conf.Expiry)

server-config.go

@@ -26,6 +26,7 @@ import (
 	"github.com/fatih/color"
 	"github.com/minio/minio-xl/pkg/probe"
 	"github.com/minio/minio-xl/pkg/quick"
+	"github.com/minio/minio/pkg/user"
 )
 
 // configV1
@@ -118,19 +119,33 @@ func (c *configV2) JSON() string {
 	return string(loggerBytes)
 }
 
+// configPath for custom config path only for testing purposes
+var customConfigPath string
+
+// Sets a new config path.
+func setGlobalConfigPath(configPath string) {
+	customConfigPath = configPath
+}
+
 // getConfigPath get users config path
 func getConfigPath() (string, *probe.Error) {
 	if customConfigPath != "" {
 		return customConfigPath, nil
 	}
-	u, err := userCurrent()
-	if err != nil {
-		return "", err.Trace()
+	homeDir, e := user.HomeDir()
+	if e != nil {
+		return "", probe.NewError(e)
 	}
-	configPath := filepath.Join(u.HomeDir, ".minio")
+	configPath := filepath.Join(homeDir, ".minio")
 	return configPath, nil
 }
 
+func mustGetConfigPath() string {
+	configPath, err := getConfigPath()
+	fatalIf(err.Trace(), "Unable to get config path.", nil)
+	return configPath
+}
+
 // createConfigPath create users config path
 func createConfigPath() *probe.Error {
 	configPath, err := getConfigPath()
@@ -157,9 +172,7 @@ func isConfigFileExists() bool {
 // mustGetConfigFile always get users config file, if not panic
 func mustGetConfigFile() string {
 	configFile, err := getConfigFile()
-	if err != nil {
-		panic(err)
-	}
+	fatalIf(err.Trace(), "Unable to get config file.", nil)
 	return configFile
 }
 
@@ -172,9 +185,6 @@ func getConfigFile() (string, *probe.Error) {
 	return filepath.Join(configPath, "config.json"), nil
 }
 
-// configPath for custom config path only for testing purposes
-var customConfigPath string
-
 // saveConfig save config
 func saveConfig(a *configV2) *probe.Error {
 	configFile, err := getConfigFile()

server-main.go

@@ -243,14 +243,9 @@ func initServer() *probe.Error {
 		return err.Trace()
 	}
 	if conf != nil {
-		if globalJSONFlag {
-			Println(accessKeys{conf}.JSON())
-		} else {
 		Println()
 		Println(accessKeys{conf})
 	}
-	}
-	if !globalJSONFlag {
 	Println("\nTo configure Minio Client.")
 	if runtime.GOOS == "windows" {
 		Println("\n\tDownload https://dl.minio.io:9000/updates/2015/Nov/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe")
@@ -265,7 +260,6 @@ func initServer() *probe.Error {
 		Println("\t$ ./mc cp ~/Photos... localhost/photobucket")
 	}
 	Println()
-	}
 	return nil
 }
 

server_fs_test.go

@@ -21,7 +21,6 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"sort"
 	"strings"
 	"time"
@@ -59,20 +58,6 @@ func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) {
 	fsroot, err := ioutil.TempDir(os.TempDir(), "api-")
 	c.Assert(err, IsNil)
 
-	fs.SetFSMultipartsConfigPath(filepath.Join(root, "multiparts-session.json"))
-	fs.SetFSBucketsConfigPath(filepath.Join(root, "buckets.json"))
-	multiparts := &fs.Multiparts{}
-	multiparts.Version = "1"
-	multiparts.ActiveSession = make(map[string]*fs.MultipartSession)
-	perr := fs.SaveMultipartsSession(multiparts)
-	c.Assert(perr, IsNil)
-
-	buckets := &fs.Buckets{}
-	buckets.Version = "1"
-	buckets.Metadata = make(map[string]*fs.BucketMetadata)
-	perr = fs.SaveBucketsMetadata(buckets)
-	c.Assert(perr, IsNil)
-
 	accessKeyID, perr := generateAccessKeyID()
 	c.Assert(perr, IsNil)
 	secretAccessKey, perr := generateSecretAccessKey()
@@ -85,7 +70,7 @@ func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) {
 	s.secretAccessKey = string(secretAccessKey)
 
 	// do this only once here
-	customConfigPath = root
+	setGlobalConfigPath(root)
 
 	perr = saveConfig(conf)
 	c.Assert(perr, IsNil)

update-main.go

@@ -162,12 +162,8 @@ func getReleaseUpdate(updateURL string) {
 		updateMsg.Update = true
 	}
 
-	if globalJSONFlag {
-		Println(updateMsg.JSON())
-	} else {
 	Println(updateMsg)
 }
-}
 
 // main entry point for update command.
 func mainUpdate(ctx *cli.Context) {

utils.go

@@ -18,13 +18,8 @@ package main
 
 import (
 	"encoding/base64"
-	"os"
-	"os/user"
-	"runtime"
 	"strconv"
 	"strings"
-
-	"github.com/minio/minio-xl/pkg/probe"
 )
 
 // isValidMD5 - verify if valid md5
@@ -56,30 +51,3 @@ func isMaxObjectSize(size string) bool {
 	}
 	return false
 }
-
-// Workaround for docker images with fully static binary and 32bit linux operating systems.
-// For static binaries NSS library will not be a part of the static binary hence user.Current() fails.
-// For 32bit linux CGO is not enabled so it will not provide linux specific codebase.
-func userCurrent() (*user.User, *probe.Error) {
-	if os.Getenv("DOCKERIMAGE") == "1" {
-		wd, err := os.Getwd()
-		if err != nil {
-			return nil, probe.NewError(err)
-		}
-		return &user.User{Uid: "0", Gid: "0", Username: "root", Name: "root", HomeDir: wd}, nil
-	}
-	if runtime.GOARCH == "386" && runtime.GOOS == "linux" {
-		return &user.User{
-			Uid:      strconv.Itoa(os.Getuid()),
-			Gid:      strconv.Itoa(os.Getgid()),
-			Username: os.Getenv("USER"),
-			Name:     os.Getenv("USER"),
-			HomeDir:  os.Getenv("HOME"),
-		}, nil
-	}
-	user, err := user.Current()
-	if err != nil {
-		return nil, probe.NewError(err)
-	}
-	return user, nil
-}