minio: Add config-folder option.

Fixes #997
master
Harshavardhana 9 years ago
parent a97c4ebce3
commit 836f5204af
  1. 4
      config-logger-main.go
  2. 17
      config-version-main.go
  3. 6
      flags.go
  4. 21
      globals.go
  5. 26
      main.go
  6. 2
      pkg/contentdb/contentdb.go
  7. 4
      pkg/contentdb/db.go
  8. 62
      pkg/fs/config.go
  9. 6
      pkg/fs/fs-bucket.go
  10. 4
      pkg/fs/fs-multipart.go
  11. 18
      pkg/fs/fs.go
  12. 8
      pkg/fs/fs_test.go
  13. 49
      pkg/user/user.go
  14. 3
      routers.go
  15. 30
      server-config.go
  16. 36
      server-main.go
  17. 17
      server_fs_test.go
  18. 6
      update-main.go
  19. 32
      utils.go

@ -106,10 +106,6 @@ func mainConfigLogger(ctx *cli.Context) {
}
}
if ctx.Args().Get(0) == "list" {
if globalJSONFlag {
Println(conf.JSON())
return
}
Println(conf)
}
}

@ -16,12 +16,7 @@
package main
import (
"encoding/json"
"github.com/minio/cli"
"github.com/minio/minio-xl/pkg/probe"
)
import "github.com/minio/cli"
// Print config version.
var configVersionCmd = cli.Command{
@ -48,15 +43,5 @@ func mainConfigVersion(ctx *cli.Context) {
// convert interface{} back to its original struct
newConf := config
type Version string
if globalJSONFlag {
tB, e := json.Marshal(
struct {
Version Version `json:"version"`
}{Version: Version(newConf.Version)},
)
fatalIf(probe.NewError(e), "Unable to construct version string.", nil)
Println(string(tB))
return
}
Println(newConf.Version)
}

@ -22,6 +22,12 @@ import "github.com/minio/cli"
var flags = []cli.Flag{}
var (
configFolderFlag = cli.StringFlag{
Name: "config-folder, C",
Value: mustGetConfigPath(),
Usage: "Path to configuration folder.",
}
addressFlag = cli.StringFlag{
Name: "address",
Value: ":9000",

@ -1,21 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
var (
globalJSONFlag = false // Json flag set via command line
)

@ -19,6 +19,7 @@ package main
import (
"fmt"
"os"
"os/user"
"runtime"
"strconv"
@ -55,9 +56,9 @@ func init() {
checkGolangRuntimeVersion()
// Check for the environment early on and gracefuly report.
_, err := userCurrent()
_, err := user.Current()
if err != nil {
Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
}
if os.Getenv("DOCKERIMAGE") == "1" {
@ -118,6 +119,7 @@ func registerApp() *cli.App {
registerCommand(updateCmd)
// register all flags
registerFlag(configFolderFlag)
registerFlag(addressFlag)
registerFlag(accessLogFlag)
registerFlag(rateLimitFlag)
@ -146,10 +148,19 @@ func registerApp() *cli.App {
}
Fatalln(msg)
}
return app
}
func checkMainSyntax(c *cli.Context) {
configPath, err := getConfigPath()
if err != nil {
Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
}
if configPath == "" {
Fatalf("Config folder cannot be empty, please specify --config-folder <foldername>.")
}
}
func main() {
probe.Init() // Set project's root source path.
probe.SetAppInfo("Release-Tag", minioReleaseTag)
@ -160,8 +171,15 @@ func main() {
app := registerApp()
app.Before = func(c *cli.Context) error {
globalJSONFlag = c.GlobalBool("json")
// Sets new config folder.
setGlobalConfigPath(c.GlobalString("config-folder"))
// Valid input arguments to main.
checkMainSyntax(c)
// Migrate any old version of config / state files to newer format.
migrate()
return nil
}
app.ExtraInfo = func() map[string]string {

@ -102,7 +102,7 @@ func Init() error {
// Lookup returns matching content-type for known types of file extensions.
func Lookup(extension string) (contentType string, e error) {
if !isInitialized {
return "", errors.New("contentdb is not initialized.")
return "", errors.New("contentdb is not initialized")
}
return extDB[extension], e

@ -184,8 +184,8 @@ type bintree struct {
}
var _bintree = &bintree{nil, map[string]*bintree{
"db": &bintree{nil, map[string]*bintree{
"db.json": &bintree{dbDbJSON, map[string]*bintree{}},
"db": {nil, map[string]*bintree{
"db.json": {dbDbJSON, map[string]*bintree{}},
}},
}}

@ -17,52 +17,22 @@
package fs
import (
"os"
"os/user"
"path/filepath"
"runtime"
"strconv"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio-xl/pkg/quick"
"github.com/minio/minio/pkg/user"
)
// Workaround for docker images with fully static binary and 32bit linux operating systems.
// For static binaries NSS library will not be a part of the static binary hence user.Current() fails.
// For 32bit linux CGO is not enabled so it will not provide linux specific codebase.
func userCurrent() (*user.User, *probe.Error) {
if os.Getenv("DOCKERIMAGE") == "1" {
wd, err := os.Getwd()
if err != nil {
return nil, probe.NewError(err)
}
return &user.User{Uid: "0", Gid: "0", Username: "root", Name: "root", HomeDir: wd}, nil
}
if runtime.GOARCH == "386" && runtime.GOOS == "linux" {
return &user.User{
Uid: strconv.Itoa(os.Getuid()),
Gid: strconv.Itoa(os.Getgid()),
Username: os.Getenv("USER"),
Name: os.Getenv("USER"),
HomeDir: os.Getenv("HOME"),
}, nil
}
user, err := user.Current()
if err != nil {
return nil, probe.NewError(err)
}
return user, nil
}
func getFSBucketsConfigPath() (string, *probe.Error) {
if customBucketsConfigPath != "" {
return customBucketsConfigPath, nil
}
u, err := userCurrent()
if err != nil {
return "", err.Trace()
homeDir, e := user.HomeDir()
if e != nil {
return "", probe.NewError(e)
}
fsBucketsConfigPath := filepath.Join(u.HomeDir, ".minio", "buckets.json")
fsBucketsConfigPath := filepath.Join(homeDir, ".minio", "$buckets.json")
return fsBucketsConfigPath, nil
}
@ -70,29 +40,29 @@ func getFSMultipartsSessionConfigPath() (string, *probe.Error) {
if customMultipartsConfigPath != "" {
return customMultipartsConfigPath, nil
}
u, err := userCurrent()
if err != nil {
return "", err.Trace()
homeDir, e := user.HomeDir()
if e != nil {
return "", probe.NewError(e)
}
fsMultipartsConfigPath := filepath.Join(u.HomeDir, ".minio", "multiparts-session.json")
fsMultipartsConfigPath := filepath.Join(homeDir, ".minio", "$multiparts-session.json")
return fsMultipartsConfigPath, nil
}
// internal variable only accessed via get/set methods
var customMultipartsConfigPath, customBucketsConfigPath string
// SetFSBucketsConfigPath - set custom fs buckets config path
func SetFSBucketsConfigPath(configPath string) {
// setFSBucketsConfigPath - set custom fs buckets config path
func setFSBucketsConfigPath(configPath string) {
customBucketsConfigPath = configPath
}
// SetFSMultipartsConfigPath - set custom multiparts session config path
func SetFSMultipartsConfigPath(configPath string) {
func setFSMultipartsConfigPath(configPath string) {
customMultipartsConfigPath = configPath
}
// SaveMultipartsSession - save multiparts
func SaveMultipartsSession(multiparts *Multiparts) *probe.Error {
// saveMultipartsSession - save multiparts
func saveMultipartsSession(multiparts *Multiparts) *probe.Error {
fsMultipartsConfigPath, err := getFSMultipartsSessionConfigPath()
if err != nil {
return err.Trace()
@ -107,8 +77,8 @@ func SaveMultipartsSession(multiparts *Multiparts) *probe.Error {
return nil
}
// SaveBucketsMetadata - save metadata of all buckets
func SaveBucketsMetadata(buckets *Buckets) *probe.Error {
// saveBucketsMetadata - save metadata of all buckets
func saveBucketsMetadata(buckets *Buckets) *probe.Error {
fsBucketsConfigPath, err := getFSBucketsConfigPath()
if err != nil {
return err.Trace()

@ -54,7 +54,7 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
return probe.NewError(err)
}
delete(fs.buckets.Metadata, bucket)
if err := SaveBucketsMetadata(fs.buckets); err != nil {
if err := saveBucketsMetadata(fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil
@ -147,7 +147,7 @@ func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
bucketMetadata.Created = fi.ModTime()
bucketMetadata.ACL = BucketACL(acl)
fs.buckets.Metadata[bucket] = bucketMetadata
if err := SaveBucketsMetadata(fs.buckets); err != nil {
if err := saveBucketsMetadata(fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil
@ -211,7 +211,7 @@ func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string
}
bucketMetadata.ACL = BucketACL(acl)
fs.buckets.Metadata[bucket] = bucketMetadata
if err := SaveBucketsMetadata(fs.buckets); err != nil {
if err := saveBucketsMetadata(fs.buckets); err != nil {
return err.Trace(bucket)
}
return nil

@ -201,7 +201,7 @@ func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.E
if err != nil {
return "", probe.NewError(err)
}
if err := SaveMultipartsSession(fs.multiparts); err != nil {
if err := saveMultipartsSession(fs.multiparts); err != nil {
return "", err.Trace()
}
return uploadID, nil
@ -419,7 +419,7 @@ func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, da
file.CloseAndPurge()
return ObjectMetadata{}, probe.NewError(err)
}
if err := SaveMultipartsSession(fs.multiparts); err != nil {
if err := saveMultipartsSession(fs.multiparts); err != nil {
file.CloseAndPurge()
return ObjectMetadata{}, err.Trace()
}

@ -18,6 +18,7 @@ package fs
import (
"os"
"path/filepath"
"sync"
"time"
@ -54,7 +55,10 @@ type Multiparts struct {
}
// New instantiate a new donut
func New() (Filesystem, *probe.Error) {
func New(rootPath string) (Filesystem, *probe.Error) {
setFSBucketsConfigPath(filepath.Join(rootPath, "$buckets.json"))
setFSMultipartsConfigPath(filepath.Join(rootPath, "$multiparts-session.json"))
var err *probe.Error
// load multiparts session from disk
var multiparts *Multiparts
@ -65,7 +69,7 @@ func New() (Filesystem, *probe.Error) {
Version: "1",
ActiveSession: make(map[string]*MultipartSession),
}
if err := SaveMultipartsSession(multiparts); err != nil {
if err := saveMultipartsSession(multiparts); err != nil {
return Filesystem{}, err.Trace()
}
} else {
@ -80,7 +84,7 @@ func New() (Filesystem, *probe.Error) {
Version: "1",
Metadata: make(map[string]*BucketMetadata),
}
if err := SaveBucketsMetadata(buckets); err != nil {
if err := saveBucketsMetadata(buckets); err != nil {
return Filesystem{}, err.Trace()
}
} else {
@ -88,18 +92,12 @@ func New() (Filesystem, *probe.Error) {
}
}
a := Filesystem{lock: new(sync.Mutex)}
a.path = rootPath
a.multiparts = multiparts
a.buckets = buckets
return a, nil
}
// SetRootPath - set root path
func (fs *Filesystem) SetRootPath(path string) {
fs.lock.Lock()
defer fs.lock.Unlock()
fs.path = path
}
// SetMinFreeDisk - set min free disk
func (fs *Filesystem) SetMinFreeDisk(minFreeDisk int64) {
fs.lock.Lock()

@ -19,7 +19,6 @@ package fs
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
. "gopkg.in/check.v1"
@ -34,15 +33,10 @@ var _ = Suite(&MySuite{})
func (s *MySuite) TestAPISuite(c *C) {
var storageList []string
create := func() Filesystem {
configPath, err := ioutil.TempDir(os.TempDir(), "minio-")
c.Check(err, IsNil)
path, err := ioutil.TempDir(os.TempDir(), "minio-")
c.Check(err, IsNil)
SetFSMultipartsConfigPath(filepath.Join(configPath, "multiparts-session.json"))
SetFSBucketsConfigPath(filepath.Join(configPath, "buckets.json"))
storageList = append(storageList, path)
store, perr := New()
store.SetRootPath(path)
store, perr := New(path)
store.SetMinFreeDisk(0)
c.Check(perr, IsNil)
return store

@ -0,0 +1,49 @@
package user
import (
"os"
"os/user"
"runtime"
"strconv"
)
// Current is a portable implementation to determine the current user.
// Golang's user.Current does not work reliably under docker or 32bit linux
//
// Two issues this code handles :-
//
// Docker Container - For static binaries NSS library will not be a part of the static binary hence user.Current() fails.
// Linux Intel 32 bit - CGO is not enabled so it will not link with NSS library.
//
func Current() (*user.User, error) {
if os.Getenv("DOCKERIMAGE") == "1" {
wd, err := os.Getwd()
if err != nil {
return nil, err
}
return &user.User{Uid: "0", Gid: "0", Username: "root", Name: "root", HomeDir: wd}, nil
}
if runtime.GOARCH == "386" && runtime.GOOS == "linux" {
return &user.User{
Uid: strconv.Itoa(os.Getuid()),
Gid: strconv.Itoa(os.Getgid()),
Username: os.Getenv("USER"),
Name: os.Getenv("USER"),
HomeDir: os.Getenv("HOME"),
}, nil
}
user, e := user.Current()
if e != nil {
return nil, e
}
return user, nil
}
// HomeDir - return current home directory.
func HomeDir() (string, error) {
user, err := Current()
if err != nil {
return "", err
}
return user.HomeDir, nil
}

@ -64,10 +64,9 @@ func registerCloudStorageAPI(mux *router.Router, a CloudStorageAPI) {
// getNewCloudStorageAPI instantiate a new CloudStorageAPI
func getNewCloudStorageAPI(conf cloudServerConfig) CloudStorageAPI {
fs, err := fs.New()
fs, err := fs.New(conf.Path)
fatalIf(err.Trace(), "Initializing filesystem failed.", nil)
fs.SetRootPath(conf.Path)
fs.SetMinFreeDisk(conf.MinFreeDisk)
if conf.Expiry > 0 {
go fs.AutoExpiryThread(conf.Expiry)

@ -26,6 +26,7 @@ import (
"github.com/fatih/color"
"github.com/minio/minio-xl/pkg/probe"
"github.com/minio/minio-xl/pkg/quick"
"github.com/minio/minio/pkg/user"
)
// configV1
@ -118,19 +119,33 @@ func (c *configV2) JSON() string {
return string(loggerBytes)
}
// configPath for custom config path only for testing purposes
var customConfigPath string
// Sets a new config path.
func setGlobalConfigPath(configPath string) {
customConfigPath = configPath
}
// getConfigPath get users config path
func getConfigPath() (string, *probe.Error) {
if customConfigPath != "" {
return customConfigPath, nil
}
u, err := userCurrent()
if err != nil {
return "", err.Trace()
homeDir, e := user.HomeDir()
if e != nil {
return "", probe.NewError(e)
}
configPath := filepath.Join(u.HomeDir, ".minio")
configPath := filepath.Join(homeDir, ".minio")
return configPath, nil
}
func mustGetConfigPath() string {
configPath, err := getConfigPath()
fatalIf(err.Trace(), "Unable to get config path.", nil)
return configPath
}
// createConfigPath create users config path
func createConfigPath() *probe.Error {
configPath, err := getConfigPath()
@ -157,9 +172,7 @@ func isConfigFileExists() bool {
// mustGetConfigFile always get users config file, if not panic
func mustGetConfigFile() string {
configFile, err := getConfigFile()
if err != nil {
panic(err)
}
fatalIf(err.Trace(), "Unable to get config file.", nil)
return configFile
}
@ -172,9 +185,6 @@ func getConfigFile() (string, *probe.Error) {
return filepath.Join(configPath, "config.json"), nil
}
// configPath for custom config path only for testing purposes
var customConfigPath string
// saveConfig save config
func saveConfig(a *configV2) *probe.Error {
configFile, err := getConfigFile()

@ -243,29 +243,23 @@ func initServer() *probe.Error {
return err.Trace()
}
if conf != nil {
if globalJSONFlag {
Println(accessKeys{conf}.JSON())
} else {
Println()
Println(accessKeys{conf})
}
}
if !globalJSONFlag {
Println("\nTo configure Minio Client.")
if runtime.GOOS == "windows" {
Println("\n\tDownload https://dl.minio.io:9000/updates/2015/Nov/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe")
Println("\t$ mc.exe config host add http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
Println("\t$ mc.exe mb localhost/photobucket")
Println("\t$ mc.exe cp C:\\Photos... localhost/photobucket")
} else {
Println("\n\t$ wget https://dl.minio.io:9000/updates/2015/Nov/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc")
Println("\t$ chmod 755 mc")
Println("\t$ ./mc config host add http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
Println("\t$ ./mc mb localhost/photobucket")
Println("\t$ ./mc cp ~/Photos... localhost/photobucket")
}
Println()
Println(accessKeys{conf})
}
Println("\nTo configure Minio Client.")
if runtime.GOOS == "windows" {
Println("\n\tDownload https://dl.minio.io:9000/updates/2015/Nov/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe")
Println("\t$ mc.exe config host add http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
Println("\t$ mc.exe mb localhost/photobucket")
Println("\t$ mc.exe cp C:\\Photos... localhost/photobucket")
} else {
Println("\n\t$ wget https://dl.minio.io:9000/updates/2015/Nov/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc")
Println("\t$ chmod 755 mc")
Println("\t$ ./mc config host add http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
Println("\t$ ./mc mb localhost/photobucket")
Println("\t$ ./mc cp ~/Photos... localhost/photobucket")
}
Println()
return nil
}

@ -21,7 +21,6 @@ import (
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"time"
@ -59,20 +58,6 @@ func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) {
fsroot, err := ioutil.TempDir(os.TempDir(), "api-")
c.Assert(err, IsNil)
fs.SetFSMultipartsConfigPath(filepath.Join(root, "multiparts-session.json"))
fs.SetFSBucketsConfigPath(filepath.Join(root, "buckets.json"))
multiparts := &fs.Multiparts{}
multiparts.Version = "1"
multiparts.ActiveSession = make(map[string]*fs.MultipartSession)
perr := fs.SaveMultipartsSession(multiparts)
c.Assert(perr, IsNil)
buckets := &fs.Buckets{}
buckets.Version = "1"
buckets.Metadata = make(map[string]*fs.BucketMetadata)
perr = fs.SaveBucketsMetadata(buckets)
c.Assert(perr, IsNil)
accessKeyID, perr := generateAccessKeyID()
c.Assert(perr, IsNil)
secretAccessKey, perr := generateSecretAccessKey()
@ -85,7 +70,7 @@ func (s *MyAPIFSCacheSuite) SetUpSuite(c *C) {
s.secretAccessKey = string(secretAccessKey)
// do this only once here
customConfigPath = root
setGlobalConfigPath(root)
perr = saveConfig(conf)
c.Assert(perr, IsNil)

@ -162,11 +162,7 @@ func getReleaseUpdate(updateURL string) {
updateMsg.Update = true
}
if globalJSONFlag {
Println(updateMsg.JSON())
} else {
Println(updateMsg)
}
Println(updateMsg)
}
// main entry point for update command.

@ -18,13 +18,8 @@ package main
import (
"encoding/base64"
"os"
"os/user"
"runtime"
"strconv"
"strings"
"github.com/minio/minio-xl/pkg/probe"
)
// isValidMD5 - verify if valid md5
@ -56,30 +51,3 @@ func isMaxObjectSize(size string) bool {
}
return false
}
// Workaround for docker images with fully static binary and 32bit linux operating systems.
// For static binaries NSS library will not be a part of the static binary hence user.Current() fails.
// For 32bit linux CGO is not enabled so it will not provide linux specific codebase.
func userCurrent() (*user.User, *probe.Error) {
if os.Getenv("DOCKERIMAGE") == "1" {
wd, err := os.Getwd()
if err != nil {
return nil, probe.NewError(err)
}
return &user.User{Uid: "0", Gid: "0", Username: "root", Name: "root", HomeDir: wd}, nil
}
if runtime.GOARCH == "386" && runtime.GOOS == "linux" {
return &user.User{
Uid: strconv.Itoa(os.Getuid()),
Gid: strconv.Itoa(os.Getgid()),
Username: os.Getenv("USER"),
Name: os.Getenv("USER"),
HomeDir: os.Getenv("HOME"),
}, nil
}
user, err := user.Current()
if err != nil {
return nil, probe.NewError(err)
}
return user, nil
}

Loading…
Cancel
Save