From 5ae8d34f763132bfb22aee6922605cf7b1459305 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 23 Apr 2015 11:26:47 -0700 Subject: [PATCH] Donut mode will now take multiple paths as argument, updated docs as well --- main.go | 22 ++++++--- pkg/api/api_test.go | 4 +- pkg/storage/drivers/donut/donut.go | 59 ++++++++++++++++++------- pkg/storage/drivers/donut/donut_test.go | 9 ++-- 4 files changed, 67 insertions(+), 27 deletions(-) diff --git a/main.go b/main.go index 67dfc8050..5d63510cc 100644 --- a/main.go +++ b/main.go @@ -94,6 +94,9 @@ EXAMPLES: 2. Create a temporary donut volume under "/tmp" $ minio mode {{.Name}} /tmp + 3. Create a donut volume under collection of paths + $ minio mode {{.Name}} /mnt/backup2014feb /mnt/backup2014feb + `, } @@ -135,7 +138,7 @@ func (f webFactory) getStartServerFunc() startServerFunc { type donutFactory struct { server.Config - path string + paths []string } func (f donutFactory) getStartServerFunc() startServerFunc { @@ -146,7 +149,7 @@ func (f donutFactory) getStartServerFunc() startServerFunc { httpConfig.CertFile = f.CertFile httpConfig.KeyFile = f.KeyFile httpConfig.Websocket = false - _, _, driver := donut.Start(f.path) + _, _, driver := donut.Start(f.paths) ctrl, status, _ := httpserver.Start(api.HTTPHandler(f.Domain, driver), httpConfig) return ctrl, status } @@ -223,14 +226,21 @@ func runDonut(c *cli.Context) { if len(c.Args()) < 1 { cli.ShowCommandHelpAndExit(c, "donut", 1) // last argument is exit code } - p := c.Args().First() - if strings.TrimSpace(p) == "" { - p = path.Join(u.HomeDir, "minio-storage", "donut") + + // supporting multiple paths + var paths []string + if strings.TrimSpace(c.Args().First()) == "" { + p := path.Join(u.HomeDir, "minio-storage", "donut") + paths = append(paths, p) + } else { + for _, arg := range c.Args() { + paths = append(paths, strings.TrimSpace(arg)) + } } apiServerConfig := getAPIServerConfig(c) donutDriver := donutFactory{ Config: apiServerConfig, - path: p, + paths: paths, } apiServer := donutDriver.getStartServerFunc() webServer := getWebServerConfigFunc(c) diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 842665626..42a3acf2b 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -70,7 +70,9 @@ var _ = Suite(&MySuite{ var _ = Suite(&MySuite{ initDriver: func() (drivers.Driver, string) { root, _ := ioutil.TempDir(os.TempDir(), "minio-api") - _, _, driver := donut.Start(root) + var roots []string + roots = append(roots, root) + _, _, driver := donut.Start(roots) return driver, root }, }) diff --git a/pkg/storage/drivers/donut/donut.go b/pkg/storage/drivers/donut/donut.go index 7ef131d6a..147d7bfbe 100644 --- a/pkg/storage/drivers/donut/donut.go +++ b/pkg/storage/drivers/donut/donut.go @@ -38,7 +38,7 @@ import ( // donutDriver - creates a new single disk drivers driver using donut type donutDriver struct { donut donut.Donut - path string + paths []string } const ( @@ -47,9 +47,9 @@ const ( // This is a dummy nodeDiskMap which is going to be deprecated soon // once the Management API is standardized, this map is useful for now -// to show multi disk API correctness behavior. +// to show multi disk API correctness and parity calculation // -// This should be obtained from donut configuration file +// Ideally this should be obtained from per node configuration file func createNodeDiskMap(p string) map[string][]string { nodes := make(map[string][]string) nodes["localhost"] = make([]string, 16) @@ -65,24 +65,51 @@ func createNodeDiskMap(p string) map[string][]string { return nodes } +// This is a dummy nodeDiskMap which is going to be deprecated soon +// once the Management API is standardized, and we have way of adding +// and removing disks. This is useful for now to take inputs from CLI +func createNodeDiskMapFromSlice(paths []string) map[string][]string { + diskPaths := make([]string, len(paths)) + nodes := make(map[string][]string) + for i, p := range paths { + diskPath := path.Join(p, strconv.Itoa(i)) + if _, err := os.Stat(diskPath); err != nil { + if os.IsNotExist(err) { + os.MkdirAll(diskPath, 0700) + } + } + diskPaths[i] = diskPath + } + nodes["localhost"] = diskPaths + return nodes +} + // Start a single disk subsystem -func Start(path string) (chan<- string, <-chan error, drivers.Driver) { +func Start(paths []string) (chan<- string, <-chan error, drivers.Driver) { ctrlChannel := make(chan string) errorChannel := make(chan error) - errParams := map[string]string{"path": path} - // Soon to be user configurable, when Management API - // is finished we remove "default" to something - // which is passed down from configuration - donut, err := donut.NewDonut("default", createNodeDiskMap(path)) - if err != nil { - err = iodine.New(err, errParams) - log.Error.Println(err) + // Soon to be user configurable, when Management API is available + // we should remove "default" to something which is passed down + // from configuration paramters + var d donut.Donut + var err error + if len(paths) == 1 { + d, err = donut.NewDonut("default", createNodeDiskMap(paths[0])) + if err != nil { + err = iodine.New(err, nil) + log.Error.Println(err) + } + } else { + d, err = donut.NewDonut("default", createNodeDiskMapFromSlice(paths)) + if err != nil { + err = iodine.New(err, nil) + log.Error.Println(err) + } } - s := new(donutDriver) - s.donut = donut - s.path = path + s.donut = d + s.paths = paths go start(ctrlChannel, errorChannel, s) return ctrlChannel, errorChannel, s @@ -143,7 +170,7 @@ func (d donutDriver) GetBucketMetadata(bucketName string) (drivers.BucketMetadat } acl, ok := metadata["acl"] if !ok { - return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{Path: d.path}, nil) + return drivers.BucketMetadata{}, iodine.New(drivers.BackendCorrupted{}, nil) } bucketMetadata := drivers.BucketMetadata{ Name: metadata["name"], diff --git a/pkg/storage/drivers/donut/donut_test.go b/pkg/storage/drivers/donut/donut_test.go index 6dcb74341..f3d33c8d4 100644 --- a/pkg/storage/drivers/donut/donut_test.go +++ b/pkg/storage/drivers/donut/donut_test.go @@ -32,13 +32,14 @@ type MySuite struct{} var _ = Suite(&MySuite{}) func (s *MySuite) TestAPISuite(c *C) { - // c.Skip("Not Implemented") var storageList []string create := func() drivers.Driver { - path, err := ioutil.TempDir(os.TempDir(), "minio-fs-") + var paths []string + p, err := ioutil.TempDir(os.TempDir(), "minio-fs-") c.Check(err, IsNil) - storageList = append(storageList, path) - _, _, store := Start(path) // TODO Make InMemory driver + storageList = append(storageList, p) + paths = append(paths, p) + _, _, store := Start(paths) return store } drivers.APITestSuite(c, create)