From d1ebf935da8f5c7bf90e55f426a0977394ffa4e2 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Mon, 8 Dec 2014 19:40:48 -0800 Subject: [PATCH] Adding extremely simple fs based get and put in erasure-demo --- cmd/erasure-demo/fs.go | 40 ++++++++++++++ cmd/erasure-demo/get.go | 52 ++++++++++++++++++ cmd/erasure-demo/main.go | 115 ++++++++++++++++++++++++++++++--------- cmd/erasure-demo/put.go | 1 + 4 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 cmd/erasure-demo/fs.go create mode 100644 cmd/erasure-demo/get.go create mode 100644 cmd/erasure-demo/put.go diff --git a/cmd/erasure-demo/fs.go b/cmd/erasure-demo/fs.go new file mode 100644 index 000000000..842c8fe5e --- /dev/null +++ b/cmd/erasure-demo/fs.go @@ -0,0 +1,40 @@ +package main + +import ( + "bytes" + "io" + "os" + "path" + + "github.com/minio-io/minio/pkgs/storage" + "github.com/minio-io/minio/pkgs/storage/fsstorage" +) + +func fsGet(config inputConfig, objectPath string) (io.Reader, error) { + var objectStorage storage.ObjectStorage + rootDir := path.Join(config.rootDir, config.storageDriver) + objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir} + object, err := objectStorage.Get(objectPath) + if err != nil { + return nil, err + } + objectBuffer := bytes.NewBuffer(object) + return objectBuffer, nil +} + +func fsPut(config inputConfig, objectPath string, reader io.Reader) error { + var err error + rootDir := path.Join(config.rootDir, config.storageDriver) + if err := os.MkdirAll(config.rootDir, 0700); err != nil { + return err + } + var objectStorage storage.ObjectStorage + buffer := new(bytes.Buffer) + buffer.ReadFrom(reader) + object := buffer.Bytes() + objectStorage = fsstorage.FileSystemStorage{RootDir: rootDir} + if err = objectStorage.Put(objectPath, object); err != nil { + return err + } + return nil +} diff --git a/cmd/erasure-demo/get.go b/cmd/erasure-demo/get.go new file mode 100644 index 000000000..e989a06bd --- /dev/null +++ b/cmd/erasure-demo/get.go @@ -0,0 +1,52 @@ +package main + +import ( + "io" + "log" + "os" + + "github.com/codegangsta/cli" +) + +func get(c *cli.Context) { + config, err := parseInput(c) + if err != nil { + log.Fatal(err) + } + var objectReader io.Reader + switch config.storageDriver { + case "fs": + { + if objectReader, err = fsGet(config, c.Args().Get(0)); err != nil { + log.Fatal(err) + } + } + default: + { + log.Fatal("Unknown driver") + } + } + io.Copy(os.Stdout, objectReader) +} + +func put(c *cli.Context) { + config, err := parseInput(c) + if err != nil { + log.Fatal(err) + } + filePath := c.Args().Get(1) + inputFile, err := os.Open(filePath) + if err != nil { + log.Fatal(err) + } + switch config.storageDriver { + case "fs": + { + fsPut(config, c.Args().Get(0), inputFile) + } + default: + { + log.Fatal("Unknown driver") + } + } +} diff --git a/cmd/erasure-demo/main.go b/cmd/erasure-demo/main.go index 3831fbb4f..d9efc0ba9 100644 --- a/cmd/erasure-demo/main.go +++ b/cmd/erasure-demo/main.go @@ -2,7 +2,10 @@ package main import ( "errors" + "log" "os" + "os/user" + "path" "strconv" "strings" @@ -54,17 +57,53 @@ func main() { }, }, }, + { + Name: "get", + Usage: "get an object", + Action: get, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "root", + Value: getMinioDir(), + Usage: "", + }, + cli.StringFlag{ + Name: "driver", + Value: "fs", + Usage: "fs", + }, + }, + }, + { + Name: "put", + Usage: "put an object", + Action: put, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "root", + Value: getMinioDir(), + Usage: "", + }, + cli.StringFlag{ + Name: "driver", + Value: "fs", + Usage: "fs", + }, + }, + }, } app.Run(os.Args) } // config representing cli input type inputConfig struct { - input string - output string - k int - m int - blockSize uint64 + input string + output string + k int + m int + blockSize uint64 + rootDir string + storageDriver string } // parses input and returns an inputConfig with parsed input @@ -78,26 +117,30 @@ func parseInput(c *cli.Context) (inputConfig, error) { outputFilePath = c.String("output") } - protectionLevel := c.String("protection-level") - protectionLevelSplit := strings.Split(protectionLevel, ",") - if len(protectionLevelSplit) != 2 { - return inputConfig{}, errors.New("Malformed input for protection-level") - } - - k, err := strconv.Atoi(protectionLevelSplit[0]) - if err != nil { - return inputConfig{}, err - } + var k, m int + if c.String("protection-level") != "" { + protectionLevel := c.String("protection-level") + protectionLevelSplit := strings.Split(protectionLevel, ",") + if len(protectionLevelSplit) != 2 { + return inputConfig{}, errors.New("Malformed input for protection-level") + } - m, err := strconv.Atoi(protectionLevelSplit[1]) - if err != nil { - return inputConfig{}, err + var err error + k, err = strconv.Atoi(protectionLevelSplit[0]) + if err != nil { + return inputConfig{}, err + } + m, err = strconv.Atoi(protectionLevelSplit[1]) + if err != nil { + return inputConfig{}, err + } } var blockSize uint64 blockSize = 0 if c.String("block-size") != "" { if c.String("block-size") != "full" { + var err error blockSize, err = strbyteconv.StringToBytes(c.String("block-size")) if err != nil { return inputConfig{}, err @@ -105,11 +148,33 @@ func parseInput(c *cli.Context) (inputConfig, error) { } } - return inputConfig{ - input: inputFilePath, - output: outputFilePath, - k: k, - m: m, - blockSize: blockSize, - }, nil + var rootDir string + if c.String("root") != "" { + rootDir = c.String("root") + } + + var storageDriver string + if c.String("driver") != "" { + storageDriver = c.String("driver") + } + + config := inputConfig{ + input: inputFilePath, + output: outputFilePath, + k: k, + m: m, + blockSize: blockSize, + rootDir: rootDir, + storageDriver: storageDriver, + } + return config, nil +} + +func getMinioDir() string { + user, err := user.Current() + if err != nil { + log.Fatal(err) + } + homePath := user.HomeDir + return path.Join(homePath, ".minio") } diff --git a/cmd/erasure-demo/put.go b/cmd/erasure-demo/put.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/cmd/erasure-demo/put.go @@ -0,0 +1 @@ +package main