From 3c0902904930f169231a0ea25cb0d705352e5679 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Sat, 29 Nov 2014 22:48:37 -0800 Subject: [PATCH] Merging encode and decode to single minio-demo binary --- Makefile | 3 +- cmd/minio-decode/main.go | 90 ---------------------------------------- cmd/minio-demo/decode.go | 78 ++++++++++++++++++++++++++++++++++ cmd/minio-demo/encode.go | 68 ++++++++++++++++++++++++++++++ cmd/minio-demo/main.go | 50 ++++++++++++++++++++++ cmd/minio-encode/main.go | 84 ------------------------------------- 6 files changed, 197 insertions(+), 176 deletions(-) delete mode 100644 cmd/minio-decode/main.go create mode 100644 cmd/minio-demo/decode.go create mode 100644 cmd/minio-demo/encode.go create mode 100644 cmd/minio-demo/main.go delete mode 100644 cmd/minio-encode/main.go diff --git a/Makefile b/Makefile index 27aa61b49..65ea56850 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,7 @@ test: build-erasure build-signify godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway install: build-erasure - godep go install github.com/minio-io/minio/cmd/minio-encode - godep go install github.com/minio-io/minio/cmd/minio-decode + godep go install github.com/minio-io/minio/cmd/minio-demo save: godep save ./... diff --git a/cmd/minio-decode/main.go b/cmd/minio-decode/main.go deleted file mode 100644 index 29ff43e05..000000000 --- a/cmd/minio-decode/main.go +++ /dev/null @@ -1,90 +0,0 @@ -package main - -import ( - "io/ioutil" - "log" - "os" - "strconv" - - "github.com/codegangsta/cli" - "github.com/minio-io/minio/pkgs/erasure" -) - -func main() { - app := cli.NewApp() - app.Name = "minio-decode" - app.Usage = "erasure decode a byte stream" - app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "input,i", - Value: "", - Usage: "Input file", - }, - cli.StringFlag{ - Name: "output,o", - Value: "", - Usage: "Output file", - }, - cli.IntFlag{ - Name: "k", - Value: 10, - Usage: "k value of encoder parameters", - }, - cli.IntFlag{ - Name: "m", - Value: 5, - Usage: "m value of encoder parameters", - }, - } - app.Action = func(c *cli.Context) { - // check if minio-encode called without parameters - if len(c.Args()) == 1 { - cli.ShowAppHelp(c) - } - - // get input path - if c.String("input") == "" { - log.Fatal("No input specified") - } - inputFilePath := c.String("input") - - // get output path - outputFilePath := inputFilePath - if c.String("output") != "" { - outputFilePath = c.String("output") - } - - k := c.Int("k") - m := c.Int("m") - - // get chunks - chunks := make([][]byte, k+m) - for i := 0; i < k+m; i++ { - chunks[i], _ = ioutil.ReadFile(inputFilePath + "." + strconv.Itoa(i)) - } - - // get length - lengthBytes, err := ioutil.ReadFile(inputFilePath + ".length") - if err != nil { - log.Fatal(err) - } - lengthString := string(lengthBytes) - length, err := strconv.Atoi(lengthString) - if err != nil { - log.Fatal(err) - } - - // set up encoder - erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY) - - // decode data - decodedData, err := erasure.Decode(chunks, erasureParameters, length) - if err != nil { - log.Fatal(err) - } - - // write decode data out - ioutil.WriteFile(outputFilePath, decodedData, 0600) - } - app.Run(os.Args) -} diff --git a/cmd/minio-demo/decode.go b/cmd/minio-demo/decode.go new file mode 100644 index 000000000..db96c2833 --- /dev/null +++ b/cmd/minio-demo/decode.go @@ -0,0 +1,78 @@ +package main + +import ( + "io/ioutil" + "log" + "os" + "strconv" + "strings" + + "github.com/codegangsta/cli" + "github.com/minio-io/minio/pkgs/erasure" +) + +func decode(c *cli.Context) { + // check if minio-encode called without parameters + if len(c.Args()) != 1 { + cli.ShowCommandHelp(c, "decode") + return + } + + // get input path + inputFilePath := c.Args().Get(0) + + // get output path + outputFilePath := inputFilePath + if c.String("output") != "" { + outputFilePath = c.String("output") + } + + protectionLevel := c.String("protection-level") + protectionLevelSplit := strings.Split(protectionLevel, ",") + if len(protectionLevelSplit) != 2 { + log.Fatal("Malformed input for protection-level") + } + + k, err := strconv.Atoi(protectionLevelSplit[0]) + if err != nil { + log.Fatal(err) + } + + m, err := strconv.Atoi(protectionLevelSplit[1]) + if err != nil { + log.Fatal(err) + } + + // get chunks + chunks := make([][]byte, k+m) + for i := 0; i < k+m; i++ { + chunks[i], _ = ioutil.ReadFile(inputFilePath + "." + strconv.Itoa(i)) + } + + // get length + lengthBytes, err := ioutil.ReadFile(inputFilePath + ".length") + if err != nil { + log.Fatal(err) + } + lengthString := string(lengthBytes) + length, err := strconv.Atoi(lengthString) + if err != nil { + log.Fatal(err) + } + + // set up encoder + erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY) + + // decode data + decodedData, err := erasure.Decode(chunks, erasureParameters, length) + if err != nil { + log.Fatal(err) + } + + // write decode data out + if _, err := os.Stat(outputFilePath); os.IsNotExist(err) { + ioutil.WriteFile(outputFilePath, decodedData, 0600) + } else { + log.Fatal("Output file already exists") + } +} diff --git a/cmd/minio-demo/encode.go b/cmd/minio-demo/encode.go new file mode 100644 index 000000000..cd2e67685 --- /dev/null +++ b/cmd/minio-demo/encode.go @@ -0,0 +1,68 @@ +package main + +import ( + "io/ioutil" + "log" + "os" + "strconv" + "strings" + + "github.com/codegangsta/cli" + "github.com/minio-io/minio/pkgs/erasure" +) + +func encode(c *cli.Context) { + // check if minio-encode called without parameters + if len(c.Args()) != 1 { + cli.ShowCommandHelp(c, "encode") + return + } + + // get input path + inputFilePath := c.Args().Get(0) + + // get output path + outputFilePath := inputFilePath + if c.String("output") != "" { + outputFilePath = c.String("output") + } + + protectionLevel := c.String("protection-level") + protectionLevelSplit := strings.Split(protectionLevel, ",") + if len(protectionLevelSplit) != 2 { + log.Fatal("Malformed input for protection-level") + } + + k, err := strconv.Atoi(protectionLevelSplit[0]) + if err != nil { + log.Fatal(err) + } + + m, err := strconv.Atoi(protectionLevelSplit[1]) + if err != nil { + log.Fatal(err) + } + + // get file + inputFile, err := os.Open(inputFilePath) + if err != nil { + log.Fatal(err) + } + + // read file + input, err := ioutil.ReadAll(inputFile) + if err != nil { + log.Fatal(err) + } + + // set up encoder + erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY) + // encode data + encodedData, length := erasure.Encode(input, erasureParameters) + + // write encoded data out + for key, data := range encodedData { + ioutil.WriteFile(outputFilePath+"."+strconv.Itoa(key), data, 0600) + } + ioutil.WriteFile(outputFilePath+".length", []byte(strconv.Itoa(length)), 0600) +} diff --git a/cmd/minio-demo/main.go b/cmd/minio-demo/main.go new file mode 100644 index 000000000..2b50b2390 --- /dev/null +++ b/cmd/minio-demo/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "os" + + "github.com/codegangsta/cli" +) + +func main() { + app := cli.NewApp() + app.Name = "minio-encode" + app.Usage = "erasure encode a byte stream" + app.Commands = []cli.Command{ + { + Name: "encode", + Usage: "erasure encode a byte stream", + Action: encode, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "output,o", + Value: "", + Usage: "Output file", + }, + cli.StringFlag{ + Name: "protection-level", + Value: "10,5", + Usage: "data,parity", + }, + }, + }, + { + Name: "decode", + Usage: "erasure decode a byte stream", + Action: decode, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "output,o", + Value: "", + Usage: "Output file", + }, + cli.StringFlag{ + Name: "protection-level", + Value: "10,5", + Usage: "data,parity", + }, + }, + }, + } + app.Run(os.Args) +} diff --git a/cmd/minio-encode/main.go b/cmd/minio-encode/main.go deleted file mode 100644 index 3eaa8fe3f..000000000 --- a/cmd/minio-encode/main.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import ( - "io/ioutil" - "log" - "os" - "strconv" - - "github.com/codegangsta/cli" - "github.com/minio-io/minio/pkgs/erasure" -) - -func main() { - app := cli.NewApp() - app.Name = "minio-encode" - app.Usage = "erasure encode a byte stream" - app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "input,i", - Value: "", - Usage: "Input file", - }, - cli.StringFlag{ - Name: "output,o", - Value: "", - Usage: "Output file", - }, - cli.IntFlag{ - Name: "k", - Value: 10, - Usage: "k value of encoder parameters", - }, - cli.IntFlag{ - Name: "m", - Value: 5, - Usage: "m value of encoder parameters", - }, - } - app.Action = func(c *cli.Context) { - // check if minio-encode called without parameters - if len(c.Args()) == 1 { - cli.ShowAppHelp(c) - } - - // get input path - if c.String("input") == "" { - log.Fatal("No input specified") - } - inputFilePath := c.String("input") - - // get output path - outputFilePath := inputFilePath - if c.String("output") != "" { - outputFilePath = c.String("output") - } - - k := c.Int("k") - m := c.Int("m") - - // get file - inputFile, err := os.Open(inputFilePath) - if err != nil { - log.Fatal(err) - } - - // read file - input, err := ioutil.ReadAll(inputFile) - if err != nil { - log.Fatal(err) - } - - // set up encoder - erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY) - // encode data - encodedData, length := erasure.Encode(input, erasureParameters) - - // write encoded data out - for key, data := range encodedData { - ioutil.WriteFile(outputFilePath+"."+strconv.Itoa(key), data, 0600) - } - ioutil.WriteFile(outputFilePath+".length", []byte(strconv.Itoa(length)), 0600) - } - app.Run(os.Args) -}