From 46b08681a44839a9fd4274272769e9ca8812f807 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Mon, 1 Dec 2014 21:06:36 -0800 Subject: [PATCH] Adding chunking by block to erasure-demo via --block-size parameter --- cmd/erasure-demo/decode.go | 76 +++++++++++++++++++++++++++----------- cmd/erasure-demo/encode.go | 31 +++++++++++++--- cmd/erasure-demo/main.go | 28 +++++++++++--- 3 files changed, 102 insertions(+), 33 deletions(-) diff --git a/cmd/erasure-demo/decode.go b/cmd/erasure-demo/decode.go index a36190b03..4cecca563 100644 --- a/cmd/erasure-demo/decode.go +++ b/cmd/erasure-demo/decode.go @@ -24,36 +24,68 @@ func decode(c *cli.Context) { k := config.k m := config.m - // get chunks - chunks := make([][]byte, k+m) - for i := 0; i < k+m; i++ { - chunks[i], _ = ioutil.ReadFile(config.input + "." + strconv.Itoa(i)) + + // check if output file exists, fail if so + if _, err := os.Stat(config.output); !os.IsNotExist(err) { + log.Fatal("Output file exists") } - // get length - lengthBytes, err := ioutil.ReadFile(config.input + ".length") - if err != nil { - log.Fatal(err) + // get list of files + var inputFiles []string + if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) { + err = nil + chunkCount := 0 + for !os.IsNotExist(err) { + _, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length") + chunkCount += 1 + } + chunkCount = chunkCount - 1 + inputFiles = make([]string, chunkCount) + for i := 0; i < chunkCount; i++ { + inputFiles[i] = config.input + "." + strconv.Itoa(i) + } + } else { + inputFiles = []string{config.input} } - lengthString := string(lengthBytes) - length, err := strconv.Atoi(lengthString) + + // open file to write + outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600) + defer outputFile.Close() if err != nil { log.Fatal(err) } - // set up encoder - erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY) + for _, inputFile := range inputFiles { + // get chunks + chunks := make([][]byte, k+m) + for i := 0; i < k+m; i++ { + chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i)) + } - // decode data - decodedData, err := erasure.Decode(chunks, erasureParameters, length) - if err != nil { - log.Fatal(err) - } + // get length + lengthBytes, err := ioutil.ReadFile(inputFile + ".length") + if err != nil { + log.Fatal(err) + } + lengthString := string(lengthBytes) + length, err := strconv.Atoi(lengthString) + if err != nil { + log.Fatal(err) + } - // write decode data out - if _, err := os.Stat(config.output); os.IsNotExist(err) { - ioutil.WriteFile(config.output, decodedData, 0600) - } else { - log.Fatal("Output file already exists") + // set up encoder + erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY) + + // decode data + decodedData, err := erasure.Decode(chunks, erasureParameters, length) + if err != nil { + log.Fatal(err) + } + // append decoded data + length, err = outputFile.Write(decodedData) + if err != nil { + + log.Fatal(err) + } } } diff --git a/cmd/erasure-demo/encode.go b/cmd/erasure-demo/encode.go index 72e13ac75..b33f731ed 100644 --- a/cmd/erasure-demo/encode.go +++ b/cmd/erasure-demo/encode.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "io/ioutil" "log" "os" @@ -8,6 +9,7 @@ import ( "github.com/codegangsta/cli" "github.com/minio-io/minio/pkgs/erasure" + "github.com/minio-io/minio/pkgs/split" ) func encode(c *cli.Context) { @@ -24,6 +26,7 @@ func encode(c *cli.Context) { // get file inputFile, err := os.Open(config.input) + defer inputFile.Close() if err != nil { log.Fatal(err) } @@ -37,11 +40,27 @@ func encode(c *cli.Context) { // set up encoder erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY) // encode data - encodedData, length := erasure.Encode(input, erasureParameters) - - // write encoded data out - for key, data := range encodedData { - ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600) + if config.blockSize == 0 { + encodedData, length := erasure.Encode(input, erasureParameters) + for key, data := range encodedData { + ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600) + ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600) + } + } else { + chunkCount := 0 + splitChannel := make(chan split.ByteMessage) + inputReader := bytes.NewReader(input) + go split.SplitStream(inputReader, config.blockSize, splitChannel) + for chunk := range splitChannel { + if chunk.Err != nil { + log.Fatal(chunk.Err) + } + encodedData, length := erasure.Encode(chunk.Data, erasureParameters) + for key, data := range encodedData { + ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600) + ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600) + } + chunkCount += 1 + } } - ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600) } diff --git a/cmd/erasure-demo/main.go b/cmd/erasure-demo/main.go index 244608cd2..3831fbb4f 100644 --- a/cmd/erasure-demo/main.go +++ b/cmd/erasure-demo/main.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/codegangsta/cli" + "github.com/minio-io/minio/pkgs/strbyteconv" ) func main() { @@ -29,6 +30,11 @@ func main() { Value: "10,6", Usage: "data,parity", }, + cli.StringFlag{ + Name: "block-size", + Value: "1M", + Usage: "Size of blocks. Examples: 1K, 1M, full", + }, }, }, { @@ -58,7 +64,7 @@ type inputConfig struct { output string k int m int - blockSize int + blockSize uint64 } // parses input and returns an inputConfig with parsed input @@ -88,10 +94,22 @@ func parseInput(c *cli.Context) (inputConfig, error) { return inputConfig{}, err } + var blockSize uint64 + blockSize = 0 + if c.String("block-size") != "" { + if c.String("block-size") != "full" { + blockSize, err = strbyteconv.StringToBytes(c.String("block-size")) + if err != nil { + return inputConfig{}, err + } + } + } + return inputConfig{ - input: inputFilePath, - output: outputFilePath, - k: k, - m: m, + input: inputFilePath, + output: outputFilePath, + k: k, + m: m, + blockSize: blockSize, }, nil }