Adding chunking by block to erasure-demo via --block-size parameter

cmd/erasure-demo/decode.go

@@ -24,14 +24,46 @@ func decode(c *cli.Context) {
 
 	k := config.k
 	m := config.m
+
+	// check if output file exists, fail if so
+	if _, err := os.Stat(config.output); !os.IsNotExist(err) {
+		log.Fatal("Output file exists")
+	}
+
+	// get list of files
+	var inputFiles []string
+	if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) {
+		err = nil
+		chunkCount := 0
+		for !os.IsNotExist(err) {
+			_, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length")
+			chunkCount += 1
+		}
+		chunkCount = chunkCount - 1
+		inputFiles = make([]string, chunkCount)
+		for i := 0; i < chunkCount; i++ {
+			inputFiles[i] = config.input + "." + strconv.Itoa(i)
+		}
+	} else {
+		inputFiles = []string{config.input}
+	}
+
+	// open file to write
+	outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600)
+	defer outputFile.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, inputFile := range inputFiles {
 		// get chunks
 		chunks := make([][]byte, k+m)
 		for i := 0; i < k+m; i++ {
-		chunks[i], _ = ioutil.ReadFile(config.input + "." + strconv.Itoa(i))
+			chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
 		}
 
 		// get length
-	lengthBytes, err := ioutil.ReadFile(config.input + ".length")
+		lengthBytes, err := ioutil.ReadFile(inputFile + ".length")
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -49,11 +81,11 @@ func decode(c *cli.Context) {
 		if err != nil {
 			log.Fatal(err)
 		}
+		// append decoded data
+		length, err = outputFile.Write(decodedData)
+		if err != nil {
 
-	// write decode data out
+			log.Fatal(err)
-	if _, err := os.Stat(config.output); os.IsNotExist(err) {
+		}
-		ioutil.WriteFile(config.output, decodedData, 0600)
-	} else {
-		log.Fatal("Output file already exists")
 	}
 }

cmd/erasure-demo/encode.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"io/ioutil"
 	"log"
 	"os"
@@ -8,6 +9,7 @@ import (
 
 	"github.com/codegangsta/cli"
 	"github.com/minio-io/minio/pkgs/erasure"
+	"github.com/minio-io/minio/pkgs/split"
 )
 
 func encode(c *cli.Context) {
@@ -24,6 +26,7 @@ func encode(c *cli.Context) {
 
 	// get file
 	inputFile, err := os.Open(config.input)
+	defer inputFile.Close()
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -37,11 +40,27 @@ func encode(c *cli.Context) {
 	// set up encoder
 	erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
 	// encode data
+	if config.blockSize == 0 {
 		encodedData, length := erasure.Encode(input, erasureParameters)
-
-	// write encoded data out
 		for key, data := range encodedData {
 			ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
-	}
 			ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
 		}
+	} else {
+		chunkCount := 0
+		splitChannel := make(chan split.ByteMessage)
+		inputReader := bytes.NewReader(input)
+		go split.SplitStream(inputReader, config.blockSize, splitChannel)
+		for chunk := range splitChannel {
+			if chunk.Err != nil {
+				log.Fatal(chunk.Err)
+			}
+			encodedData, length := erasure.Encode(chunk.Data, erasureParameters)
+			for key, data := range encodedData {
+				ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
+				ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
+			}
+			chunkCount += 1
+		}
+	}
+}

cmd/erasure-demo/main.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/codegangsta/cli"
+	"github.com/minio-io/minio/pkgs/strbyteconv"
 )
 
 func main() {
@@ -29,6 +30,11 @@ func main() {
 					Value: "10,6",
 					Usage: "data,parity",
 				},
+				cli.StringFlag{
+					Name:  "block-size",
+					Value: "1M",
+					Usage: "Size of blocks. Examples: 1K, 1M, full",
+				},
 			},
 		},
 		{
@@ -58,7 +64,7 @@ type inputConfig struct {
 	output    string
 	k         int
 	m         int
-	blockSize int
+	blockSize uint64
 }
 
 // parses input and returns an inputConfig with parsed input
@@ -88,10 +94,22 @@ func parseInput(c *cli.Context) (inputConfig, error) {
 		return inputConfig{}, err
 	}
 
+	var blockSize uint64
+	blockSize = 0
+	if c.String("block-size") != "" {
+		if c.String("block-size") != "full" {
+			blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
+			if err != nil {
+				return inputConfig{}, err
+			}
+		}
+	}
+
 	return inputConfig{
 		input:     inputFilePath,
 		output:    outputFilePath,
 		k:         k,
 		m:         m,
+		blockSize: blockSize,
 	}, nil
 }