Merge pull request #100 from harshavardhana/pr_out_add_crc32c_checksum_support_for_erasure_chunks

Add crc32c checksum support for erasure chunks

cmd/erasure-demo/decode.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 
 	"github.com/codegangsta/cli"
+	"github.com/minio-io/minio/pkgs/checksum/crc32c"
 	"github.com/minio-io/minio/pkgs/erasure"
 )
 
@@ -58,8 +59,23 @@ func decode(c *cli.Context) {
 	for _, inputFile := range inputFiles {
 		// get chunks
 		chunks := make([][]byte, k+m)
+		// get checksum
+		cksums := make([][]byte, k+m)
 		for i := 0; i < k+m; i++ {
 			chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
+			cksums[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i) + ".cksum")
+		}
+
+		for i := 0; i < k+m; i++ {
+			crcChunk, err := crc32c.Crc32c(chunks[i])
+			if err != nil {
+				chunks[i] = nil
+				continue
+			}
+			crcChunkStr := strconv.Itoa(int(crcChunk))
+			if string(cksums[i]) != crcChunkStr {
+				chunks[i] = nil
+			}
 		}
 
 		// get length
@@ -67,6 +83,7 @@ func decode(c *cli.Context) {
 		if err != nil {
 			log.Fatal(err)
 		}
+
 		lengthString := string(lengthBytes)
 		length, err := strconv.Atoi(lengthString)
 		if err != nil {

cmd/erasure-demo/encode.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 
 	"github.com/codegangsta/cli"
+	"github.com/minio-io/minio/pkgs/checksum/crc32c"
 	"github.com/minio-io/minio/pkgs/erasure"
 	"github.com/minio-io/minio/pkgs/split"
 )
@@ -47,6 +48,12 @@ func encode(c *cli.Context) {
 	if config.blockSize == 0 {
 		encodedData, length := encoder.Encode(input)
 		for key, data := range encodedData {
+			crc, err := crc32c.Crc32c(data)
+			if err != nil {
+				log.Fatal(err)
+			}
+			crcString := strconv.Itoa(int(crc))
+			ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
 			ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
 			ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
 		}
@@ -61,9 +68,15 @@ func encode(c *cli.Context) {
 			}
 			encodedData, length := encoder.Encode(chunk.Data)
 			for key, data := range encodedData {
+				crc, err := crc32c.Crc32c(data)
+				if err != nil {
+					log.Fatal(err)
+				}
+				crcString := strconv.Itoa(int(crc))
+				ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
 				ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
-				ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
 			}
+			ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
 			chunkCount += 1
 		}
 	}