Merge pull request #100 from harshavardhana/pr_out_add_crc32c_checksum_support_for_erasure_chunks

Add crc32c checksum support for erasure chunks
master
Harshavardhana 10 years ago
commit 9dd3dd8b53
  1. 17
      cmd/erasure-demo/decode.go
  2. 15
      cmd/erasure-demo/encode.go

@ -7,6 +7,7 @@ import (
"strconv"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/checksum/crc32c"
"github.com/minio-io/minio/pkgs/erasure"
)
@ -58,8 +59,23 @@ func decode(c *cli.Context) {
for _, inputFile := range inputFiles {
// get chunks
chunks := make([][]byte, k+m)
// get checksum
cksums := make([][]byte, k+m)
for i := 0; i < k+m; i++ {
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
cksums[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i) + ".cksum")
}
for i := 0; i < k+m; i++ {
crcChunk, err := crc32c.Crc32c(chunks[i])
if err != nil {
chunks[i] = nil
continue
}
crcChunkStr := strconv.Itoa(int(crcChunk))
if string(cksums[i]) != crcChunkStr {
chunks[i] = nil
}
}
// get length
@ -67,6 +83,7 @@ func decode(c *cli.Context) {
if err != nil {
log.Fatal(err)
}
lengthString := string(lengthBytes)
length, err := strconv.Atoi(lengthString)
if err != nil {

@ -8,6 +8,7 @@ import (
"strconv"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/checksum/crc32c"
"github.com/minio-io/minio/pkgs/erasure"
"github.com/minio-io/minio/pkgs/split"
)
@ -47,6 +48,12 @@ func encode(c *cli.Context) {
if config.blockSize == 0 {
encodedData, length := encoder.Encode(input)
for key, data := range encodedData {
crc, err := crc32c.Crc32c(data)
if err != nil {
log.Fatal(err)
}
crcString := strconv.Itoa(int(crc))
ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
}
@ -61,9 +68,15 @@ func encode(c *cli.Context) {
}
encodedData, length := encoder.Encode(chunk.Data)
for key, data := range encodedData {
crc, err := crc32c.Crc32c(data)
if err != nil {
log.Fatal(err)
}
crcString := strconv.Itoa(int(crc))
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
}
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
chunkCount += 1
}
}

Loading…
Cancel
Save