Merge pull request #107 from harshavardhana/pr_out_storage_erasure_encoding_info_at_storageentry_and_also_store_crc_at_blockheader

master
Harshavardhana 10 years ago
commit e107c3204d
  1. 34
      pkgs/erasure/encode.go
  2. 5
      pkgs/storage/appendstorage/append_storage.go
  3. 23
      pkgs/storage/encodedstorage/encoded_storage.go

@ -43,9 +43,9 @@ const (
// EncoderParams is a configuration set for building an encoder. It is created using ValidateParams.
type EncoderParams struct {
k,
m,
technique int // cauchy or vandermonde matrix (RS)
K,
M,
Technique int // cauchy or vandermonde matrix (RS)
}
// Encoder is an object used to encode and decode data.
@ -87,21 +87,21 @@ func ParseEncoderParams(k, m, technique int) (*EncoderParams, error) {
}
return &EncoderParams{
k: k,
m: m,
technique: technique,
K: k,
M: m,
Technique: technique,
}, nil
}
// NewEncoder creates an encoder with a given set of parameters.
func NewEncoder(ep *EncoderParams) *Encoder {
var k = C.int(ep.k)
var m = C.int(ep.m)
var k = C.int(ep.K)
var m = C.int(ep.M)
var encode_matrix *C.uint8_t
var encode_tbls *C.uint8_t
C.minio_init_encoder(C.int(ep.technique), k, m, &encode_matrix,
C.minio_init_encoder(C.int(ep.Technique), k, m, &encode_matrix,
&encode_tbls)
return &Encoder{
@ -122,7 +122,7 @@ func (e *Encoder) Encode(block []byte) ([][]byte, int) {
var block_len = len(block)
chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(block_len)))
chunk_len := chunk_size * e.p.k
chunk_len := chunk_size * e.p.K
pad_len := chunk_len - block_len
if pad_len > 0 {
@ -131,28 +131,28 @@ func (e *Encoder) Encode(block []byte) ([][]byte, int) {
block = append(block, s...)
}
coded_len := chunk_size * e.p.m
coded_len := chunk_size * e.p.M
c := make([]byte, coded_len)
block = append(block, c...)
// Allocate chunks
chunks := make([][]byte, e.p.k+e.p.m)
pointers := make([]*byte, e.p.k+e.p.m)
chunks := make([][]byte, e.p.K+e.p.M)
pointers := make([]*byte, e.p.K+e.p.M)
var i int
// Add data blocks to chunks
for i = 0; i < e.p.k; i++ {
for i = 0; i < e.p.K; i++ {
chunks[i] = block[i*chunk_size : (i+1)*chunk_size]
pointers[i] = &chunks[i][0]
}
for i = e.p.k; i < (e.p.k + e.p.m); i++ {
for i = e.p.K; i < (e.p.K + e.p.M); i++ {
chunks[i] = make([]byte, chunk_size)
pointers[i] = &chunks[i][0]
}
data := (**C.uint8_t)(unsafe.Pointer(&pointers[:e.p.k][0]))
coding := (**C.uint8_t)(unsafe.Pointer(&pointers[e.p.k:][0]))
data := (**C.uint8_t)(unsafe.Pointer(&pointers[:e.p.K][0]))
coding := (**C.uint8_t)(unsafe.Pointer(&pointers[e.p.K:][0]))
C.ec_encode_data(C.int(chunk_size), e.k, e.m, e.encode_tbls, data,
coding)

@ -10,6 +10,7 @@ import (
"path"
"strconv"
"github.com/minio-io/minio/pkgs/checksum/crc32c"
"github.com/minio-io/minio/pkgs/storage"
)
@ -96,6 +97,10 @@ func (aStorage *appendStorage) Put(objectPath string, object io.Reader) error {
}
header.Offset = offset
header.Length = len(objectBytes)
header.Crc, err = crc32c.Crc32c(objectBytes)
if err != nil {
return err
}
aStorage.objects[objectPath] = header
var mapBuffer bytes.Buffer
encoder := gob.NewEncoder(&mapBuffer)

@ -28,9 +28,8 @@ type encodedStorage struct {
type StorageEntry struct {
Path string
Md5sum string
Crc uint32
Blocks []StorageBlockEntry
Encoderparams *erasure.EncoderParams
Encoderparams erasure.EncoderParams
}
type StorageBlockEntry struct {
@ -109,11 +108,14 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
}
encoder := erasure.NewEncoder(encoderParameters)
entry := StorageEntry{
Path: objectPath,
Md5sum: "md5sum",
Crc: 0,
Blocks: make([]StorageBlockEntry, 0),
Encoderparams: encoderParameters,
Path: objectPath,
Md5sum: "md5sum",
Blocks: make([]StorageBlockEntry, 0),
Encoderparams: erasure.EncoderParams{
K: eStorage.K,
M: eStorage.M,
Technique: erasure.CAUCHY,
},
}
i := 0
// encode
@ -163,7 +165,12 @@ func (eStorage *encodedStorage) storeBlocks(path string, blocks [][]byte) []erro
}
func (eStorage *encodedStorage) readObject(objectPath string, entry StorageEntry, writer *io.PipeWriter) {
encoder := erasure.NewEncoder(entry.Encoderparams)
ep, err := erasure.ParseEncoderParams(entry.Encoderparams.K, entry.Encoderparams.M, entry.Encoderparams.Technique)
if err != nil {
writer.CloseWithError(err)
return
}
encoder := erasure.NewEncoder(ep)
for i, chunk := range entry.Blocks {
blockSlices := eStorage.getBlockSlices(objectPath + "$" + strconv.Itoa(i))
var blocks [][]byte

Loading…
Cancel
Save