From f4b55194513266aaea8a2c407652295ec8a4d0b3 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 23 Feb 2015 13:39:16 -0800 Subject: [PATCH] Update erasure documentation --- pkg/storage/erasure/doc.go | 30 ++++++++++++++++++++++----- pkg/storage/erasure/erasure_decode.go | 4 ++++ pkg/storage/erasure/erasure_encode.go | 6 +++--- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pkg/storage/erasure/doc.go b/pkg/storage/erasure/doc.go index 4e20398c1..e56801a52 100644 --- a/pkg/storage/erasure/doc.go +++ b/pkg/storage/erasure/doc.go @@ -1,15 +1,20 @@ // Package erasure is a Go wrapper for the Intel Intelligent Storage // Acceleration Library (Intel ISA-L). Intel ISA-L is a CPU optimized -// implementation of erasure code. +// implementation of erasure coding algorithms. // // For more information on Intel ISA-L, please visit: // https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version // -// Usage +// Usage: // -// TODO: Explain Encode and Decode inputs and outputs +// Encode encodes a block of data. The input is the original data. The output +// is a 2 tuple containing (k + m) chunks of erasure encoded data and the +// length of the original object. // -// TODO: Explain matrix size and how it corresponds to protection level +// Decode decodes 2 tuple data containing (k + m) chunks back into its original form. +// Additionally original block length should also be provided as input. +// +// Decoded data is exactly similar in length and content as the original data. // // Encoding data may be performed in 3 steps. // @@ -23,13 +28,27 @@ // 2. Create a new encoder // 3. Decode data // -// Encoder parameters contain four configurable elements: +// Encoder parameters contain three configurable elements: // ParseEncoderParams(k, m, technique int) (EncoderParams, error) // k - Number of rows in matrix // m - Number of colums in matrix // technique - Matrix type, can be either CAUCHY (recommended) or VANDERMONDE // constraints: k + m < Galois Field (2^8) // +// Choosing right parity and matrix technique is left for application to decide. +// +// But here are the few points to keep in mind +// +// Techniques: +// - Vandermonde is most commonly used method for choosing coefficients in erasure +// encoding but does not guarantee invertable for every sub matrix. +// Users may want to adjust for k > 5. (k is data blocks) +// - Whereas Cauchy is our recommended method for choosing coefficients in erasure coding. +// Since any sub-matrix of a Cauchy matrix is invertable. +// +// Total blocks: +// - Data blocks and Parity blocks should not be greater than 'Galois Field' (2^8) +// // Example // // Creating and using an encoder @@ -44,4 +63,5 @@ // params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY) // encoder := erasure.NewEncoder(params) // originalData, err := encoder.Decode(encodedData, length) +// package erasure diff --git a/pkg/storage/erasure/erasure_decode.go b/pkg/storage/erasure/erasure_decode.go index 4e821ca0f..f72e3312e 100644 --- a/pkg/storage/erasure/erasure_decode.go +++ b/pkg/storage/erasure/erasure_decode.go @@ -27,6 +27,10 @@ import ( "unsafe" ) +// Decode decodes 2 tuple data containing (k + m) chunks back into its original form. +// Additionally original block length should also be provided as input. +// +// Decoded data is exactly similar in length and content as the original data. func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { var decode_matrix *C.uint8_t var decode_tbls *C.uint8_t diff --git a/pkg/storage/erasure/erasure_encode.go b/pkg/storage/erasure/erasure_encode.go index 637e4e6b1..a9137cade 100644 --- a/pkg/storage/erasure/erasure_encode.go +++ b/pkg/storage/erasure/erasure_encode.go @@ -56,9 +56,9 @@ type Encoder struct { // ParseEncoderParams creates an EncoderParams object. // -// k and n represent the matrix size, which corresponds to the protection level. -// +// k and m represent the matrix size, which corresponds to the protection level // technique is the matrix type. Valid inputs are CAUCHY (recommended) or VANDERMONDE. +// func ParseEncoderParams(k, m, technique int) (*EncoderParams, error) { if k < 1 { return nil, errors.New("k cannot be zero") @@ -88,7 +88,7 @@ func ParseEncoderParams(k, m, technique int) (*EncoderParams, error) { }, nil } -// NewEncoder creates an encoder with a given set of parameters. +// NewEncoder creates an encoder object with a given set of parameters. func NewEncoder(ep *EncoderParams) *Encoder { var k = C.int(ep.K) var m = C.int(ep.M)