From 4d7c1e3b6858cbf27da8dfb5d78b41123179efad Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 29 Nov 2014 14:58:40 -0800 Subject: [PATCH] Decoding bug fixes #42 --- pkgs/erasure/common.h | 45 ++++++++++++++++++++++++++++++++++++ pkgs/erasure/decode.c | 52 +++++++++++++++++++++++++++++++++++++++--- pkgs/erasure/decode.go | 21 ++++++----------- pkgs/erasure/decode.h | 25 -------------------- pkgs/erasure/doc.go | 13 +++++------ pkgs/erasure/encode.c | 16 ++++++------- pkgs/erasure/encode.go | 4 ++-- pkgs/erasure/encode.h | 25 -------------------- 8 files changed, 116 insertions(+), 85 deletions(-) create mode 100644 pkgs/erasure/common.h delete mode 100644 pkgs/erasure/decode.h delete mode 100644 pkgs/erasure/encode.h diff --git a/pkgs/erasure/common.h b/pkgs/erasure/common.h new file mode 100644 index 000000000..925272810 --- /dev/null +++ b/pkgs/erasure/common.h @@ -0,0 +1,45 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __COMMON_H__ +#define __COMMON_H__ + +#include + +#define SIMD_ALIGN 32 + +int32_t minio_init_encoder (int technique, int k, int m, + unsigned char **encode_matrix, + unsigned char **encode_tbls); + +uint32_t minio_calc_chunk_size (int k, + uint32_t split_len); + +int32_t minio_init_decoder (int *src_err_list, + unsigned char *encoding_matrix, + unsigned char **decode_matrix, + unsigned char **decode_tbls, + int k, int n, int errs); + +int32_t minio_src_in_err (int r, int *src_err_list); + +int32_t minio_get_source_target(int *src_err_list, + int errs, int k, int m, + unsigned char **data, + unsigned char **coding, + unsigned char ***source, + unsigned char ***target); +#endif /* __COMMON_H__ */ diff --git a/pkgs/erasure/decode.c b/pkgs/erasure/decode.c index a5153e404..e6946aa80 100644 --- a/pkgs/erasure/decode.c +++ b/pkgs/erasure/decode.c @@ -19,13 +19,14 @@ #include #include -#include "decode.h" +#include "common.h" -static int src_in_err (int r, int *src_err_list) +int32_t minio_src_in_err (int r, int *src_err_list) { int i; for (i = 0; src_err_list[i] != -1; i++) { if (src_err_list[i] == r) { + // true return 1; } } @@ -33,6 +34,51 @@ static int src_in_err (int r, int *src_err_list) return 0; } +int32_t minio_get_source_target(int *src_err_list, + int errs, int k, int m, + unsigned char **data, + unsigned char **coding, + unsigned char ***source, + unsigned char ***target) +{ + int i, j, l; + unsigned char *tmp_source[k]; + unsigned char *tmp_target[m]; + + // Fill zeroes + memset (tmp_source, 0, sizeof(tmp_source)); + memset (tmp_target, 0, sizeof(tmp_target)); + + // Separate out source and target buffers from input data/coding chunks + // This separation needs to happen at error chunks from input chunks + for (i = 0, j = 0, l = 0; + ((l < k) || (j < errs)) && (i < (k + m)); i++) { + if (!minio_src_in_err(i, src_err_list)) { + if (l < k) { + if (i < k) + tmp_source[l] = + (unsigned char *) data[i]; + else + tmp_source[l] = + (unsigned char *) coding[i - k]; + l++; + } + } else { + if (j < m) { + if (i < k) + tmp_target[j] = + (unsigned char *) data[i]; + else + tmp_target[j] = + (unsigned char *) coding[i - k]; + j++; + } + } + } + *source = tmp_source; + *target = tmp_target; +} + /* Generate decode matrix during the decoding phase */ @@ -58,7 +104,7 @@ int minio_init_decoder (int *src_err_list, return -1; for (i = 0, r = 0; i < k; i++, r++) { - while (src_in_err(r, src_err_list)) + while (minio_src_in_err(r, src_err_list)) r++; for (j = 0; j < k; j++) { input_matrix[k * i + j] = encode_matrix[k * r + j]; diff --git a/pkgs/erasure/decode.go b/pkgs/erasure/decode.go index d78e0ab13..cce12a7c0 100644 --- a/pkgs/erasure/decode.go +++ b/pkgs/erasure/decode.go @@ -24,8 +24,7 @@ package erasure // #include // #include // -// #include "decode.h" -// #include "encode.h" +// #include "common.h" import "C" import ( "errors" @@ -36,6 +35,7 @@ import ( func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { var decode_matrix *C.uchar var decode_tbls *C.uchar + var source, target **C.uchar k := int(e.p.k) n := int(e.p.k + e.p.m) @@ -44,7 +44,7 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { return nil, errors.New(fmt.Sprintf("chunks length must be %d", n)) } - chunk_size := int(C.calc_chunk_size(e.k, C.uint(length))) + chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(length))) src_err_list := make([]int, n+1) var err_count int = 0 @@ -84,21 +84,14 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { pointers[i] = &chunks[i][0] } - /* - // Pack recovery array as list of valid sources - // Its order must be the same as the order - // to generate matrix b in gf_gen_decode_matrix - var i int - for i = 0; i < e.p.k; i++ { - recov[i] = buffs[decode_index[i]] - } - */ - data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0])) coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0])) + C.minio_get_source_target(src_err_list_ptr, C.int(err_count-1), + e.k, e.m, data, coding, &source, &target) + C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls, - data, coding) + source, target) recovered_output := make([]byte, 0, chunk_size*k) for i := 0; i < k; i++ { diff --git a/pkgs/erasure/decode.h b/pkgs/erasure/decode.h deleted file mode 100644 index 4c3844a78..000000000 --- a/pkgs/erasure/decode.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Mini Object Storage, (C) 2014 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __DECODE_H__ -#define __DECODE_H__ - -int minio_init_decoder (int *src_err_list, - unsigned char *encoding_matrix, - unsigned char **decode_matrix, - unsigned char **decode_tbls, - int k, int n, int errs); -#endif /* __DECODE_H__ */ diff --git a/pkgs/erasure/doc.go b/pkgs/erasure/doc.go index 33cb4a1dd..f458ce415 100644 --- a/pkgs/erasure/doc.go +++ b/pkgs/erasure/doc.go @@ -13,37 +13,36 @@ // // Encoding data may be performed in 3 steps. // -// 1. Create a validated set of encoder parameters +// 1. Create a parse set of encoder parameters // 2. Create a new encoder // 3. Encode data // // Decoding data is also performed in 3 steps. // -// 1. Create a validated set of encoder parameters +// 1. Create a parse set of encoder parameters for validation // 2. Create a new encoder // 3. Decode data // // Encoder parameters contain four configurable elements: -// ValidateParams(k, m, w, technique int) (EncoderParams, error) +// ParseEncoderParams(k, m, technique int) (EncoderParams, error) // k - Number of rows in matrix // m - Number of colums in matrix -// w - Word size, typically 8 or aligned by 8. // technique - Matrix type, can be either CAUCHY (recommended) or VANDERMONDE -// constraints: k + m < 256 +// constraints: k + m < Galois Field (2^8) // // Example // // // Creating and using an encoder // var bytes []byte -// params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY) +// params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY) // encoder := erasure.NewEncoder(params) // encodedData, length := encoder.Encode(bytes) // // Creating and using a decoder // var encodedData [][]byte // var length int -// params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY) +// params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY) // encoder := erasure.NewEncoder(params) // originalData, err := encoder.Decode(encodedData, length) package erasure diff --git a/pkgs/erasure/encode.c b/pkgs/erasure/encode.c index 756f82da2..173d02cc4 100644 --- a/pkgs/erasure/encode.c +++ b/pkgs/erasure/encode.c @@ -18,11 +18,11 @@ #include #include -#include "encode.h" +#include "common.h" -void minio_init_encoder (int technique, int k, int m, - unsigned char **encode_matrix, - unsigned char **encode_tbls) +int32_t minio_init_encoder (int technique, int k, int m, + unsigned char **encode_matrix, + unsigned char **encode_tbls) { size_t encode_matrix_size; size_t encode_tbls_size; @@ -52,9 +52,11 @@ void minio_init_encoder (int technique, int k, int m, *encode_matrix = tmp_matrix; *encode_tbls = tmp_tbls; + + return 0; } -unsigned int calc_chunk_size (int k, unsigned int split_len) +uint32_t minio_calc_chunk_size (int k, uint32_t split_len) { int alignment; int remainder; @@ -69,7 +71,3 @@ unsigned int calc_chunk_size (int k, unsigned int split_len) } return padded_len / k; } -/* -void minio_encode (int k, int m, ) -{ -*/ diff --git a/pkgs/erasure/encode.go b/pkgs/erasure/encode.go index 32ce9dbf0..97511f7aa 100644 --- a/pkgs/erasure/encode.go +++ b/pkgs/erasure/encode.go @@ -24,7 +24,7 @@ package erasure // #include // #include // -// #include "encode.h" +// #include "common.h" import "C" import ( "errors" @@ -123,7 +123,7 @@ func NewEncoder(ep *EncoderParams) *Encoder { func (e *Encoder) Encode(block []byte) ([][]byte, int) { var block_len = len(block) - chunk_size := int(C.calc_chunk_size(e.k, C.uint(block_len))) + chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(block_len))) chunk_len := chunk_size * e.p.k pad_len := chunk_len - block_len diff --git a/pkgs/erasure/encode.h b/pkgs/erasure/encode.h deleted file mode 100644 index aded47dc1..000000000 --- a/pkgs/erasure/encode.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Mini Object Storage, (C) 2014 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __ENCODE_H__ -#define __ENCODE_H__ - -#define SIMD_ALIGN 32 -void minio_init_encoder (int technique, int k, int m, - unsigned char **encode_matrix, - unsigned char **encode_tbls); -unsigned int calc_chunk_size (int k, unsigned int split_len); -#endif /* __ENCODE_H__ */