Decoding bug fixes #42

master
Harshavardhana 10 years ago
parent 5e0c5089e6
commit 4d7c1e3b68
  1. 45
      pkgs/erasure/common.h
  2. 52
      pkgs/erasure/decode.c
  3. 21
      pkgs/erasure/decode.go
  4. 25
      pkgs/erasure/decode.h
  5. 13
      pkgs/erasure/doc.go
  6. 16
      pkgs/erasure/encode.c
  7. 4
      pkgs/erasure/encode.go
  8. 25
      pkgs/erasure/encode.h

@ -0,0 +1,45 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __COMMON_H__
#define __COMMON_H__
#include <stdint.h>
#define SIMD_ALIGN 32
int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls);
uint32_t minio_calc_chunk_size (int k,
uint32_t split_len);
int32_t minio_init_decoder (int *src_err_list,
unsigned char *encoding_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
int k, int n, int errs);
int32_t minio_src_in_err (int r, int *src_err_list);
int32_t minio_get_source_target(int *src_err_list,
int errs, int k, int m,
unsigned char **data,
unsigned char **coding,
unsigned char ***source,
unsigned char ***target);
#endif /* __COMMON_H__ */

@ -19,13 +19,14 @@
#include <string.h> #include <string.h>
#include <erasure-code.h> #include <erasure-code.h>
#include "decode.h" #include "common.h"
static int src_in_err (int r, int *src_err_list) int32_t minio_src_in_err (int r, int *src_err_list)
{ {
int i; int i;
for (i = 0; src_err_list[i] != -1; i++) { for (i = 0; src_err_list[i] != -1; i++) {
if (src_err_list[i] == r) { if (src_err_list[i] == r) {
// true
return 1; return 1;
} }
} }
@ -33,6 +34,51 @@ static int src_in_err (int r, int *src_err_list)
return 0; return 0;
} }
int32_t minio_get_source_target(int *src_err_list,
int errs, int k, int m,
unsigned char **data,
unsigned char **coding,
unsigned char ***source,
unsigned char ***target)
{
int i, j, l;
unsigned char *tmp_source[k];
unsigned char *tmp_target[m];
// Fill zeroes
memset (tmp_source, 0, sizeof(tmp_source));
memset (tmp_target, 0, sizeof(tmp_target));
// Separate out source and target buffers from input data/coding chunks
// This separation needs to happen at error chunks from input chunks
for (i = 0, j = 0, l = 0;
((l < k) || (j < errs)) && (i < (k + m)); i++) {
if (!minio_src_in_err(i, src_err_list)) {
if (l < k) {
if (i < k)
tmp_source[l] =
(unsigned char *) data[i];
else
tmp_source[l] =
(unsigned char *) coding[i - k];
l++;
}
} else {
if (j < m) {
if (i < k)
tmp_target[j] =
(unsigned char *) data[i];
else
tmp_target[j] =
(unsigned char *) coding[i - k];
j++;
}
}
}
*source = tmp_source;
*target = tmp_target;
}
/* /*
Generate decode matrix during the decoding phase Generate decode matrix during the decoding phase
*/ */
@ -58,7 +104,7 @@ int minio_init_decoder (int *src_err_list,
return -1; return -1;
for (i = 0, r = 0; i < k; i++, r++) { for (i = 0, r = 0; i < k; i++, r++) {
while (src_in_err(r, src_err_list)) while (minio_src_in_err(r, src_err_list))
r++; r++;
for (j = 0; j < k; j++) { for (j = 0; j < k; j++) {
input_matrix[k * i + j] = encode_matrix[k * r + j]; input_matrix[k * i + j] = encode_matrix[k * r + j];

@ -24,8 +24,7 @@ package erasure
// #include <erasure-code.h> // #include <erasure-code.h>
// #include <stdlib.h> // #include <stdlib.h>
// //
// #include "decode.h" // #include "common.h"
// #include "encode.h"
import "C" import "C"
import ( import (
"errors" "errors"
@ -36,6 +35,7 @@ import (
func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) { func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
var decode_matrix *C.uchar var decode_matrix *C.uchar
var decode_tbls *C.uchar var decode_tbls *C.uchar
var source, target **C.uchar
k := int(e.p.k) k := int(e.p.k)
n := int(e.p.k + e.p.m) n := int(e.p.k + e.p.m)
@ -44,7 +44,7 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n)) return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
} }
chunk_size := int(C.calc_chunk_size(e.k, C.uint(length))) chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(length)))
src_err_list := make([]int, n+1) src_err_list := make([]int, n+1)
var err_count int = 0 var err_count int = 0
@ -84,21 +84,14 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
pointers[i] = &chunks[i][0] pointers[i] = &chunks[i][0]
} }
/*
// Pack recovery array as list of valid sources
// Its order must be the same as the order
// to generate matrix b in gf_gen_decode_matrix
var i int
for i = 0; i < e.p.k; i++ {
recov[i] = buffs[decode_index[i]]
}
*/
data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0])) data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0]))
coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0])) coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0]))
C.minio_get_source_target(src_err_list_ptr, C.int(err_count-1),
e.k, e.m, data, coding, &source, &target)
C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls, C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls,
data, coding) source, target)
recovered_output := make([]byte, 0, chunk_size*k) recovered_output := make([]byte, 0, chunk_size*k)
for i := 0; i < k; i++ { for i := 0; i < k; i++ {

@ -1,25 +0,0 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __DECODE_H__
#define __DECODE_H__
int minio_init_decoder (int *src_err_list,
unsigned char *encoding_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
int k, int n, int errs);
#endif /* __DECODE_H__ */

@ -13,37 +13,36 @@
// //
// Encoding data may be performed in 3 steps. // Encoding data may be performed in 3 steps.
// //
// 1. Create a validated set of encoder parameters // 1. Create a parse set of encoder parameters
// 2. Create a new encoder // 2. Create a new encoder
// 3. Encode data // 3. Encode data
// //
// Decoding data is also performed in 3 steps. // Decoding data is also performed in 3 steps.
// //
// 1. Create a validated set of encoder parameters // 1. Create a parse set of encoder parameters for validation
// 2. Create a new encoder // 2. Create a new encoder
// 3. Decode data // 3. Decode data
// //
// Encoder parameters contain four configurable elements: // Encoder parameters contain four configurable elements:
// ValidateParams(k, m, w, technique int) (EncoderParams, error) // ParseEncoderParams(k, m, technique int) (EncoderParams, error)
// k - Number of rows in matrix // k - Number of rows in matrix
// m - Number of colums in matrix // m - Number of colums in matrix
// w - Word size, typically 8 or aligned by 8.
// technique - Matrix type, can be either CAUCHY (recommended) or VANDERMONDE // technique - Matrix type, can be either CAUCHY (recommended) or VANDERMONDE
// constraints: k + m < 256 // constraints: k + m < Galois Field (2^8)
// //
// Example // Example
// //
// //
// Creating and using an encoder // Creating and using an encoder
// var bytes []byte // var bytes []byte
// params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY) // params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY)
// encoder := erasure.NewEncoder(params) // encoder := erasure.NewEncoder(params)
// encodedData, length := encoder.Encode(bytes) // encodedData, length := encoder.Encode(bytes)
// //
// Creating and using a decoder // Creating and using a decoder
// var encodedData [][]byte // var encodedData [][]byte
// var length int // var length int
// params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY) // params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY)
// encoder := erasure.NewEncoder(params) // encoder := erasure.NewEncoder(params)
// originalData, err := encoder.Decode(encodedData, length) // originalData, err := encoder.Decode(encodedData, length)
package erasure package erasure

@ -18,11 +18,11 @@
#include <stdio.h> #include <stdio.h>
#include <erasure-code.h> #include <erasure-code.h>
#include "encode.h" #include "common.h"
void minio_init_encoder (int technique, int k, int m, int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix, unsigned char **encode_matrix,
unsigned char **encode_tbls) unsigned char **encode_tbls)
{ {
size_t encode_matrix_size; size_t encode_matrix_size;
size_t encode_tbls_size; size_t encode_tbls_size;
@ -52,9 +52,11 @@ void minio_init_encoder (int technique, int k, int m,
*encode_matrix = tmp_matrix; *encode_matrix = tmp_matrix;
*encode_tbls = tmp_tbls; *encode_tbls = tmp_tbls;
return 0;
} }
unsigned int calc_chunk_size (int k, unsigned int split_len) uint32_t minio_calc_chunk_size (int k, uint32_t split_len)
{ {
int alignment; int alignment;
int remainder; int remainder;
@ -69,7 +71,3 @@ unsigned int calc_chunk_size (int k, unsigned int split_len)
} }
return padded_len / k; return padded_len / k;
} }
/*
void minio_encode (int k, int m, )
{
*/

@ -24,7 +24,7 @@ package erasure
// #include <erasure-code.h> // #include <erasure-code.h>
// #include <stdlib.h> // #include <stdlib.h>
// //
// #include "encode.h" // #include "common.h"
import "C" import "C"
import ( import (
"errors" "errors"
@ -123,7 +123,7 @@ func NewEncoder(ep *EncoderParams) *Encoder {
func (e *Encoder) Encode(block []byte) ([][]byte, int) { func (e *Encoder) Encode(block []byte) ([][]byte, int) {
var block_len = len(block) var block_len = len(block)
chunk_size := int(C.calc_chunk_size(e.k, C.uint(block_len))) chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(block_len)))
chunk_len := chunk_size * e.p.k chunk_len := chunk_size * e.p.k
pad_len := chunk_len - block_len pad_len := chunk_len - block_len

@ -1,25 +0,0 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __ENCODE_H__
#define __ENCODE_H__
#define SIMD_ALIGN 32
void minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls);
unsigned int calc_chunk_size (int k, unsigned int split_len);
#endif /* __ENCODE_H__ */
Loading…
Cancel
Save