Decoding bug fixes #42

pkgs/erasure/common.h

@@ -0,0 +1,45 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __COMMON_H__
+#define __COMMON_H__
+
+#include <stdint.h>
+
+#define SIMD_ALIGN 32
+
+int32_t minio_init_encoder (int technique, int k, int m,
+                            unsigned char **encode_matrix,
+                            unsigned char **encode_tbls);
+
+uint32_t minio_calc_chunk_size (int k,
+                                uint32_t split_len);
+
+int32_t minio_init_decoder (int *src_err_list,
+                            unsigned char *encoding_matrix,
+                            unsigned char **decode_matrix,
+                            unsigned char **decode_tbls,
+                            int k, int n, int errs);
+
+int32_t minio_src_in_err (int r, int *src_err_list);
+
+int32_t minio_get_source_target(int *src_err_list,
+                                int errs, int k, int m,
+                                unsigned char **data,
+                                unsigned char **coding,
+                                unsigned char ***source,
+                                unsigned char ***target);
+#endif /* __COMMON_H__ */

pkgs/erasure/decode.c

@@ -19,13 +19,14 @@
 #include <string.h>
 
 #include <erasure-code.h>
-#include "decode.h"
+#include "common.h"
 
-static int src_in_err (int r, int *src_err_list)
+int32_t minio_src_in_err (int r, int *src_err_list)
 {
         int i;
         for (i = 0; src_err_list[i] != -1; i++) {
                 if (src_err_list[i] == r) {
+                        // true
                         return 1;
                 }
         }
@@ -33,6 +34,51 @@ static int src_in_err (int r, int *src_err_list)
         return 0;
 }
 
+int32_t minio_get_source_target(int *src_err_list,
+                                int errs, int k, int m,
+                                unsigned char **data,
+                                unsigned char **coding,
+                                unsigned char ***source,
+                                unsigned char ***target)
+{
+        int i, j, l;
+        unsigned char *tmp_source[k];
+        unsigned char *tmp_target[m];
+
+        // Fill zeroes
+        memset (tmp_source, 0, sizeof(tmp_source));
+        memset (tmp_target, 0, sizeof(tmp_target));
+
+        // Separate out source and target buffers from input data/coding chunks
+        // This separation needs to happen at error chunks from input chunks
+        for (i = 0, j = 0, l = 0;
+             ((l < k) || (j < errs)) && (i < (k + m)); i++) {
+                if (!minio_src_in_err(i, src_err_list)) {
+                        if (l < k) {
+                                if (i < k)
+                                        tmp_source[l] =
+                                                (unsigned char *) data[i];
+                                else
+                                        tmp_source[l] =
+                                                (unsigned char *) coding[i - k];
+                                l++;
+                        }
+                } else {
+                        if (j < m) {
+                                if (i < k)
+                                        tmp_target[j] =
+                                                (unsigned char *) data[i];
+                                else
+                                        tmp_target[j] =
+                                                (unsigned char *) coding[i - k];
+                                j++;
+                        }
+                }
+        }
+        *source = tmp_source;
+        *target = tmp_target;
+}
+
 /*
   Generate decode matrix during the decoding phase
 */
@@ -58,7 +104,7 @@ int minio_init_decoder (int *src_err_list,
                 return -1;
 
         for (i = 0, r = 0; i < k; i++, r++) {
-                while (src_in_err(r, src_err_list))
+                while (minio_src_in_err(r, src_err_list))
                         r++;
                 for (j = 0; j < k; j++) {
                         input_matrix[k * i + j] = encode_matrix[k * r + j];

pkgs/erasure/decode.go

@@ -24,8 +24,7 @@ package erasure
 // #include <erasure-code.h>
 // #include <stdlib.h>
 //
-// #include "decode.h"
-// #include "encode.h"
+// #include "common.h"
 import "C"
 import (
 	"errors"
@@ -36,6 +35,7 @@ import (
 func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
 	var decode_matrix *C.uchar
 	var decode_tbls *C.uchar
+	var source, target **C.uchar
 
 	k := int(e.p.k)
 	n := int(e.p.k + e.p.m)
@@ -44,7 +44,7 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
 		return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
 	}
 
-	chunk_size := int(C.calc_chunk_size(e.k, C.uint(length)))
+	chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(length)))
 
 	src_err_list := make([]int, n+1)
 	var err_count int = 0
@@ -84,21 +84,14 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
 		pointers[i] = &chunks[i][0]
 	}
 
-	/*
-		// Pack recovery array as list of valid sources
-		// Its order must be the same as the order
-		// to generate matrix b in gf_gen_decode_matrix
-		var i int
-		for i = 0; i < e.p.k; i++ {
-			recov[i] = buffs[decode_index[i]]
-		}
-	*/
-
 	data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0]))
 	coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0]))
 
+	C.minio_get_source_target(src_err_list_ptr, C.int(err_count-1),
+		e.k, e.m, data, coding, &source, &target)
+
 	C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls,
-		data, coding)
+		source, target)
 
 	recovered_output := make([]byte, 0, chunk_size*k)
 	for i := 0; i < k; i++ {

pkgs/erasure/decode.h

@@ -1,25 +0,0 @@
-/*
- * Mini Object Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __DECODE_H__
-#define __DECODE_H__
-
-int minio_init_decoder (int *src_err_list,
-                        unsigned char *encoding_matrix,
-                        unsigned char **decode_matrix,
-                        unsigned char **decode_tbls,
-                        int k, int n, int errs);
-#endif /* __DECODE_H__  */

pkgs/erasure/doc.go

@@ -13,37 +13,36 @@
 //
 // Encoding data may be performed in 3 steps.
 //
-//  1. Create a validated set of encoder parameters
+//  1. Create a parse set of encoder parameters
 //  2. Create a new encoder
 //  3. Encode data
 //
 // Decoding data is also performed in 3 steps.
 //
-//  1. Create a validated set of encoder parameters
+//  1. Create a parse set of encoder parameters for validation
 //  2. Create a new encoder
 //  3. Decode data
 //
 // Encoder parameters contain four configurable elements:
-//  ValidateParams(k, m, w, technique int) (EncoderParams, error)
+//  ParseEncoderParams(k, m, technique int) (EncoderParams, error)
 //  k - Number of rows in matrix
 //  m - Number of colums in matrix
-//  w - Word size, typically 8 or aligned by 8.
 //  technique - Matrix type, can be either CAUCHY (recommended) or VANDERMONDE
-//  constraints: k + m < 256
+//  constraints: k + m < Galois Field (2^8)
 //
 // Example
 //
 //
 // Creating and using an encoder
 //  var bytes []byte
-//  params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY)
+//  params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY)
 //  encoder := erasure.NewEncoder(params)
 //  encodedData, length := encoder.Encode(bytes)
 //
 // Creating and using a decoder
 //  var encodedData [][]byte
 //  var length int
-//  params := erasure.ValidateParams(10,5, 8, erasure.CAUCHY)
+//  params := erasure.ParseEncoderParams(10, 5, erasure.CAUCHY)
 //  encoder := erasure.NewEncoder(params)
 //  originalData, err := encoder.Decode(encodedData, length)
 package erasure

pkgs/erasure/encode.c

@@ -18,11 +18,11 @@
 #include <stdio.h>
 
 #include <erasure-code.h>
-#include "encode.h"
+#include "common.h"
 
-void minio_init_encoder (int technique, int k, int m,
-                         unsigned char **encode_matrix,
-                         unsigned char **encode_tbls)
+int32_t minio_init_encoder (int technique, int k, int m,
+                            unsigned char **encode_matrix,
+                            unsigned char **encode_tbls)
 {
         size_t encode_matrix_size;
         size_t encode_tbls_size;
@@ -52,9 +52,11 @@ void minio_init_encoder (int technique, int k, int m,
 
         *encode_matrix = tmp_matrix;
         *encode_tbls = tmp_tbls;
+
+        return 0;
 }
 
-unsigned int calc_chunk_size (int k, unsigned int split_len)
+uint32_t minio_calc_chunk_size (int k, uint32_t split_len)
 {
         int alignment;
         int remainder;
@@ -69,7 +71,3 @@ unsigned int calc_chunk_size (int k, unsigned int split_len)
         }
         return padded_len / k;
 }
-/*
-void minio_encode (int k, int m, )
-{
-*/

pkgs/erasure/encode.go

@@ -24,7 +24,7 @@ package erasure
 // #include <erasure-code.h>
 // #include <stdlib.h>
 //
-// #include "encode.h"
+// #include "common.h"
 import "C"
 import (
 	"errors"
@@ -123,7 +123,7 @@ func NewEncoder(ep *EncoderParams) *Encoder {
 func (e *Encoder) Encode(block []byte) ([][]byte, int) {
 	var block_len = len(block)
 
-	chunk_size := int(C.calc_chunk_size(e.k, C.uint(block_len)))
+	chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(block_len)))
 	chunk_len := chunk_size * e.p.k
 	pad_len := chunk_len - block_len
 

pkgs/erasure/encode.h

@@ -1,25 +0,0 @@
-/*
- * Mini Object Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __ENCODE_H__
-#define __ENCODE_H__
-
-#define SIMD_ALIGN 32
-void minio_init_encoder (int technique, int k, int m,
-                         unsigned char **encode_matrix,
-                         unsigned char **encode_tbls);
-unsigned int calc_chunk_size (int k, unsigned int split_len);
-#endif /* __ENCODE_H__ */