oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixes #71 - some crazy races inside erasure and simplify the code

Browse Source 
- This change also brings in changing 'unsigned char' to 'uint8_t'
for brevity
master
Harshavardhana 10 years ago
parent d57700922d
commit 7314b5e37d
18 changed files with 210 additions and 265 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      Makefile
  2. 6
      cmd/erasure-demo/decode.go
  3. 8
      cmd/erasure-demo/encode.go
  4. 1
      pkgs/erasure/.gitignore
  5. 9
      pkgs/erasure/Makefile
  6. 16
      pkgs/erasure/cauchy_test.go
  7. 38
      pkgs/erasure/common.h
  8. 12
      pkgs/erasure/ctypes.go
  9. 108
      pkgs/erasure/decode.c
  10. 42
      pkgs/erasure/decode.go
  11. 14
      pkgs/erasure/encode.c
  12. 19
      pkgs/erasure/encode.go
  13. 10
      pkgs/erasure/isal/include/ec-base.h
  14. 99
      pkgs/erasure/isal/include/erasure-code.h
  15. 12
      pkgs/erasure/isal/include/gf-vect-mul.h
  16. 47
      pkgs/erasure/isal/src/ec-base.c
  17. 15
      pkgs/erasure/isal/src/ec-highlevel-func.c
  18. 16
      pkgs/erasure/vandermonde_test.go

3
Makefile
Unescape View File

@ -12,8 +12,7 @@ getdeps: checkdeps
build-erasure:
@$(MAKE) $(MAKE_OPTIONS) -C pkgs/erasure/isal lib
@godep go test -race github.com/minio-io/minio/pkgs/erasure
@godep go test -coverprofile=cover.out github.com/minio-io/minio/pkgs/erasure
@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/erasure
build-signify:
@$(MAKE) $(MAKE_OPTIONS) -C pkgs/signify

6
cmd/erasure-demo/decode.go
Unescape View File

@ -76,11 +76,15 @@ func decode(c *cli.Context) {
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
// Get decoder
encoder := erasure.NewEncoder(erasureParameters)
// decode data
decodedData, err := erasure.Decode(chunks, erasureParameters, length)
decodedData, err := encoder.Decode(chunks, length)
if err != nil {
log.Fatal(err)
}
// append decoded data
length, err = outputFile.Write(decodedData)
if err != nil {

8
cmd/erasure-demo/encode.go
Unescape View File

@ -39,9 +39,13 @@ func encode(c *cli.Context) {
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
// Init new encoder
encoder := erasure.NewEncoder(erasureParameters)
// encode data
if config.blockSize == 0 {
encodedData, length := erasure.Encode(input, erasureParameters)
encodedData, length := encoder.Encode(input)
for key, data := range encodedData {
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
@ -55,7 +59,7 @@ func encode(c *cli.Context) {
if chunk.Err != nil {
log.Fatal(chunk.Err)
}
encodedData, length := erasure.Encode(chunk.Data, erasureParameters)
encodedData, length := encoder.Encode(chunk.Data)
for key, data := range encodedData {
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)

1
pkgs/erasure/.gitignore vendored
Unescape View File

@ -0,0 +1 @@
erasure.test

9
pkgs/erasure/Makefile
Unescape View File

@ -1,13 +1,8 @@
all: build test
.PHONY: all
test: test-race test-cover
test-race:
@godep go test -race
test-cover:
@godep go test -coverprofile=cover.out
test:
@godep go test -race -coverprofile=cover.out
isal/isal-l.a:
@$(MAKE) -C isal lib

16
pkgs/erasure/cauchy_test.go
Unescape View File

@ -28,23 +28,13 @@ var _ = Suite(&MySuite{})
func Test(t *testing.T) { TestingT(t) }
func (s *MySuite) TestCauchyEncode(c *C) {
ep, _ := ParseEncoderParams(10, 5, CAUCHY)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
chunks, length := Encode(data, ep)
c.Logf("chunks length: %d;\nlength: %d\n", len(chunks), length)
c.Assert(length, Equals, len(data))
}
func (s *MySuite) TestCauchyDecode(c *C) {
ep, _ := ParseEncoderParams(10, 5, CAUCHY)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
chunks, length := Encode(data, ep)
e := NewEncoder(ep)
chunks, length := e.Encode(data)
c.Assert(length, Equals, len(data))
chunks[0] = nil
@ -53,7 +43,7 @@ func (s *MySuite) TestCauchyDecode(c *C) {
chunks[9] = nil
chunks[13] = nil
recovered_data, err := Decode(chunks, ep, length)
recovered_data, err := e.Decode(chunks, length)
c.Assert(err, IsNil)
if !bytes.Equal(data, recovered_data) {

38
pkgs/erasure/common.h
Unescape View File

@ -22,24 +22,22 @@
#define SIMD_ALIGN 32
int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls);
uint32_t minio_calc_chunk_size (int k,
uint32_t split_len);
int32_t minio_init_decoder (int *src_err_list,
unsigned char *encoding_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
int k, int n, int errs);
int32_t minio_src_in_err (int r, int *src_err_list);
int32_t minio_get_source_target(int *src_err_list,
int errs, int k, int m,
unsigned char **data,
unsigned char **coding,
unsigned char ***source,
unsigned char ***target);
uint8_t **encode_matrix,
uint8_t **encode_tbls);
uint32_t minio_calc_chunk_size (int k, uint32_t split_len);
int32_t minio_init_decoder (int32_t *error_index,
int k, int n, int errs,
uint8_t *encoding_matrix,
uint8_t **decode_matrix,
uint8_t **decode_tbls,
uint32_t **decode_index);
int32_t minio_get_source_target (int errs, int k, int m,
int32_t *error_index,
uint32_t *decode_index,
uint8_t **buffs,
uint8_t ***source,
uint8_t ***target);
#endif /* __COMMON_H__ */

12
pkgs/erasure/ctypes.go
Unescape View File

@ -23,35 +23,35 @@ import (
)
// Integer to Int conversion
func int2cInt(src_err_list []int) *C.int {
func int2cInt(src_err_list []int) *C.int32_t {
var sizeErrInt = int(unsafe.Sizeof(src_err_list[0]))
switch sizeInt {
case sizeErrInt:
return (*C.int)(unsafe.Pointer(&src_err_list[0]))
return (*C.int32_t)(unsafe.Pointer(&src_err_list[0]))
case sizeInt8:
int8Array := make([]int8, len(src_err_list))
for i, v := range src_err_list {
int8Array[i] = int8(v)
}
return (*C.int)(unsafe.Pointer(&int8Array[0]))
return (*C.int32_t)(unsafe.Pointer(&int8Array[0]))
case sizeInt16:
int16Array := make([]int16, len(src_err_list))
for i, v := range src_err_list {
int16Array[i] = int16(v)
}
return (*C.int)(unsafe.Pointer(&int16Array[0]))
return (*C.int32_t)(unsafe.Pointer(&int16Array[0]))
case sizeInt32:
int32Array := make([]int32, len(src_err_list))
for i, v := range src_err_list {
int32Array[i] = int32(v)
}
return (*C.int)(unsafe.Pointer(&int32Array[0]))
return (*C.int32_t)(unsafe.Pointer(&int32Array[0]))
case sizeInt64:
int64Array := make([]int64, len(src_err_list))
for i, v := range src_err_list {
int64Array[i] = int64(v)
}
return (*C.int)(unsafe.Pointer(&int64Array[0]))
return (*C.int32_t)(unsafe.Pointer(&int64Array[0]))
default:
panic(fmt.Sprintf("Unsupported: %d", sizeInt))
}

108
pkgs/erasure/decode.c
Unescape View File

@ -21,11 +21,12 @@
#include <erasure-code.h>
#include "common.h"
int32_t minio_src_in_err (int r, int *src_err_list)
static
int32_t _minio_src_index_in_error (int r, int32_t *error_index)
{
int i;
for (i = 0; src_err_list[i] != -1; i++) {
if (src_err_list[i] == r) {
for (i = 0; error_index[i] != -1; i++) {
if (error_index[i] == r) {
// true
return 1;
}
@ -34,47 +35,29 @@ int32_t minio_src_in_err (int r, int *src_err_list)
return 0;
}
int32_t minio_get_source_target(int *src_err_list,
int errs, int k, int m,
unsigned char **data,
unsigned char **coding,
unsigned char ***source,
unsigned char ***target)
int32_t minio_get_source_target (int errs, int k, int m,
int32_t *error_index,
uint32_t *decode_index,
uint8_t **buffs,
uint8_t ***source,
uint8_t ***target)
{
int i, j, l;
unsigned char *tmp_source[k];
unsigned char *tmp_target[m];
// Fill zeroes
memset (tmp_source, 0, sizeof(tmp_source));
memset (tmp_target, 0, sizeof(tmp_target));
// Separate out source and target buffers from input data/coding chunks
// This separation needs to happen at error chunks from input chunks
for (i = 0, j = 0, l = 0;
((l < k) || (j < errs)) && (i < (k + m)); i++) {
if (!minio_src_in_err(i, src_err_list)) {
if (l < k) {
if (i < k)
tmp_source[l] =
(unsigned char *) data[i];
else
tmp_source[l] =
(unsigned char *) coding[i - k];
l++;
}
} else {
if (j < m) {
if (i < k)
tmp_target[j] =
(unsigned char *) data[i];
else
tmp_target[j] =
(unsigned char *) coding[i - k];
j++;
}
}
int i;
uint8_t *tmp_source[k];
uint8_t *tmp_target[m];
memset (tmp_source, 0, k);
memset (tmp_target, 0, m);
for (i = 0; i < k; i++) {
tmp_source[i] = (uint8_t *) buffs[decode_index[i]];
}
for (i = 0; i < m; i++) {
if (i < errs)
tmp_target[i] = (uint8_t *) buffs[error_index[i]];
}
*source = tmp_source;
*target = tmp_target;
}
@ -83,32 +66,27 @@ int32_t minio_get_source_target(int *src_err_list,
Generate decode matrix during the decoding phase
*/
int minio_init_decoder (int *src_err_list,
unsigned char *encode_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
int k, int n, int errs)
int minio_init_decoder (int32_t *error_index,
int k, int n, int errs,
uint8_t *encode_matrix,
uint8_t **decode_matrix,
uint8_t **decode_tbls,
uint32_t **decode_index)
{
int i, j, r, s, l, z;
unsigned char input_matrix[k * n];
unsigned char inverse_matrix[k * n];
unsigned char *tmp_decode_matrix;
unsigned char *tmp_decode_tbls;
tmp_decode_matrix = (unsigned char *) malloc (k * n);
if (!tmp_decode_matrix)
return -1;
tmp_decode_tbls = (unsigned char *) malloc (k * n * 32);
if (!tmp_decode_tbls)
return -1;
uint8_t input_matrix[k * n];
uint8_t inverse_matrix[k * n];
uint8_t tmp_decode_matrix[k * n];
uint8_t tmp_decode_tbls[k * n * 32];
uint32_t tmp_decode_index[k];
for (i = 0, r = 0; i < k; i++, r++) {
while (minio_src_in_err(r, src_err_list))
while (_minio_src_index_in_error(r, error_index))
r++;
for (j = 0; j < k; j++) {
input_matrix[k * i + j] = encode_matrix[k * r + j];
}
tmp_decode_index[i] = r;
}
// Not all Vandermonde matrix can be inverted
@ -117,12 +95,12 @@ int minio_init_decoder (int *src_err_list,
}
for (l = 0; l < errs; l++) {
if (src_err_list[l] < k) {
if (error_index[l] < k) {
// decoding matrix elements for data chunks
for (j = 0; j < k; j++) {
tmp_decode_matrix[k * l + j] =
inverse_matrix[k *
src_err_list[l] + j];
error_index[l] + j];
}
} else {
int s = 0;
@ -132,16 +110,18 @@ int minio_init_decoder (int *src_err_list,
for (j = 0; j < k; j++) {
s ^= gf_mul(inverse_matrix[j * k + i],
encode_matrix[k *
src_err_list[l] + j]);
error_index[l] + j]);
}
tmp_decode_matrix[k * l + i] = s;
}
}
}
ec_init_tables(k, errs, tmp_decode_matrix, tmp_decode_tbls);
ec_init_tables (k, errs, tmp_decode_matrix, tmp_decode_tbls);
*decode_matrix = tmp_decode_matrix;
*decode_tbls = tmp_decode_tbls;
*decode_index = tmp_decode_index;
return 0;
}

42
pkgs/erasure/decode.go
Unescape View File

@ -21,7 +21,6 @@ package erasure
// #cgo CPPFLAGS: -Iisal/include
// #include <stdlib.h>
// #include <erasure-code.h>
// #include <stdlib.h>
//
// #include "common.h"
import "C"
@ -32,12 +31,13 @@ import (
)
func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
var decode_matrix *C.uchar
var decode_tbls *C.uchar
var source, target **C.uchar
var decode_matrix *C.uint8_t
var decode_tbls *C.uint8_t
var decode_index *C.uint32_t
var source, target **C.uint8_t
k := int(e.p.k)
n := int(e.p.k + e.p.m)
k := int(e.k)
n := int(e.k + e.m)
if len(chunks) != n {
return nil, errors.New(fmt.Sprintf("chunks length must be %d", n))
@ -45,25 +45,25 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
chunk_size := int(C.minio_calc_chunk_size(e.k, C.uint32_t(length)))
src_err_list := make([]int, n+1)
error_index := make([]int, n+1)
var err_count int = 0
for i := range chunks {
// Check of chunks are really null
if chunks[i] == nil || len(chunks[i]) == 0 {
src_err_list[err_count] = i
error_index[err_count] = i
err_count++
}
}
src_err_list[err_count] = -1
error_index[err_count] = -1
err_count++
// Too many missing chunks, cannot be more than parity `m`
if err_count-1 > e.p.m {
if err_count-1 > (n - k) {
return nil, errors.New("too many erasures requested, can't decode")
}
src_err_list_ptr := int2cInt(src_err_list[:err_count])
error_index_ptr := int2cInt(error_index[:err_count])
for i := range chunks {
if chunks[i] == nil || len(chunks[i]) == 0 {
@ -71,23 +71,18 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
}
}
C.minio_init_decoder(src_err_list_ptr, e.encode_matrix,
&decode_matrix, &decode_tbls,
e.k, e.k+e.m, C.int(err_count-1))
defer C.free(unsafe.Pointer(decode_matrix))
defer C.free(unsafe.Pointer(decode_tbls))
C.minio_init_decoder(error_index_ptr, e.k, e.k+e.m, C.int(err_count-1),
e.encode_matrix, &decode_matrix, &decode_tbls, &decode_index)
pointers := make([]*byte, n)
for i := range chunks {
pointers[i] = &chunks[i][0]
}
data := (**C.uchar)(unsafe.Pointer(&pointers[:k][0]))
coding := (**C.uchar)(unsafe.Pointer(&pointers[k:][0]))
data := (**C.uint8_t)(unsafe.Pointer(&pointers[0]))
C.minio_get_source_target(src_err_list_ptr, C.int(err_count-1),
e.k, e.m, data, coding, &source, &target)
C.minio_get_source_target(C.int(err_count-1), e.k, e.m, error_index_ptr,
decode_index, data, &source, &target)
C.ec_encode_data(C.int(chunk_size), e.k, C.int(err_count-1), decode_tbls,
source, target)
@ -102,8 +97,3 @@ func (e *Encoder) Decode(chunks [][]byte, length int) ([]byte, error) {
return recovered_output[:length], nil
}
func Decode(block [][]byte, ep *EncoderParams, length int) ([]byte, error) {
encoder := NewEncoder(ep)
return encoder.Decode(block, length)
}

14
pkgs/erasure/encode.c
Unescape View File

@ -21,18 +21,16 @@
#include "common.h"
int32_t minio_init_encoder (int technique, int k, int m,
unsigned char **encode_matrix,
unsigned char **encode_tbls)
uint8_t **encode_matrix,
uint8_t **encode_tbls)
{
size_t encode_matrix_size;
size_t encode_tbls_size;
unsigned char *tmp_matrix, *tmp_tbls;
uint8_t *tmp_matrix;
uint8_t *tmp_tbls;
encode_matrix_size = k * (k + m);
encode_tbls_size = k * (k + m) * 32;
tmp_matrix = (unsigned char *) malloc (encode_matrix_size);
tmp_tbls = (unsigned char *) malloc (encode_tbls_size);
tmp_matrix = (uint8_t *) malloc (k * (k + m));
tmp_tbls = (uint8_t *) malloc (k * (k + m) * 32);
if (technique == 0) {
/*

19
pkgs/erasure/encode.go
Unescape View File

@ -22,7 +22,6 @@ package erasure
// #cgo LDFLAGS: isal/isa-l.a
// #include <stdlib.h>
// #include <erasure-code.h>
// #include <stdlib.h>
//
// #include "common.h"
import "C"
@ -57,7 +56,7 @@ type Encoder struct {
encode_matrix,
encode_tbls,
decode_matrix,
decode_tbls *C.uchar
decode_tbls *C.uint8_t
}
// ParseEncoderParams creates an EncoderParams object.
@ -99,13 +98,12 @@ func NewEncoder(ep *EncoderParams) *Encoder {
var k = C.int(ep.k)
var m = C.int(ep.m)
var encode_matrix *C.uchar
var encode_tbls *C.uchar
var encode_matrix *C.uint8_t
var encode_tbls *C.uint8_t
C.minio_init_encoder(C.int(ep.technique), k, m, &encode_matrix,
&encode_tbls)
defer C.free(unsafe.Pointer(encode_matrix))
defer C.free(unsafe.Pointer(encode_tbls))
return &Encoder{
p: ep,
k: k,
@ -153,15 +151,10 @@ func (e *Encoder) Encode(block []byte) ([][]byte, int) {
pointers[i] = &chunks[i][0]
}
data := (**C.uchar)(unsafe.Pointer(&pointers[:e.p.k][0]))
coding := (**C.uchar)(unsafe.Pointer(&pointers[e.p.k:][0]))
data := (**C.uint8_t)(unsafe.Pointer(&pointers[:e.p.k][0]))
coding := (**C.uint8_t)(unsafe.Pointer(&pointers[e.p.k:][0]))
C.ec_encode_data(C.int(chunk_size), e.k, e.m, e.encode_tbls, data,
coding)
return chunks, block_len
}
func Encode(block []byte, ep *EncoderParams) ([][]byte, int) {
encoder := NewEncoder(ep)
return encoder.Encode(block)
}

10
pkgs/erasure/isal/include/ec-base.h
Unescape View File

@ -2,7 +2,7 @@
Copyright(c) 2011-2014 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
modification, are permitted provided that the following conditions
are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
@ -32,7 +32,7 @@
// Global GF(256) tables
#ifndef GF_LARGE_TABLES
unsigned char gff_base[] = {
uint8_t gff_base[] = {
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1d, 0x3a,
0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26, 0x4c, 0x98, 0x2d, 0x5a,
0xb4, 0x75, 0xea, 0xc9, 0x8f, 0x03, 0x06, 0x0c, 0x18, 0x30,
@ -61,7 +61,7 @@ unsigned char gff_base[] = {
0x6c, 0xd8, 0xad, 0x47, 0x8e, 0x01
};
unsigned char gflog_base[] = {
uint8_t gflog_base[] = {
0x00, 0xff, 0x01, 0x19, 0x02, 0x32, 0x1a, 0xc6, 0x03, 0xdf,
0x33, 0xee, 0x1b, 0x68, 0xc7, 0x4b, 0x04, 0x64, 0xe0, 0x0e,
0x34, 0x8d, 0xef, 0x81, 0x1c, 0xc1, 0x69, 0xf8, 0xc8, 0x08,
@ -90,7 +90,7 @@ unsigned char gflog_base[] = {
0xf4, 0xea, 0xa8, 0x50, 0x58, 0xaf
};
#else
unsigned char gf_mul_table_base[] = {
uint8_t gf_mul_table_base[] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@ -6647,7 +6647,7 @@ unsigned char gf_mul_table_base[] = {
0xc6, 0x39, 0xfe, 0x01, 0x1d, 0xe2
};
unsigned char gf_inv_table_base[] = {
uint8_t gf_inv_table_base[] = {
0x00, 0x01, 0x8e, 0xf4, 0x47, 0xa7, 0x7a, 0xba, 0xad, 0x9d,
0xdd, 0x98, 0x3d, 0xaa, 0x5d, 0x96, 0xd8, 0x72, 0xc0, 0x58,
0xe0, 0x3e, 0x4c, 0x66, 0x90, 0xde, 0x55, 0x80, 0xa0, 0x83,

99
pkgs/erasure/isal/include/erasure-code.h
Unescape View File

@ -31,6 +31,7 @@
#ifndef _ERASURE_CODE_H_
#define _ERASURE_CODE_H_
#include <stdint.h>
/**
* @file erasure_code.h
* @brief Interface to functions supporting erasure code encode and decode.
@ -71,7 +72,7 @@ extern "C" {
* @returns none
*/
void ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls);
void ec_init_tables(int k, int rows, uint8_t* a, uint8_t* gftbls);
/**
* @brief Generate or decode erasure codes on blocks of data.
@ -93,7 +94,7 @@ void ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls);
* @returns none
*/
void ec_encode_data_sse(int len, int k, int rows, unsigned char *gftbls, unsigned char **data, unsigned char **coding);
void ec_encode_data_sse(int len, int k, int rows, uint8_t *gftbls, uint8_t **data, uint8_t **coding);
/**
@ -118,7 +119,7 @@ void ec_encode_data_sse(int len, int k, int rows, unsigned char *gftbls, unsigne
* @returns none
*/
void ec_encode_data(int len, int k, int rows, unsigned char *gftbls, unsigned char **data, unsigned char **coding);
void ec_encode_data(int len, int k, int rows, uint8_t *gftbls, uint8_t **data, uint8_t **coding);
/**
@ -140,7 +141,7 @@ void ec_encode_data(int len, int k, int rows, unsigned char *gftbls, unsigned ch
* @returns none
*/
void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v, unsigned char **src, unsigned char **dest);
void ec_encode_data_base(int len, int srcs, int dests, uint8_t *v, uint8_t **src, uint8_t **dest);
/**
@ -161,8 +162,8 @@ void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v, unsigne
* @returns none
*/
void gf_vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char *dest);
void gf_vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t *dest);
/**
* @brief GF(2^8) vector dot product.
@ -182,8 +183,8 @@ void gf_vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char *dest);
void gf_vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t *dest);
/**
* @brief GF(2^8) vector dot product.
@ -203,8 +204,8 @@ void gf_vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char *dest);
void gf_vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t *dest);
/**
* @brief GF(2^8) vector dot product with two outputs.
@ -225,8 +226,8 @@ void gf_vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_2vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_2vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with two outputs.
@ -247,8 +248,8 @@ void gf_2vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_2vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_2vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with two outputs.
@ -269,8 +270,8 @@ void gf_2vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_2vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_2vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with three outputs.
@ -291,8 +292,8 @@ void gf_2vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_3vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_3vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with three outputs.
@ -313,8 +314,8 @@ void gf_3vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_3vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_3vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with three outputs.
@ -335,8 +336,8 @@ void gf_3vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_3vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_3vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with four outputs.
@ -357,8 +358,8 @@ void gf_3vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_4vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_4vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with four outputs.
@ -379,8 +380,8 @@ void gf_4vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_4vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_4vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with four outputs.
@ -401,8 +402,8 @@ void gf_4vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_4vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_4vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with five outputs.
@ -423,8 +424,8 @@ void gf_4vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_5vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_5vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with five outputs.
@ -445,8 +446,8 @@ void gf_5vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_5vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_5vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with five outputs.
@ -467,8 +468,8 @@ void gf_5vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_5vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_5vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with six outputs.
@ -489,8 +490,8 @@ void gf_5vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_6vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_6vect_dot_prod_sse(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with six outputs.
@ -511,8 +512,8 @@ void gf_6vect_dot_prod_sse(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_6vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_6vect_dot_prod_avx(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product with six outputs.
@ -533,8 +534,8 @@ void gf_6vect_dot_prod_avx(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_6vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char **dest);
void gf_6vect_dot_prod_avx2(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t **dest);
/**
* @brief GF(2^8) vector dot product, runs baseline version.
@ -556,8 +557,8 @@ void gf_6vect_dot_prod_avx2(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_vect_dot_prod_base(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char *dest);
void gf_vect_dot_prod_base(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t *dest);
/**
* @brief GF(2^8) vector dot product, runs appropriate version.
@ -579,8 +580,8 @@ void gf_vect_dot_prod_base(int len, int vlen, unsigned char *gftbls,
* @returns none
*/
void gf_vect_dot_prod(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char *dest);
void gf_vect_dot_prod(int len, int vlen, uint8_t *gftbls,
uint8_t **src, uint8_t *dest);
/**********************************************************************
* The remaining are lib support functions used in GF(2^8) operations.
@ -594,7 +595,7 @@ void gf_vect_dot_prod(int len, int vlen, unsigned char *gftbls,
* @returns Product of a and b in GF(2^8)
*/
unsigned char gf_mul(unsigned char a, unsigned char b);
uint8_t gf_mul(uint8_t a, uint8_t b);
/**
* @brief Single element GF(2^8) inverse.
@ -603,7 +604,7 @@ unsigned char gf_mul(unsigned char a, unsigned char b);
* @returns Field element b such that a x b = {1}
*/
unsigned char gf_inv(unsigned char a);
uint8_t gf_inv(uint8_t a);
/**
* @brief Generate a matrix of coefficients to be used for encoding.
@ -622,7 +623,7 @@ unsigned char gf_inv(unsigned char a);
* @returns none
*/
void gf_gen_rs_matrix(unsigned char *a, int m, int k);
void gf_gen_rs_matrix(uint8_t *a, int m, int k);
/**
* @brief Generate a Cauchy matrix of coefficients to be used for encoding.
@ -637,7 +638,7 @@ void gf_gen_rs_matrix(unsigned char *a, int m, int k);
* @returns none
*/
void gf_gen_cauchy1_matrix(unsigned char *a, int m, int k);
void gf_gen_cauchy1_matrix(uint8_t *a, int m, int k);
/**
* @brief Invert a matrix in GF(2^8)
@ -648,7 +649,7 @@ void gf_gen_cauchy1_matrix(unsigned char *a, int m, int k);
* @returns 0 successful, other fail on singular input matrix
*/
int gf_invert_matrix(unsigned char *in, unsigned char *out, const int n);
int gf_invert_matrix(uint8_t *in, uint8_t *out, const int n);
/*************************************************************/

12
pkgs/erasure/isal/include/gf-vect-mul.h
Unescape View File

@ -61,7 +61,7 @@ extern "C" {
* @returns 0 pass, other fail
*/
int gf_vect_mul_sse(int len, unsigned char *gftbl, void *src, void *dest);
int gf_vect_mul_sse(int len, uint8_t *gftbl, void *src, void *dest);
/**
@ -82,7 +82,7 @@ int gf_vect_mul_sse(int len, unsigned char *gftbl, void *src, void *dest);
* @returns 0 pass, other fail
*/
int gf_vect_mul_avx(int len, unsigned char *gftbl, void *src, void *dest);
int gf_vect_mul_avx(int len, uint8_t *gftbl, void *src, void *dest);
/**
@ -105,7 +105,7 @@ int gf_vect_mul_avx(int len, unsigned char *gftbl, void *src, void *dest);
* @returns 0 pass, other fail
*/
int gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest);
int gf_vect_mul(int len, uint8_t *gftbl, void *src, void *dest);
/**
@ -118,7 +118,7 @@ int gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest);
* @param gftbl Table output.
*/
void gf_vect_mul_init(unsigned char c, unsigned char* gftbl);
void gf_vect_mul_init(uint8_t c, uint8_t* gftbl);
/**
@ -138,8 +138,8 @@ void gf_vect_mul_init(unsigned char c, unsigned char* gftbl);
* @param dest Pointer to destination data array. Must be aligned to 32B.
*/
void gf_vect_mul_base(int len, unsigned char *a, unsigned char *src,
unsigned char *dest);
void gf_vect_mul_base(int len, uint8_t *a, uint8_t *src,
uint8_t *dest);
#ifdef __cplusplus
}

47
pkgs/erasure/isal/src/ec-base.c
Unescape View File

@ -28,12 +28,13 @@
**********************************************************************/
#include <limits.h>
#include <stdint.h>
#include <string.h> // for memset
#include "erasure-code.h"
#include "ec-base.h" // for GF tables
#include "erasure/types.h"
unsigned char gf_mul(unsigned char a, unsigned char b)
uint8_t gf_mul(uint8_t a, uint8_t b)
{
#ifndef GF_LARGE_TABLES
int i;
@ -47,7 +48,7 @@ unsigned char gf_mul(unsigned char a, unsigned char b)
#endif
}
unsigned char gf_inv(unsigned char a)
uint8_t gf_inv(uint8_t a)
{
#ifndef GF_LARGE_TABLES
if (a == 0)
@ -59,10 +60,10 @@ unsigned char gf_inv(unsigned char a)
#endif
}
void gf_gen_rs_matrix(unsigned char *a, int m, int k)
void gf_gen_rs_matrix(uint8_t *a, int m, int k)
{
int i, j;
unsigned char p, gen = 1;
uint8_t p, gen = 1;
memset(a, 0, k * m);
for (i = 0; i < k; i++)
@ -78,10 +79,10 @@ void gf_gen_rs_matrix(unsigned char *a, int m, int k)
}
}
void gf_gen_cauchy1_matrix(unsigned char *a, int m, int k)
void gf_gen_cauchy1_matrix(uint8_t *a, int m, int k)
{
int i, j;
unsigned char *p;
uint8_t *p;
// Identity matrix in high position
memset(a, 0, k * m);
@ -96,10 +97,10 @@ void gf_gen_cauchy1_matrix(unsigned char *a, int m, int k)
}
int gf_invert_matrix(unsigned char *in_mat, unsigned char *out_mat, const int n)
int gf_invert_matrix(uint8_t *in_mat, uint8_t *out_mat, const int n)
{
int i, j, k;
unsigned char temp;
uint8_t temp;
// Set out_mat[] to the identity matrix
for (i = 0; i < n * n; i++) // memset(out_mat, 0, n*n)
@ -154,16 +155,16 @@ int gf_invert_matrix(unsigned char *in_mat, unsigned char *out_mat, const int n)
// Calculates const table gftbl in GF(2^8) from single input A
// gftbl(A) = {A{00}, A{01}, A{02}, ... , A{0f} }, {A{00}, A{10}, A{20}, ... , A{f0} }
void gf_vect_mul_init(unsigned char c, unsigned char *tbl)
void gf_vect_mul_init(uint8_t c, uint8_t *tbl)
{
unsigned char c2 = (c << 1) ^ ((c & 0x80) ? 0x1d : 0); //Mult by GF{2}
unsigned char c4 = (c2 << 1) ^ ((c2 & 0x80) ? 0x1d : 0); //Mult by GF{2}
unsigned char c8 = (c4 << 1) ^ ((c4 & 0x80) ? 0x1d : 0); //Mult by GF{2}
uint8_t c2 = (c << 1) ^ ((c & 0x80) ? 0x1d : 0); //Mult by GF{2}
uint8_t c4 = (c2 << 1) ^ ((c2 & 0x80) ? 0x1d : 0); //Mult by GF{2}
uint8_t c8 = (c4 << 1) ^ ((c4 & 0x80) ? 0x1d : 0); //Mult by GF{2}
#if __WORDSIZE == 64 || _WIN64 || __x86_64__
unsigned long long v1, v2, v4, v8, *t;
unsigned long long v10, v20, v40, v80;
unsigned char c17, c18, c20, c24;
uint8_t c17, c18, c20, c24;
t = (unsigned long long *)tbl;
@ -191,8 +192,8 @@ void gf_vect_mul_init(unsigned char c, unsigned char *tbl)
t[3] = v80 ^ v40;
#else // 32-bit or other
unsigned char c3, c5, c6, c7, c9, c10, c11, c12, c13, c14, c15;
unsigned char c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30,
uint8_t c3, c5, c6, c7, c9, c10, c11, c12, c13, c14, c15;
uint8_t c17, c18, c19, c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, c30,
c31;
c3 = c2 ^ c;
@ -261,11 +262,11 @@ void gf_vect_mul_init(unsigned char c, unsigned char *tbl)
#endif //__WORDSIZE == 64 || _WIN64 || __x86_64__
}
void gf_vect_dot_prod_base(int len, int vlen, unsigned char *v,
unsigned char **src, unsigned char *dest)
void gf_vect_dot_prod_base(int len, int vlen, uint8_t *v,
uint8_t **src, uint8_t *dest)
{
int i, j;
unsigned char s;
uint8_t s;
for (i = 0; i < len; i++) {
s = 0;
for (j = 0; j < vlen; j++)
@ -275,11 +276,11 @@ void gf_vect_dot_prod_base(int len, int vlen, unsigned char *v,
}
}
void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v,
unsigned char **src, unsigned char **dest)
void ec_encode_data_base(int len, int srcs, int dests, uint8_t *v,
uint8_t **src, uint8_t **dest)
{
int i, j, l;
unsigned char s;
uint8_t s;
for (l = 0; l < dests; l++) {
for (i = 0; i < len; i++) {
@ -292,10 +293,10 @@ void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v,
}
}
void gf_vect_mul_base(int len, unsigned char *a, unsigned char *src, unsigned char *dest)
void gf_vect_mul_base(int len, uint8_t *a, uint8_t *src, uint8_t *dest)
{
//2nd element of table array is ref value used to fill it in
unsigned char c = a[1];
uint8_t c = a[1];
while (len-- > 0)
*dest++ = gf_mul(c, *src++);
}

15
pkgs/erasure/isal/src/ec-highlevel-func.c
Unescape View File

@ -27,10 +27,11 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**********************************************************************/
#include <limits.h>
#include <stdint.h>
#include "erasure-code.h"
#include "erasure/types.h"
void ec_init_tables(int k, int rows, unsigned char *a, unsigned char *g_tbls)
void ec_init_tables(int k, int rows, uint8_t *a, uint8_t *g_tbls)
{
int i, j;
@ -43,8 +44,8 @@ void ec_init_tables(int k, int rows, unsigned char *a, unsigned char *g_tbls)
}
#if __WORDSIZE == 64 || _WIN64 || __x86_64__
void ec_encode_data_sse(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data,
unsigned char **coding)
void ec_encode_data_sse(int len, int k, int rows, uint8_t *g_tbls, uint8_t **data,
uint8_t **coding)
{
if (len < 16) {
@ -74,8 +75,8 @@ void ec_encode_data_sse(int len, int k, int rows, unsigned char *g_tbls, unsigne
}
void ec_encode_data_avx(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data,
unsigned char **coding)
void ec_encode_data_avx(int len, int k, int rows, uint8_t *g_tbls, uint8_t **data,
uint8_t **coding)
{
if (len < 16) {
@ -105,8 +106,8 @@ void ec_encode_data_avx(int len, int k, int rows, unsigned char *g_tbls, unsigne
}
void ec_encode_data_avx2(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data,
unsigned char **coding)
void ec_encode_data_avx2(int len, int k, int rows, uint8_t *g_tbls, uint8_t **data,
uint8_t **coding)
{
if (len < 32) {

16
pkgs/erasure/vandermonde_test.go
Unescape View File

@ -21,23 +21,13 @@ import (
. "gopkg.in/check.v1"
)
func (s *MySuite) TestVanderMondeEncode(c *C) {
ep, _ := ParseEncoderParams(10, 5, VANDERMONDE)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
chunks, length := Encode(data, ep)
c.Logf("chunks length: %d;\nlength: %d\n", len(chunks), length)
c.Assert(length, Equals, len(data))
}
func (s *MySuite) TestVanderMondeDecode(c *C) {
ep, _ := ParseEncoderParams(10, 5, VANDERMONDE)
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
chunks, length := Encode(data, ep)
e := NewEncoder(ep)
chunks, length := e.Encode(data)
c.Logf("chunks length: %d", len(chunks))
c.Logf("length: %d", length)
c.Assert(length, Equals, len(data))
@ -48,7 +38,7 @@ func (s *MySuite) TestVanderMondeDecode(c *C) {
chunks[9] = nil
chunks[13] = nil
recovered_data, err := Decode(chunks, ep, length)
recovered_data, err := e.Decode(chunks, length)
c.Assert(err, IsNil)
if !bytes.Equal(recovered_data, data) {

Write Preview
Loading…
Cancel
Save