Crc32c re-implementation with Convenience functions

- Sum32([]byte) --> uint32
   - Sum(io.Reader) --> uint32, error

pkg/utils/checksum/crc32c/crc32c_darwin.go

@@ -23,11 +23,25 @@ import (
 
 var castanagoliTable = crc32.MakeTable(crc32.Castagnoli)
 
-func Crc32c(buffer []byte) (uint32, error) {
+func Sum32(buffer []byte) uint32 {
 	crc := crc32.New(castanagoliTable)
-	if len(buffer) <= 0 {
+	crc.Reset()
-		return 0, errors.New("input buffer cannot be null")
-	}
 	crc.Write(buffer)
-	return crc.Sum32(), nil
+	return crc.Sum32()
+}
+
+func Sum(reader io.Reader) (uint32, error) {
+	h := New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		h.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+	return h.Sum32(), nil
 }

pkg/utils/checksum/crc32c/crc32c_intel.go

@@ -0,0 +1,31 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crc32c
+
+// #include <stdint.h>
+// uint32_t crc32c_pcl(uint8_t *buf, int32_t len, uint32_t prev_crc);
+import "C"
+import (
+	"unsafe"
+)
+
+func updateCastanagoliPCL(crc uint32, p []byte) uint32 {
+	if len(p) == 0 {
+		return 0
+	}
+	return uint32(C.crc32c_pcl((*C.uint8_t)(unsafe.Pointer(&p[0])), C.int32_t(len(p)), C.uint32_t(crc)))
+}

pkg/utils/checksum/crc32c/crc32c_linux.go

@@ -1,38 +1,75 @@
-/*
+// Copyright 2009 The Go Authors. All rights reserved.
- * Mini Object Storage, (C) 2014 Minio, Inc.
+// Use of this source code is governed by a BSD-style
- *
+// license that can be found in the LICENSE file.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 
+// Package crc32 implements the 32-bit cyclic redundancy check, or CRC-32,
+// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for
+// information.
 package crc32c
 
-// #include <stdint.h>
-// uint32_t crc32c_pcl(uint8_t *buf, int32_t len, uint32_t prev_crc);
-import "C"
 import (
-	"errors"
+	"hash"
-	"unsafe"
+	"io"
 )
 
-func Crc32c(buffer []byte) (uint32, error) {
+// The size of a CRC-32 checksum in bytes.
-	var length = len(buffer)
+const Size = 4
-	if length == 0 {
+
-		return 0, errors.New("Invalid input")
+// digest represents the partial evaluation of a checksum.
-	}
+type digest struct {
+	crc uint32
+}
+
+// New creates a new hash.Hash32 computing the CRC-32 checksum
+// using the polynomial represented by the Table.
+func New() hash.Hash32 {
+	return &digest{crc: 0}
+}
+
+func (d *digest) Size() int { return Size }
+
+func (d *digest) BlockSize() int { return 1 }
+
+func (d *digest) Sum(in []byte) []byte {
+	s := d.crc
+	return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s))
+}
+
+func (d *digest) Sum32() uint32 { return d.crc }
 
-	var cbuf *C.uint8_t
+func (d *digest) Reset() { d.crc = 0 }
-	cbuf = (*C.uint8_t)(unsafe.Pointer(&buffer[0]))
-	crc := C.crc32c_pcl(cbuf, C.int32_t(length), C.uint32_t(0))
 
-	return uint32(crc), nil
+// Update returns the result of adding the bytes in p to the crc.
+func (d *digest) update(crc uint32, p []byte) uint32 {
+	return updateCastanagoliPCL(crc, p)
+}
+
+func (d *digest) Write(p []byte) (n int, err error) {
+	d.crc = d.update(d.crc, p)
+	return len(p), nil
+}
+
+// Convenience functions
+
+func Sum32(data []byte) uint32 {
+	crc32 := New()
+	crc32.Reset()
+	crc32.Write(data)
+	return crc32.Sum32()
+}
+
+func Sum(reader io.Reader) (uint32, error) {
+	h := New()
+	var err error
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		h.Write(byteBuffer)
+	}
+	if err != io.EOF {
+		return 0, err
+	}
+	return h.Sum32(), nil
 }

pkg/utils/checksum/crc32c/crc32c_test.go

@@ -1,24 +1,74 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package crc32c
 
 import (
-	. "gopkg.in/check.v1"
 	"testing"
 )
 
-func Test(t *testing.T) { TestingT(t) }
+type test struct {
-
+	castagnoli uint32
-type MySuite struct{}
+	in         string
+}
 
-var _ = Suite(&MySuite{})
+var golden = []test{
+	{0x0, ""},
+	{0x93ad1061, "a"},
+	{0x13c35ee4, "ab"},
+	{0x562f9ccd, "abc"},
+	{0xdaaf41f6, "abcd"},
+	{0x8122a0a2, "abcde"},
+	{0x496937b, "abcdef"},
+	{0x5d199e2c, "abcdefg"},
+	{0x86bc933d, "abcdefgh"},
+	{0x9639f15f, "abcdefghi"},
+	{0x584645c, "abcdefghij"},
+	{0x8c13a060, "Discard medicine more than two years old."},
+	{0x629077d4, "He who has a shady past knows that nice guys finish last."},
+	{0xd20036a4, "I wouldn't marry him with a ten foot pole."},
+	{0xf283b768, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{0x9cd61a9f, "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{0x637702f5, "Nepal premier won't resign."},
+	{0x6c595588, "For every action there is an equal and opposite government program."},
+	{0x19532076, "His money is twice tainted: 'taint yours and 'taint mine."},
+	{0x9b82c857, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{0x2b485952, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{0xd3d0980c, "size:  a.out:  bad magic"},
+	{0x12aad0bb, "The major problem is with sendmail.  -Mark Horton"},
+	{0x83a0339b, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{0x1eb28fde, "If the enemy is within range, then so are you."},
+	{0xce34d559, "It's well we cannot hear the screams/That we create in others' dreams."},
+	{0x71576691, "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{0x54bf536f, "C is as portable as Stonehedge!!"},
+	{0x2313a94d, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{0x9d4e3629, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{0xc9991fb9, "How can you write a big system without C++?  -Paul Glick"},
+}
 
-func (s *MySuite) TestCrc32c(c *C) {
+func TestGolden(t *testing.T) {
-	data_1 := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+	for _, g := range golden {
-	crc, err := Crc32c(data_1)
+		s := Sum32([]byte(g.in))
-	c.Assert(err, IsNil)
+		if s != g.castagnoli {
+			t.Errorf("Castagnoli(%s) = 0x%x want 0x%x", g.in, s, g.castagnoli)
+		}
+	}
+}
 
-	data_2 := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+func BenchmarkCrc32KB(b *testing.B) {
-	newcrc, newerr := Crc32c(data_2)
+	b.SetBytes(1024)
-	c.Assert(newerr, IsNil)
+	data := make([]byte, 1024)
+	for i := range data {
+		data[i] = byte(i)
+	}
+	h := New()
+	in := make([]byte, 0, h.Size())
 
-	c.Assert(crc, Equals, newcrc)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		h.Reset()
+		h.Write(data)
+		h.Sum(in)
+	}
 }