From 856b7680dfd9243194f429c07ea542fad42f28c1 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 6 Mar 2015 02:35:54 -0800 Subject: [PATCH] Now able to use crc32c and sha1 on Darwin, with OSX specific code --- pkg/utils/checksum/crc32c/crc32c_darwin.go | 6 +- ...{crc32c_intel.go => crc32c_intel_linux.go} | 0 .../{crc32c_linux_amd64.S => crc32c_linux.S} | 0 .../{crc32c_test.go => crc32c_linux_test.go} | 0 pkg/utils/crypto/sha1/sha1_darwin.go | 167 ++++++++++++++++++ .../sha1/{sha1_amd64.S => sha1_linux.S} | 0 .../crypto/sha1/{sha1.go => sha1_linux.go} | 0 pkg/utils/crypto/sha1/sha1_yasm_darwin.go | 2 +- pkg/utils/crypto/sha1/sha1block_darwin.go | 13 ++ .../{sha1block.go => sha1block_generic.go} | 16 -- pkg/utils/crypto/sha1/sha1block_linux.go | 21 +++ 11 files changed, 205 insertions(+), 20 deletions(-) rename pkg/utils/checksum/crc32c/{crc32c_intel.go => crc32c_intel_linux.go} (100%) rename pkg/utils/checksum/crc32c/{crc32c_linux_amd64.S => crc32c_linux.S} (100%) rename pkg/utils/checksum/crc32c/{crc32c_test.go => crc32c_linux_test.go} (100%) create mode 100644 pkg/utils/crypto/sha1/sha1_darwin.go rename pkg/utils/crypto/sha1/{sha1_amd64.S => sha1_linux.S} (100%) rename pkg/utils/crypto/sha1/{sha1.go => sha1_linux.go} (100%) create mode 100644 pkg/utils/crypto/sha1/sha1block_darwin.go rename pkg/utils/crypto/sha1/{sha1block.go => sha1block_generic.go} (79%) create mode 100644 pkg/utils/crypto/sha1/sha1block_linux.go diff --git a/pkg/utils/checksum/crc32c/crc32c_darwin.go b/pkg/utils/checksum/crc32c/crc32c_darwin.go index 3977e46d9..16e758924 100644 --- a/pkg/utils/checksum/crc32c/crc32c_darwin.go +++ b/pkg/utils/checksum/crc32c/crc32c_darwin.go @@ -17,7 +17,7 @@ package crc32c import ( - "errors" + "io" "hash/crc32" ) @@ -35,7 +35,7 @@ func Sum32(buffer []byte) uint32 { // Sum - io.Reader based crc helper func Sum(reader io.Reader) (uint32, error) { - h := New() + h := crc32.New(castanagoliTable) var err error for err == nil { length := 0 @@ -45,7 +45,7 @@ func Sum(reader io.Reader) (uint32, error) { h.Write(byteBuffer) } if err != io.EOF { - return nil, err + return 0, err } return h.Sum32(), nil } diff --git a/pkg/utils/checksum/crc32c/crc32c_intel.go b/pkg/utils/checksum/crc32c/crc32c_intel_linux.go similarity index 100% rename from pkg/utils/checksum/crc32c/crc32c_intel.go rename to pkg/utils/checksum/crc32c/crc32c_intel_linux.go diff --git a/pkg/utils/checksum/crc32c/crc32c_linux_amd64.S b/pkg/utils/checksum/crc32c/crc32c_linux.S similarity index 100% rename from pkg/utils/checksum/crc32c/crc32c_linux_amd64.S rename to pkg/utils/checksum/crc32c/crc32c_linux.S diff --git a/pkg/utils/checksum/crc32c/crc32c_test.go b/pkg/utils/checksum/crc32c/crc32c_linux_test.go similarity index 100% rename from pkg/utils/checksum/crc32c/crc32c_test.go rename to pkg/utils/checksum/crc32c/crc32c_linux_test.go diff --git a/pkg/utils/crypto/sha1/sha1_darwin.go b/pkg/utils/crypto/sha1/sha1_darwin.go new file mode 100644 index 000000000..7656c8838 --- /dev/null +++ b/pkg/utils/crypto/sha1/sha1_darwin.go @@ -0,0 +1,167 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file of +// Golang project: +// https://github.com/golang/go/blob/master/LICENSE + +// Using this part of Minio codebase under the license +// Apache License Version 2.0 with modifications + +// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174. +package sha1 + +import ( + "hash" + "io" + + "github.com/minio-io/minio/pkg/utils/cpu" +) + +// The size of a SHA1 checksum in bytes. +const Size = 20 + +// The blocksize of SHA1 in bytes. +const BlockSize = 64 + +const ( + chunk = 64 + init0 = 0x67452301 + init1 = 0xEFCDAB89 + init2 = 0x98BADCFE + init3 = 0x10325476 + init4 = 0xC3D2E1F0 +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + h [5]uint32 + x [chunk]byte + nx int + len uint64 +} + +// Reset digest +func (d *digest) Reset() { + d.h[0] = init0 + d.h[1] = init1 + d.h[2] = init2 + d.h[3] = init3 + d.h[4] = init4 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA1 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func block(dig *digest, p []byte) { + switch true { + case cpu.HasSSE41() == true: + blockSSE3(dig, p) + default: + blockGeneric(dig, p) + } +} + +// Return output size +func (d *digest) Size() int { return Size } + +// Return checksum blocksize +func (d *digest) BlockSize() int { return BlockSize } + +// Write to digest +func (d *digest) Write(p []byte) (nn int, err error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := copy(d.x[d.nx:], p) + d.nx += n + if d.nx == chunk { + block(d, d.x[:]) + d.nx = 0 + } + p = p[n:] + } + if len(p) >= chunk { + n := len(p) &^ (chunk - 1) + block(d, p[:n]) + p = p[n:] + } + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +// Return checksum bytes +func (d0 *digest) Sum(in []byte) []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := *d0 + hash := d.checkSum() + return append(in, hash[:]...) +} + +// Intermediate checksum function +func (d *digest) checkSum() [Size]byte { + len := d.len + // Padding. Add a 1 bit and 0 bits until 56 bytes mod 64. + var tmp [64]byte + tmp[0] = 0x80 + if len%64 < 56 { + d.Write(tmp[0 : 56-len%64]) + } else { + d.Write(tmp[0 : 64+56-len%64]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 8; i++ { + tmp[i] = byte(len >> (56 - 8*i)) + } + d.Write(tmp[0:8]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + var digest [Size]byte + for i, s := range d.h { + digest[i*4] = byte(s >> 24) + digest[i*4+1] = byte(s >> 16) + digest[i*4+2] = byte(s >> 8) + digest[i*4+3] = byte(s) + } + + return digest +} + +/// Convenience functions + +// Sum1 - single caller sha1 helper +func Sum1(data []byte) [Size]byte { + var d digest + d.Reset() + d.Write(data) + return d.checkSum() +} + +// Sum - io.Reader based streaming sha1 helper +func Sum(reader io.Reader) ([]byte, error) { + h := New() + var err error + for err == nil { + length := 0 + byteBuffer := make([]byte, 1024*1024) + length, err = reader.Read(byteBuffer) + byteBuffer = byteBuffer[0:length] + h.Write(byteBuffer) + } + if err != io.EOF { + return nil, err + } + return h.Sum(nil), nil +} diff --git a/pkg/utils/crypto/sha1/sha1_amd64.S b/pkg/utils/crypto/sha1/sha1_linux.S similarity index 100% rename from pkg/utils/crypto/sha1/sha1_amd64.S rename to pkg/utils/crypto/sha1/sha1_linux.S diff --git a/pkg/utils/crypto/sha1/sha1.go b/pkg/utils/crypto/sha1/sha1_linux.go similarity index 100% rename from pkg/utils/crypto/sha1/sha1.go rename to pkg/utils/crypto/sha1/sha1_linux.go diff --git a/pkg/utils/crypto/sha1/sha1_yasm_darwin.go b/pkg/utils/crypto/sha1/sha1_yasm_darwin.go index 233c883c2..2825b0d4b 100644 --- a/pkg/utils/crypto/sha1/sha1_yasm_darwin.go +++ b/pkg/utils/crypto/sha1/sha1_yasm_darwin.go @@ -2,4 +2,4 @@ package sha1 -//go:generate yasm -f macho64 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso +//go:generate yasm -f macho64 -DINTEL_SHA1_UPDATE_FUNCNAME=_sha1_update_intel sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso diff --git a/pkg/utils/crypto/sha1/sha1block_darwin.go b/pkg/utils/crypto/sha1/sha1block_darwin.go new file mode 100644 index 000000000..7e49ecad1 --- /dev/null +++ b/pkg/utils/crypto/sha1/sha1block_darwin.go @@ -0,0 +1,13 @@ +// +build amd64 + +package sha1 + +// #include +// #include +// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks); +import "C" +import "unsafe" + +func blockSSE3(dig *digest, p []byte) { + C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk)) +} diff --git a/pkg/utils/crypto/sha1/sha1block.go b/pkg/utils/crypto/sha1/sha1block_generic.go similarity index 79% rename from pkg/utils/crypto/sha1/sha1block.go rename to pkg/utils/crypto/sha1/sha1block_generic.go index 82b4c9144..81f775b82 100644 --- a/pkg/utils/crypto/sha1/sha1block.go +++ b/pkg/utils/crypto/sha1/sha1block_generic.go @@ -2,14 +2,6 @@ package sha1 -// #cgo CFLAGS: -DHAS_AVX2 -// #include -// #include -// void sha1_transform(int32_t *hash, const char* input, size_t num_blocks); -// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks ); -import "C" -import "unsafe" - const ( _K0 = 0x5A827999 _K1 = 0x6ED9EBA1 @@ -17,14 +9,6 @@ const ( _K3 = 0xCA62C1D6 ) -func blockAVX2(dig *digest, p []byte) { - C.sha1_transform((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk)) -} - -func blockSSE3(dig *digest, p []byte) { - C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk)) -} - // blockGeneric is a portable, pure Go version of the SHA1 block step. // It's used by sha1block_generic.go and tests. func blockGeneric(dig *digest, p []byte) { diff --git a/pkg/utils/crypto/sha1/sha1block_linux.go b/pkg/utils/crypto/sha1/sha1block_linux.go new file mode 100644 index 000000000..4eedae796 --- /dev/null +++ b/pkg/utils/crypto/sha1/sha1block_linux.go @@ -0,0 +1,21 @@ +// +build amd64 + +package sha1 + +// #cgo CFLAGS: -DHAS_AVX2 +// #include +// #include +// void sha1_transform(int32_t *hash, const char* input, size_t num_blocks); +// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks); +import "C" +import "unsafe" + +func blockAVX2(dig *digest, p []byte) { + C.sha1_transform((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk)) +} + +func blockSSE3(dig *digest, p []byte) { + C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk)) +} + +