From 19a4998fcbef4f364f624c9705a1eda64eb80fa9 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 20 Feb 2015 16:28:36 -0800 Subject: [PATCH] SHA512 Implemention with Intel assembly code --- pkg/utils/crypto/sha512/TODO | 0 pkg/utils/crypto/sha512/sha512.go | 161 +++++++++++++++++++++++-- pkg/utils/crypto/sha512/sha512_gen.go | 65 ++++++++++ pkg/utils/crypto/sha512/sha512_test.go | 115 ++++++++++++++++-- pkg/utils/crypto/sha512/sha512block.go | 23 ++++ 5 files changed, 340 insertions(+), 24 deletions(-) delete mode 100644 pkg/utils/crypto/sha512/TODO create mode 100644 pkg/utils/crypto/sha512/sha512_gen.go create mode 100644 pkg/utils/crypto/sha512/sha512block.go diff --git a/pkg/utils/crypto/sha512/TODO b/pkg/utils/crypto/sha512/TODO deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkg/utils/crypto/sha512/sha512.go b/pkg/utils/crypto/sha512/sha512.go index 3bd4180a0..cc42a25ce 100644 --- a/pkg/utils/crypto/sha512/sha512.go +++ b/pkg/utils/crypto/sha512/sha512.go @@ -1,29 +1,168 @@ -// +build amd64 - +// Package sha512 implements the SHA512 hash algorithms as defined +// in FIPS 180-2. package sha512 -// #include -// void sha512_transform_avx(const void* M, void* D, uint64_t L); -// void sha512_transform_ssse3(const void* M, void* D, uint64_t L); -// void sha512_transform_rorx(const void* M, void* D, uint64_t L); -import "C" import ( - gosha512 "crypto/sha512" + "hash" "io" + + "github.com/minio-io/minio/pkg/utils/cpu" +) + +// The size of a SHA512 checksum in bytes. +const Size = 64 + +// The blocksize of SHA512 in bytes. +const BlockSize = 128 + +const ( + chunk = 128 + init0 = 0x6a09e667f3bcc908 + init1 = 0xbb67ae8584caa73b + init2 = 0x3c6ef372fe94f82b + init3 = 0xa54ff53a5f1d36f1 + init4 = 0x510e527fade682d1 + init5 = 0x9b05688c2b3e6c1f + init6 = 0x1f83d9abfb41bd6b + init7 = 0x5be0cd19137e2179 ) +// digest represents the partial evaluation of a checksum. +type digest struct { + h [8]uint64 + x [chunk]byte + nx int + len uint64 +} + +func block(dig *digest, p []byte) { + switch true { + case cpu.HasAVX2() == true: + blockAVX2(dig, p) + case cpu.HasAVX() == true: + blockAVX(dig, p) + case cpu.HasSSE41() == true: + blockSSE(dig, p) + } +} + +func (d *digest) Reset() { + d.h[0] = init0 + d.h[1] = init1 + d.h[2] = init2 + d.h[3] = init3 + d.h[4] = init4 + d.h[5] = init5 + d.h[6] = init6 + d.h[7] = init7 + d.nx = 0 + d.len = 0 +} + +// New returns a new hash.Hash computing the SHA512 checksum. +func New() hash.Hash { + d := new(digest) + d.Reset() + return d +} + +func (d *digest) Size() int { + return Size +} + +func (d *digest) BlockSize() int { return BlockSize } + +func (d *digest) Write(p []byte) (nn int, err error) { + nn = len(p) + d.len += uint64(nn) + if d.nx > 0 { + n := copy(d.x[d.nx:], p) + d.nx += n + if d.nx == chunk { + block(d, d.x[:]) + d.nx = 0 + } + p = p[n:] + } + if len(p) >= chunk { + n := len(p) &^ (chunk - 1) + block(d, p[:n]) + p = p[n:] + } + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d0 *digest) Sum(in []byte) []byte { + // Make a copy of d0 so that caller can keep writing and summing. + d := new(digest) + *d = *d0 + hash := d.checkSum() + return append(in, hash[:]...) +} + +func (d *digest) checkSum() [Size]byte { + // Padding. Add a 1 bit and 0 bits until 112 bytes mod 128. + len := d.len + var tmp [128]byte + tmp[0] = 0x80 + if len%128 < 112 { + d.Write(tmp[0 : 112-len%128]) + } else { + d.Write(tmp[0 : 128+112-len%128]) + } + + // Length in bits. + len <<= 3 + for i := uint(0); i < 16; i++ { + tmp[i] = byte(len >> (120 - 8*i)) + } + d.Write(tmp[0:16]) + + if d.nx != 0 { + panic("d.nx != 0") + } + + h := d.h[:] + + var digest [Size]byte + for i, s := range h { + digest[i*8] = byte(s >> 56) + digest[i*8+1] = byte(s >> 48) + digest[i*8+2] = byte(s >> 40) + digest[i*8+3] = byte(s >> 32) + digest[i*8+4] = byte(s >> 24) + digest[i*8+5] = byte(s >> 16) + digest[i*8+6] = byte(s >> 8) + digest[i*8+7] = byte(s) + } + + return digest +} + +// Convenience functions + +func Sum512(data []byte) [Size]byte { + var d digest + d.Reset() + d.Write(data) + return d.checkSum() +} + func Sum(reader io.Reader) ([]byte, error) { - hash := gosha512.New() + h := New() var err error for err == nil { length := 0 byteBuffer := make([]byte, 1024*1024) length, err = reader.Read(byteBuffer) byteBuffer = byteBuffer[0:length] - hash.Write(byteBuffer) + h.Write(byteBuffer) } if err != io.EOF { return nil, err } - return hash.Sum(nil), nil + return h.Sum(nil), nil } diff --git a/pkg/utils/crypto/sha512/sha512_gen.go b/pkg/utils/crypto/sha512/sha512_gen.go new file mode 100644 index 000000000..d95aac70c --- /dev/null +++ b/pkg/utils/crypto/sha512/sha512_gen.go @@ -0,0 +1,65 @@ +// +build ignore + +package main + +import ( + "crypto/sha512" + "encoding/hex" + "fmt" + "io" + "os" + "time" + + sha512intel "github.com/minio-io/minio/pkg/utils/crypto/sha512" +) + +func SumIntel(reader io.Reader) ([]byte, error) { + h := sha512intel.New() + var err error + for err == nil { + length := 0 + byteBuffer := make([]byte, 1024*1024) + length, err = reader.Read(byteBuffer) + byteBuffer = byteBuffer[0:length] + h.Write(byteBuffer) + } + if err != io.EOF { + return nil, err + } + return h.Sum(nil), nil +} + +func Sum(reader io.Reader) ([]byte, error) { + k := sha512.New() + var err error + for err == nil { + length := 0 + byteBuffer := make([]byte, 1024*1024) + length, err = reader.Read(byteBuffer) + byteBuffer = byteBuffer[0:length] + k.Write(byteBuffer) + } + if err != io.EOF { + return nil, err + } + return k.Sum(nil), nil +} + +func main() { + fmt.Println("-- start") + + file1, _ := os.Open("filename1") + defer file1.Close() + stark := time.Now() + sum, _ := Sum(file1) + endk := time.Since(stark) + + file2, _ := os.Open("filename2") + defer file2.Close() + starth := time.Now() + sumSSE, _ := SumIntel(file2) + endh := time.Since(starth) + + fmt.Println("std(", endk, ")", "ssse3(", endh, ")") + fmt.Println(hex.EncodeToString(sum), hex.EncodeToString(sumSSE)) +} diff --git a/pkg/utils/crypto/sha512/sha512_test.go b/pkg/utils/crypto/sha512/sha512_test.go index cf36ac239..756cefb9d 100644 --- a/pkg/utils/crypto/sha512/sha512_test.go +++ b/pkg/utils/crypto/sha512/sha512_test.go @@ -1,23 +1,112 @@ +// SHA512 hash algorithm. See FIPS 180-2. + package sha512 import ( - "bytes" - "encoding/hex" + "fmt" + "io" "testing" - - . "gopkg.in/check.v1" ) -func Test(t *testing.T) { TestingT(t) } +type sha512Test struct { + out string + in string +} + +var golden = []sha512Test{ + {"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", ""}, + {"1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75", "a"}, + {"2d408a0717ec188158278a796c689044361dc6fdde28d6f04973b80896e1823975cdbf12eb63f9e0591328ee235d80e9b5bf1aa6a44f4617ff3caf6400eb172d", "ab"}, + {"ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", "abc"}, + {"d8022f2060ad6efd297ab73dcc5355c9b214054b0d1776a136a669d26a7d3b14f73aa0d0ebff19ee333368f0164b6419a96da49e3e481753e7e96b716bdccb6f", "abcd"}, + {"878ae65a92e86cac011a570d4c30a7eaec442b85ce8eca0c2952b5e3cc0628c2e79d889ad4d5c7c626986d452dd86374b6ffaa7cd8b67665bef2289a5c70b0a1", "abcde"}, + {"e32ef19623e8ed9d267f657a81944b3d07adbb768518068e88435745564e8d4150a0a703be2a7d88b61e3d390c2bb97e2d4c311fdc69d6b1267f05f59aa920e7", "abcdef"}, + {"d716a4188569b68ab1b6dfac178e570114cdf0ea3a1cc0e31486c3e41241bc6a76424e8c37ab26f096fc85ef9886c8cb634187f4fddff645fb099f1ff54c6b8c", "abcdefg"}, + {"a3a8c81bc97c2560010d7389bc88aac974a104e0e2381220c6e084c4dccd1d2d17d4f86db31c2a851dc80e6681d74733c55dcd03dd96f6062cdda12a291ae6ce", "abcdefgh"}, + {"f22d51d25292ca1d0f68f69aedc7897019308cc9db46efb75a03dd494fc7f126c010e8ade6a00a0c1a5f1b75d81e0ed5a93ce98dc9b833db7839247b1d9c24fe", "abcdefghi"}, + {"ef6b97321f34b1fea2169a7db9e1960b471aa13302a988087357c520be957ca119c3ba68e6b4982c019ec89de3865ccf6a3cda1fe11e59f98d99f1502c8b9745", "abcdefghij"}, + {"2210d99af9c8bdecda1b4beff822136753d8342505ddce37f1314e2cdbb488c6016bdaa9bd2ffa513dd5de2e4b50f031393d8ab61f773b0e0130d7381e0f8a1d", "Discard medicine more than two years old."}, + {"a687a8985b4d8d0a24f115fe272255c6afaf3909225838546159c1ed685c211a203796ae8ecc4c81a5b6315919b3a64f10713da07e341fcdbb08541bf03066ce", "He who has a shady past knows that nice guys finish last."}, + {"8ddb0392e818b7d585ab22769a50df660d9f6d559cca3afc5691b8ca91b8451374e42bcdabd64589ed7c91d85f626596228a5c8572677eb98bc6b624befb7af8", "I wouldn't marry him with a ten foot pole."}, + {"26ed8f6ca7f8d44b6a8a54ae39640fa8ad5c673f70ee9ce074ba4ef0d483eea00bab2f61d8695d6b34df9c6c48ae36246362200ed820448bdc03a720366a87c6", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"}, + {"e5a14bf044be69615aade89afcf1ab0389d5fc302a884d403579d1386a2400c089b0dbb387ed0f463f9ee342f8244d5a38cfbc0e819da9529fbff78368c9a982", "The days of the digital watch are numbered. -Tom Stoppard"}, + {"420a1faa48919e14651bed45725abe0f7a58e0f099424c4e5a49194946e38b46c1f8034b18ef169b2e31050d1648e0b982386595f7df47da4b6fd18e55333015", "Nepal premier won't resign."}, + {"d926a863beadb20134db07683535c72007b0e695045876254f341ddcccde132a908c5af57baa6a6a9c63e6649bba0c213dc05fadcf9abccea09f23dcfb637fbe", "For every action there is an equal and opposite government program."}, + {"9a98dd9bb67d0da7bf83da5313dff4fd60a4bac0094f1b05633690ffa7f6d61de9a1d4f8617937d560833a9aaa9ccafe3fd24db418d0e728833545cadd3ad92d", "His money is twice tainted: 'taint yours and 'taint mine."}, + {"d7fde2d2351efade52f4211d3746a0780a26eec3df9b2ed575368a8a1c09ec452402293a8ea4eceb5a4f60064ea29b13cdd86918cd7a4faf366160b009804107", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"}, + {"b0f35ffa2697359c33a56f5c0cf715c7aeed96da9905ca2698acadb08fbc9e669bf566b6bd5d61a3e86dc22999bcc9f2224e33d1d4f32a228cf9d0349e2db518", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"}, + {"3d2e5f91778c9e66f7e061293aaa8a8fc742dd3b2e4f483772464b1144189b49273e610e5cccd7a81a19ca1fa70f16b10f1a100a4d8c1372336be8484c64b311", "size: a.out: bad magic"}, + {"b2f68ff58ac015efb1c94c908b0d8c2bf06f491e4de8e6302c49016f7f8a33eac3e959856c7fddbc464de618701338a4b46f76dbfaf9a1e5262b5f40639771c7", "The major problem is with sendmail. -Mark Horton"}, + {"d8c92db5fdf52cf8215e4df3b4909d29203ff4d00e9ad0b64a6a4e04dec5e74f62e7c35c7fb881bd5de95442123df8f57a489b0ae616bd326f84d10021121c57", "Give me a rock, paper and scissors and I will move the world. CCFestoon"}, + {"19a9f8dc0a233e464e8566ad3ca9b91e459a7b8c4780985b015776e1bf239a19bc233d0556343e2b0a9bc220900b4ebf4f8bdf89ff8efeaf79602d6849e6f72e", "If the enemy is within range, then so are you."}, + {"00b4c41f307bde87301cdc5b5ab1ae9a592e8ecbb2021dd7bc4b34e2ace60741cc362560bec566ba35178595a91932b8d5357e2c9cec92d393b0fa7831852476", "It's well we cannot hear the screams/That we create in others' dreams."}, + {"91eccc3d5375fd026e4d6787874b1dce201cecd8a27dbded5065728cb2d09c58a3d467bb1faf353bf7ba567e005245d5321b55bc344f7c07b91cb6f26c959be7", "You remind me of a TV show, but that's all right: I watch it anyway."}, + {"fabbbe22180f1f137cfdc9556d2570e775d1ae02a597ded43a72a40f9b485d500043b7be128fb9fcd982b83159a0d99aa855a9e7cc4240c00dc01a9bdf8218d7", "C is as portable as Stonehedge!!"}, + {"2ecdec235c1fa4fc2a154d8fba1dddb8a72a1ad73838b51d792331d143f8b96a9f6fcb0f34d7caa351fe6d88771c4f105040e0392f06e0621689d33b2f3ba92e", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"}, + {"7ad681f6f96f82f7abfa7ecc0334e8fa16d3dc1cdc45b60b7af43fe4075d2357c0c1d60e98350f1afb1f2fe7a4d7cd2ad55b88e458e06b73c40b437331f5dab4", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction. Lewis-Randall Rule"}, + {"833f9248ab4a3b9e5131f745fda1ffd2dd435b30e965957e78291c7ab73605fd1912b0794e5c233ab0a12d205a39778d19b83515d6a47003f19cdee51d98c7e0", "How can you write a big system without C++? -Paul Glick"}, +} + +func TestGolden(t *testing.T) { + for i := 0; i < len(golden); i++ { + g := golden[i] + s := fmt.Sprintf("%x", Sum512([]byte(g.in))) + if s != g.out { + t.Fatalf("Sum512 function: sha512(%s) = %s want %s", g.in, s, g.out) + } + c := New() + for j := 0; j < 3; j++ { + if j < 2 { + io.WriteString(c, g.in) + } else { + io.WriteString(c, g.in[0:len(g.in)/2]) + c.Sum(nil) + io.WriteString(c, g.in[len(g.in)/2:]) + } + s := fmt.Sprintf("%x", c.Sum(nil)) + if s != g.out { + t.Fatalf("sha512[%d](%s) = %s want %s", j, g.in, s, g.out) + } + c.Reset() + } + } +} + +func TestSize(t *testing.T) { + c := New() + if got := c.Size(); got != Size { + t.Errorf("Size = %d; want %d", got, Size) + } +} + +func TestBlockSize(t *testing.T) { + c := New() + if got := c.BlockSize(); got != BlockSize { + t.Errorf("BlockSize = %d; want %d", got, BlockSize) + } +} + +var bench = New() +var buf = make([]byte, 8192) -type MySuite struct{} +func benchmarkSize(b *testing.B, size int) { + b.SetBytes(int64(size)) + sum := make([]byte, bench.Size()) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[:size]) + bench.Sum(sum[:0]) + } +} + +func BenchmarkHash8Bytes(b *testing.B) { + benchmarkSize(b, 8) +} -var _ = Suite(&MySuite{}) +func BenchmarkHash1K(b *testing.B) { + benchmarkSize(b, 1024) +} -func (s *MySuite) TestSha512Stream(c *C) { - testString := []byte("Test string") - expectedHash, _ := hex.DecodeString("811aa0c53c0039b6ead0ca878b096eed1d39ed873fd2d2d270abfb9ca620d3ed561c565d6dbd1114c323d38e3f59c00df475451fc9b30074f2abda3529df2fa7") - hash, err := Sum(bytes.NewBuffer(testString)) - c.Assert(err, IsNil) - c.Assert(bytes.Equal(expectedHash, hash), Equals, true) +func BenchmarkHash8K(b *testing.B) { + benchmarkSize(b, 8192) } diff --git a/pkg/utils/crypto/sha512/sha512block.go b/pkg/utils/crypto/sha512/sha512block.go new file mode 100644 index 000000000..a58f25c2e --- /dev/null +++ b/pkg/utils/crypto/sha512/sha512block.go @@ -0,0 +1,23 @@ +// +build amd64 + +package sha512 + +// #cgo CFLAGS: -DHAS_SSE41 -DHAS_AVX -DHAS_AVX2 +// #include +// void sha512_transform_ssse3 (const void* M, void* D, uint64_t L); +// void sha512_transform_avx (const void* M, void* D, uint64_t L); +// void sha512_transform_rorx (const void* M, void* D, uint64_t L); +import "C" +import "unsafe" + +func blockSSE(dig *digest, p []byte) { + C.sha512_transform_ssse3(unsafe.Pointer(&p[0]), unsafe.Pointer(&dig.h[0]), (C.uint64_t)(len(p)/chunk)) +} + +func blockAVX(dig *digest, p []byte) { + C.sha512_transform_avx(unsafe.Pointer(&p[0]), unsafe.Pointer(&dig.h[0]), (C.uint64_t)(len(p)/chunk)) +} + +func blockAVX2(dig *digest, p []byte) { + C.sha512_transform_rorx(unsafe.Pointer(&p[0]), unsafe.Pointer(&dig.h[0]), (C.uint64_t)(len(p)/chunk)) +}