From 2278df9910a69c844722a8b554b6726cd09cd467 Mon Sep 17 00:00:00 2001 From: "Frederick F. Kautz IV" Date: Sun, 21 Dec 2014 13:01:27 +1300 Subject: [PATCH] Adding minio-hash with streaming crypto hashes --- cmd/minio-hash/minio-hash-options.go | 126 +++++++++++++++++++++++++++ cmd/minio-hash/minio-hash.go | 31 +++++++ cmd/minio-hash/minio-hash.md | 13 +++ pkgs/crypto/md5/md5.go | 22 +++++ pkgs/crypto/md5/md5_test.go | 23 +++++ pkgs/crypto/sha1/sha1.go | 21 ++++- pkgs/crypto/sha1/sha1_test.go | 13 ++- pkgs/crypto/sha256/sha256.go | 20 +++++ pkgs/crypto/sha256/sha256_test.go | 23 +++++ pkgs/crypto/sha512/sha512.go | 20 +++++ pkgs/crypto/sha512/sha512_test.go | 23 +++++ 11 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 cmd/minio-hash/minio-hash-options.go create mode 100644 cmd/minio-hash/minio-hash.go create mode 100644 cmd/minio-hash/minio-hash.md create mode 100644 pkgs/crypto/md5/md5.go create mode 100644 pkgs/crypto/md5/md5_test.go create mode 100644 pkgs/crypto/sha256/sha256_test.go create mode 100644 pkgs/crypto/sha512/sha512_test.go diff --git a/cmd/minio-hash/minio-hash-options.go b/cmd/minio-hash/minio-hash-options.go new file mode 100644 index 000000000..4a07456eb --- /dev/null +++ b/cmd/minio-hash/minio-hash-options.go @@ -0,0 +1,126 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "encoding/binary" + "fmt" + "io/ioutil" + "log" + "os" + + "github.com/codegangsta/cli" + "github.com/minio-io/minio/pkgs/crypto/md5" + "github.com/minio-io/minio/pkgs/crypto/sha1" + "github.com/minio-io/minio/pkgs/crypto/sha256" + "github.com/minio-io/minio/pkgs/crypto/sha512" +) + +var Options = []cli.Command{ + Md5sum, + Sha1sum, + // Sha1sumFast, // not working + Sha256sum, + Sha512sum, +} + +var Md5sum = cli.Command{ + Name: "md5sum", + Usage: "", + Description: ` +`, + Action: doMd5sum, +} + +var Sha1sum = cli.Command{ + Name: "sha1sum", + Usage: "", + Description: ` +`, + Action: doSha1sum, +} + +var Sha1sumFast = cli.Command{ + Name: "sha1sum-fast", + Usage: "", + Description: ` +`, + Action: doSha1sumFast, +} + +var Sha256sum = cli.Command{ + Name: "sha256sum", + Usage: "", + Description: ` +`, + Action: doSha256sum, +} + +var Sha512sum = cli.Command{ + Name: "sha512sum", + Usage: "", + Description: ` +`, + Action: doSha512sum, +} + +func doMd5sum(c *cli.Context) { + hash, err := md5.Sum(os.Stdin) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%x", hash) +} + +func doSha1sum(c *cli.Context) { + hash, err := sha1.Sum(os.Stdin) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%x", hash) +} + +func doSha1sumFast(c *cli.Context) { + buffer, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Fatal(err) + } + hash, err := sha1.Sha1(buffer) + if err != nil { + log.Fatal(err) + } + var bytesBuffer bytes.Buffer + binary.Write(&bytesBuffer, binary.LittleEndian, hash) + fmt.Printf("%x", bytesBuffer.Bytes()) +} + +func doSha256sum(c *cli.Context) { + hash, err := sha256.Sum(os.Stdin) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%x", hash) +} + +func doSha512sum(c *cli.Context) { + hash, err := sha512.Sum(os.Stdin) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%x", hash) +} diff --git a/cmd/minio-hash/minio-hash.go b/cmd/minio-hash/minio-hash.go new file mode 100644 index 000000000..14a60d4e9 --- /dev/null +++ b/cmd/minio-hash/minio-hash.go @@ -0,0 +1,31 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "os" + + "github.com/codegangsta/cli" +) + +func main() { + app := cli.NewApp() + app.Name = "minio-hash" + app.Usage = "" + app.Commands = Options + app.Run(os.Args) +} diff --git a/cmd/minio-hash/minio-hash.md b/cmd/minio-hash/minio-hash.md new file mode 100644 index 000000000..289db02fa --- /dev/null +++ b/cmd/minio-hash/minio-hash.md @@ -0,0 +1,13 @@ +% MINIO(1) Minio Manual +% Minio community +% December 2014 +# NAME +md5sum - + +# SYNOPSIS + +# DESCRIPTION + +# EXAMPLES + +# AUTHORS \ No newline at end of file diff --git a/pkgs/crypto/md5/md5.go b/pkgs/crypto/md5/md5.go new file mode 100644 index 000000000..765949d5b --- /dev/null +++ b/pkgs/crypto/md5/md5.go @@ -0,0 +1,22 @@ +package md5 + +import ( + "crypto/md5" + "io" +) + +func Sum(reader io.Reader) ([]byte, error) { + hash := md5.New() + var err error + for err == nil { + length := 0 + byteBuffer := make([]byte, 1024*1024) + length, err = reader.Read(byteBuffer) + byteBuffer = byteBuffer[0:length] + hash.Write(byteBuffer) + } + if err != io.EOF { + return nil, err + } + return hash.Sum(nil), nil +} diff --git a/pkgs/crypto/md5/md5_test.go b/pkgs/crypto/md5/md5_test.go new file mode 100644 index 000000000..1e29e87dd --- /dev/null +++ b/pkgs/crypto/md5/md5_test.go @@ -0,0 +1,23 @@ +package md5 + +import ( + "bytes" + "encoding/hex" + "testing" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type MySuite struct{} + +var _ = Suite(&MySuite{}) + +func (s *MySuite) TestMd5sum(c *C) { + testString := []byte("Test string") + expectedHash, _ := hex.DecodeString("0fd3dbec9730101bff92acc820befc34") + hash, err := Sum(bytes.NewBuffer(testString)) + c.Assert(err, IsNil) + c.Assert(bytes.Equal(expectedHash, hash), Equals, true) +} diff --git a/pkgs/crypto/sha1/sha1.go b/pkgs/crypto/sha1/sha1.go index cef0cc754..c9c142baf 100644 --- a/pkgs/crypto/sha1/sha1.go +++ b/pkgs/crypto/sha1/sha1.go @@ -23,9 +23,12 @@ package sha1 // void sha1_transform(int32_t *hash, const char* input, size_t num_blocks); import "C" import ( + gosha1 "crypto/sha1" "errors" - "github.com/minio-io/minio/pkgs/cpu" + "io" "unsafe" + + "github.com/minio-io/minio/pkgs/cpu" ) const ( @@ -69,3 +72,19 @@ func Sha1(buffer []byte) ([]int32, error) { return shbuf, nil } + +func Sum(reader io.Reader) ([]byte, error) { + hash := gosha1.New() + var err error + for err == nil { + length := 0 + byteBuffer := make([]byte, 1024*1024) + length, err = reader.Read(byteBuffer) + byteBuffer = byteBuffer[0:length] + hash.Write(byteBuffer) + } + if err != io.EOF { + return nil, err + } + return hash.Sum(nil), nil +} diff --git a/pkgs/crypto/sha1/sha1_test.go b/pkgs/crypto/sha1/sha1_test.go index 88d77f04c..4cc99619d 100644 --- a/pkgs/crypto/sha1/sha1_test.go +++ b/pkgs/crypto/sha1/sha1_test.go @@ -1,8 +1,11 @@ package sha1 import ( - . "gopkg.in/check.v1" + "bytes" + "encoding/hex" "testing" + + . "gopkg.in/check.v1" ) func Test(t *testing.T) { TestingT(t) } @@ -24,3 +27,11 @@ func (s *MySuite) TestSha1(c *C) { c.Assert(sha, DeepEquals, newsha) } + +func (s *MySuite) TestStreamingSha1(c *C) { + testString := []byte("Test string") + expectedHash, _ := hex.DecodeString("18af819125b70879d36378431c4e8d9bfa6a2599") + hash, err := Sum(bytes.NewBuffer(testString)) + c.Assert(err, IsNil) + c.Assert(bytes.Equal(expectedHash, hash), Equals, true) +} diff --git a/pkgs/crypto/sha256/sha256.go b/pkgs/crypto/sha256/sha256.go index e330e9010..ff208950f 100644 --- a/pkgs/crypto/sha256/sha256.go +++ b/pkgs/crypto/sha256/sha256.go @@ -17,6 +17,10 @@ package sha256 // #define SHA256_H6 0x1f83d9abUL // #define SHA256_H7 0x5be0cd19UL import "C" +import ( + gosha256 "crypto/sha256" + "io" +) /* func Sha256(buffer []byte) ([]uint32, error) { @@ -37,3 +41,19 @@ func Sha256(buffer []byte) ([]uint32, error) { } } */ + +func Sum(reader io.Reader) ([]byte, error) { + hash := gosha256.New() + var err error + for err == nil { + length := 0 + byteBuffer := make([]byte, 1024*1024) + length, err = reader.Read(byteBuffer) + byteBuffer = byteBuffer[0:length] + hash.Write(byteBuffer) + } + if err != io.EOF { + return nil, err + } + return hash.Sum(nil), nil +} diff --git a/pkgs/crypto/sha256/sha256_test.go b/pkgs/crypto/sha256/sha256_test.go new file mode 100644 index 000000000..e16c98fa0 --- /dev/null +++ b/pkgs/crypto/sha256/sha256_test.go @@ -0,0 +1,23 @@ +package sha256 + +import ( + "bytes" + "encoding/hex" + "testing" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type MySuite struct{} + +var _ = Suite(&MySuite{}) + +func (s *MySuite) TestSha256Stream(c *C) { + testString := []byte("Test string") + expectedHash, _ := hex.DecodeString("a3e49d843df13c2e2a7786f6ecd7e0d184f45d718d1ac1a8a63e570466e489dd") + hash, err := Sum(bytes.NewBuffer(testString)) + c.Assert(err, IsNil) + c.Assert(bytes.Equal(expectedHash, hash), Equals, true) +} diff --git a/pkgs/crypto/sha512/sha512.go b/pkgs/crypto/sha512/sha512.go index 1666484aa..3bd4180a0 100644 --- a/pkgs/crypto/sha512/sha512.go +++ b/pkgs/crypto/sha512/sha512.go @@ -7,3 +7,23 @@ package sha512 // void sha512_transform_ssse3(const void* M, void* D, uint64_t L); // void sha512_transform_rorx(const void* M, void* D, uint64_t L); import "C" +import ( + gosha512 "crypto/sha512" + "io" +) + +func Sum(reader io.Reader) ([]byte, error) { + hash := gosha512.New() + var err error + for err == nil { + length := 0 + byteBuffer := make([]byte, 1024*1024) + length, err = reader.Read(byteBuffer) + byteBuffer = byteBuffer[0:length] + hash.Write(byteBuffer) + } + if err != io.EOF { + return nil, err + } + return hash.Sum(nil), nil +} diff --git a/pkgs/crypto/sha512/sha512_test.go b/pkgs/crypto/sha512/sha512_test.go new file mode 100644 index 000000000..cf36ac239 --- /dev/null +++ b/pkgs/crypto/sha512/sha512_test.go @@ -0,0 +1,23 @@ +package sha512 + +import ( + "bytes" + "encoding/hex" + "testing" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type MySuite struct{} + +var _ = Suite(&MySuite{}) + +func (s *MySuite) TestSha512Stream(c *C) { + testString := []byte("Test string") + expectedHash, _ := hex.DecodeString("811aa0c53c0039b6ead0ca878b096eed1d39ed873fd2d2d270abfb9ca620d3ed561c565d6dbd1114c323d38e3f59c00df475451fc9b30074f2abda3529df2fa7") + hash, err := Sum(bytes.NewBuffer(testString)) + c.Assert(err, IsNil) + c.Assert(bytes.Equal(expectedHash, hash), Equals, true) +}