From e1e49085151e1f60f319be0861e3b4ee95db4a8e Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 14 Jul 2015 19:47:45 -0700 Subject: [PATCH] Wire up sha512 matching inside donut along with md5sum --- main.go | 2 +- pkg/donut/bucket.go | 20 +++++++++++++++----- pkg/donut/definitions.go | 12 ++++++------ pkg/donut/donut-v1.go | 4 ++-- pkg/donut/heal.go | 25 ++++++++++++++++--------- pkg/donut/interfaces.go | 3 --- pkg/donut/management.go | 33 --------------------------------- 7 files changed, 40 insertions(+), 59 deletions(-) diff --git a/main.go b/main.go index a372a4975..ce6d887b0 100644 --- a/main.go +++ b/main.go @@ -107,7 +107,7 @@ func main() { // set up app app := cli.NewApp() app.Name = "minio" - app.Version = Version + app.Version = getVersion() app.Compiled = getVersion() app.Author = "Minio.io" app.Usage = "Minimalist Object Storage" diff --git a/pkg/donut/bucket.go b/pkg/donut/bucket.go index 3be141db5..b7ef856ea 100644 --- a/pkg/donut/bucket.go +++ b/pkg/donut/bucket.go @@ -78,7 +78,7 @@ func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bu metadata.ACL = BucketACL(aclType) metadata.Created = t metadata.Metadata = make(map[string]string) - metadata.BucketObjects = make(map[string]interface{}) + metadata.BucketObjects = make(map[string]struct{}) return b, metadata, nil } @@ -455,10 +455,16 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta writer.CloseWithError(iodine.New(err, nil)) return } + expected512Sum, err := hex.DecodeString(objMetadata.SHA512Sum) + if err != nil { + writer.CloseWithError(iodine.New(err, nil)) + return + } hasher := md5.New() - mwriter := io.MultiWriter(writer, hasher) - switch len(readers) == 1 { - case false: + sum512hasher := sha256.New() + mwriter := io.MultiWriter(writer, hasher, sum512hasher) + switch len(readers) > 1 { + case true: if objMetadata.ErasureTechnique == "" { writer.CloseWithError(iodine.New(MissingErasureTechnique{}, nil)) return @@ -482,7 +488,7 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta } totalLeft = totalLeft - int64(objMetadata.BlockSize) } - case true: + case false: _, err := io.Copy(writer, readers[0]) if err != nil { writer.CloseWithError(iodine.New(err, nil)) @@ -494,6 +500,10 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta writer.CloseWithError(iodine.New(ChecksumMismatch{}, nil)) return } + if !bytes.Equal(expected512Sum, sum512hasher.Sum(nil)) { + writer.CloseWithError(iodine.New(ChecksumMismatch{}, nil)) + return + } writer.Close() return } diff --git a/pkg/donut/definitions.go b/pkg/donut/definitions.go index 75f4c5a18..23620ab1c 100644 --- a/pkg/donut/definitions.go +++ b/pkg/donut/definitions.go @@ -57,12 +57,12 @@ type AllBuckets struct { // BucketMetadata container for bucket level metadata type BucketMetadata struct { - Version string `json:"version"` - Name string `json:"name"` - ACL BucketACL `json:"acl"` - Created time.Time `json:"created"` - Metadata map[string]string `json:"metadata"` - BucketObjects map[string]interface{} `json:"objects"` + Version string `json:"version"` + Name string `json:"name"` + ACL BucketACL `json:"acl"` + Created time.Time `json:"created"` + Metadata map[string]string `json:"metadata"` + BucketObjects map[string]struct{} `json:"objects"` } // ListObjectsResults container for list objects response diff --git a/pkg/donut/donut-v1.go b/pkg/donut/donut-v1.go index 96a2d9dd7..1aa052279 100644 --- a/pkg/donut/donut-v1.go +++ b/pkg/donut/donut-v1.go @@ -153,7 +153,7 @@ func (donut API) putObject(bucket, object, expectedMD5Sum string, reader io.Read if err != nil { return ObjectMetadata{}, iodine.New(err, errParams) } - bucketMeta.Buckets[bucket].BucketObjects[object] = 1 + bucketMeta.Buckets[bucket].BucketObjects[object] = struct{}{} if err := donut.setDonutBucketMetadata(bucketMeta); err != nil { return ObjectMetadata{}, iodine.New(err, errParams) } @@ -229,7 +229,7 @@ func (donut API) getBucketMetadataWriters() ([]io.WriteCloser, error) { return writers, nil } -// getBucketMetadataReaders - +// getBucketMetadataReaders - readers are returned in map rather than slice func (donut API) getBucketMetadataReaders() (map[int]io.ReadCloser, error) { readers := make(map[int]io.ReadCloser) for _, node := range donut.nodes { diff --git a/pkg/donut/heal.go b/pkg/donut/heal.go index 30eae75a8..64d194afd 100644 --- a/pkg/donut/heal.go +++ b/pkg/donut/heal.go @@ -1,25 +1,32 @@ package donut -import ( - "fmt" +import "github.com/minio/minio/pkg/iodine" - "github.com/minio/minio/pkg/iodine" -) +type missingDisk struct { + nodeNumber int + sliceNumber int + bucketName string +} // Heal heal an existing donut func (donut API) Heal() error { - missingDisks := make(map[int]struct{}) + var missingDisks []missingDisk + nodeNumber := 0 for _, node := range donut.nodes { disks, err := node.ListDisks() if err != nil { return iodine.New(err, nil) } for i, disk := range disks { - dirs, err := disk.ListDir(donut.config.DonutName) - if err != nil { - missingDisks[i] = struct{}{} + _, err := disk.ListDir(donut.config.DonutName) + if err == nil { + continue + } + missingDisk := missingDisk{ + nodeNumber: nodeNumber, + sliceNumber: i, } - fmt.Println(dirs) + missingDisks = append(missingDisks, missingDisk) } } return nil diff --git a/pkg/donut/interfaces.go b/pkg/donut/interfaces.go index 4a8d0a88d..3470f8f2d 100644 --- a/pkg/donut/interfaces.go +++ b/pkg/donut/interfaces.go @@ -65,7 +65,4 @@ type Management interface { AttachNode(hostname string, disks []string) error DetachNode(hostname string) error - - SaveConfig() error - LoadConfig() error } diff --git a/pkg/donut/management.go b/pkg/donut/management.go index a03721657..31a35e516 100644 --- a/pkg/donut/management.go +++ b/pkg/donut/management.go @@ -17,9 +17,6 @@ package donut import ( - "encoding/json" - "path/filepath" - "github.com/minio/minio/pkg/donut/disk" "github.com/minio/minio/pkg/iodine" ) @@ -71,33 +68,3 @@ func (donut API) DetachNode(hostname string) error { delete(donut.nodes, hostname) return nil } - -// SaveConfig - save donut configuration -func (donut API) SaveConfig() error { - nodeDiskMap := make(map[string][]string) - for hostname, node := range donut.nodes { - disks, err := node.ListDisks() - if err != nil { - return iodine.New(err, nil) - } - for order, disk := range disks { - donutConfigPath := filepath.Join(donut.config.DonutName, donutConfig) - donutConfigWriter, err := disk.CreateFile(donutConfigPath) - defer donutConfigWriter.Close() - if err != nil { - return iodine.New(err, nil) - } - nodeDiskMap[hostname][order] = disk.GetPath() - jenc := json.NewEncoder(donutConfigWriter) - if err := jenc.Encode(nodeDiskMap); err != nil { - return iodine.New(err, nil) - } - } - } - return nil -} - -// LoadConfig - load configuration -func (donut API) LoadConfig() error { - return iodine.New(NotImplemented{Function: "LoadConfig"}, nil) -}