Merge pull request #753 from harshavardhana/pr_out_wire_up_sha512_matching_inside_donut_along_with_md5sum

Wire up sha512 matching inside donut along with md5sum

main.go

@@ -107,7 +107,7 @@ func main() {
 	// set up app
 	app := cli.NewApp()
 	app.Name = "minio"
-	app.Version = Version
+	app.Version = getVersion()
 	app.Compiled = getVersion()
 	app.Author = "Minio.io"
 	app.Usage = "Minimalist Object Storage"

pkg/donut/bucket.go

@@ -78,7 +78,7 @@ func newBucket(bucketName, aclType, donutName string, nodes map[string]node) (bu
 	metadata.ACL = BucketACL(aclType)
 	metadata.Created = t
 	metadata.Metadata = make(map[string]string)
-	metadata.BucketObjects = make(map[string]interface{})
+	metadata.BucketObjects = make(map[string]struct{})
 
 	return b, metadata, nil
 }
@@ -455,10 +455,16 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta
 		writer.CloseWithError(iodine.New(err, nil))
 		return
 	}
+	expected512Sum, err := hex.DecodeString(objMetadata.SHA512Sum)
+	if err != nil {
+		writer.CloseWithError(iodine.New(err, nil))
+		return
+	}
 	hasher := md5.New()
-	mwriter := io.MultiWriter(writer, hasher)
+	sum512hasher := sha256.New()
-	switch len(readers) == 1 {
+	mwriter := io.MultiWriter(writer, hasher, sum512hasher)
-	case false:
+	switch len(readers) > 1 {
+	case true:
 		if objMetadata.ErasureTechnique == "" {
 			writer.CloseWithError(iodine.New(MissingErasureTechnique{}, nil))
 			return
@@ -482,7 +488,7 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta
 			}
 			totalLeft = totalLeft - int64(objMetadata.BlockSize)
 		}
-	case true:
+	case false:
 		_, err := io.Copy(writer, readers[0])
 		if err != nil {
 			writer.CloseWithError(iodine.New(err, nil))
@@ -494,6 +500,10 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta
 		writer.CloseWithError(iodine.New(ChecksumMismatch{}, nil))
 		return
 	}
+	if !bytes.Equal(expected512Sum, sum512hasher.Sum(nil)) {
+		writer.CloseWithError(iodine.New(ChecksumMismatch{}, nil))
+		return
+	}
 	writer.Close()
 	return
 }

pkg/donut/definitions.go

@@ -57,12 +57,12 @@ type AllBuckets struct {
 
 // BucketMetadata container for bucket level metadata
 type BucketMetadata struct {
-	Version       string                 `json:"version"`
+	Version       string              `json:"version"`
-	Name          string                 `json:"name"`
+	Name          string              `json:"name"`
-	ACL           BucketACL              `json:"acl"`
+	ACL           BucketACL           `json:"acl"`
-	Created       time.Time              `json:"created"`
+	Created       time.Time           `json:"created"`
-	Metadata      map[string]string      `json:"metadata"`
+	Metadata      map[string]string   `json:"metadata"`
-	BucketObjects map[string]interface{} `json:"objects"`
+	BucketObjects map[string]struct{} `json:"objects"`
 }
 
 // ListObjectsResults container for list objects response

pkg/donut/donut-v1.go

@@ -153,7 +153,7 @@ func (donut API) putObject(bucket, object, expectedMD5Sum string, reader io.Read
 	if err != nil {
 		return ObjectMetadata{}, iodine.New(err, errParams)
 	}
-	bucketMeta.Buckets[bucket].BucketObjects[object] = 1
+	bucketMeta.Buckets[bucket].BucketObjects[object] = struct{}{}
 	if err := donut.setDonutBucketMetadata(bucketMeta); err != nil {
 		return ObjectMetadata{}, iodine.New(err, errParams)
 	}
@@ -229,7 +229,7 @@ func (donut API) getBucketMetadataWriters() ([]io.WriteCloser, error) {
 	return writers, nil
 }
 
-// getBucketMetadataReaders -
+// getBucketMetadataReaders - readers are returned in map rather than slice
 func (donut API) getBucketMetadataReaders() (map[int]io.ReadCloser, error) {
 	readers := make(map[int]io.ReadCloser)
 	for _, node := range donut.nodes {

pkg/donut/heal.go

@@ -1,25 +1,32 @@
 package donut
 
-import (
+import "github.com/minio/minio/pkg/iodine"
-	"fmt"
 
-	"github.com/minio/minio/pkg/iodine"
+type missingDisk struct {
-)
+	nodeNumber  int
+	sliceNumber int
+	bucketName  string
+}
 
 // Heal heal an existing donut
 func (donut API) Heal() error {
-	missingDisks := make(map[int]struct{})
+	var missingDisks []missingDisk
+	nodeNumber := 0
 	for _, node := range donut.nodes {
 		disks, err := node.ListDisks()
 		if err != nil {
 			return iodine.New(err, nil)
 		}
 		for i, disk := range disks {
-			dirs, err := disk.ListDir(donut.config.DonutName)
+			_, err := disk.ListDir(donut.config.DonutName)
-			if err != nil {
+			if err == nil {
-				missingDisks[i] = struct{}{}
+				continue
+			}
+			missingDisk := missingDisk{
+				nodeNumber:  nodeNumber,
+				sliceNumber: i,
 			}
-			fmt.Println(dirs)
+			missingDisks = append(missingDisks, missingDisk)
 		}
 	}
 	return nil

pkg/donut/interfaces.go

@@ -65,7 +65,4 @@ type Management interface {
 
 	AttachNode(hostname string, disks []string) error
 	DetachNode(hostname string) error
-
-	SaveConfig() error
-	LoadConfig() error
 }

pkg/donut/management.go

@@ -17,9 +17,6 @@
 package donut
 
 import (
-	"encoding/json"
-	"path/filepath"
-
 	"github.com/minio/minio/pkg/donut/disk"
 	"github.com/minio/minio/pkg/iodine"
 )
@@ -71,33 +68,3 @@ func (donut API) DetachNode(hostname string) error {
 	delete(donut.nodes, hostname)
 	return nil
 }
-
-// SaveConfig - save donut configuration
-func (donut API) SaveConfig() error {
-	nodeDiskMap := make(map[string][]string)
-	for hostname, node := range donut.nodes {
-		disks, err := node.ListDisks()
-		if err != nil {
-			return iodine.New(err, nil)
-		}
-		for order, disk := range disks {
-			donutConfigPath := filepath.Join(donut.config.DonutName, donutConfig)
-			donutConfigWriter, err := disk.CreateFile(donutConfigPath)
-			defer donutConfigWriter.Close()
-			if err != nil {
-				return iodine.New(err, nil)
-			}
-			nodeDiskMap[hostname][order] = disk.GetPath()
-			jenc := json.NewEncoder(donutConfigWriter)
-			if err := jenc.Encode(nodeDiskMap); err != nil {
-				return iodine.New(err, nil)
-			}
-		}
-	}
-	return nil
-}
-
-// LoadConfig - load configuration
-func (donut API) LoadConfig() error {
-	return iodine.New(NotImplemented{Function: "LoadConfig"}, nil)
-}