From de21126f7e0565105b467134da16d01465e0c7ac Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 2 Jun 2016 01:49:46 -0700 Subject: [PATCH] XL: Re-align the code again. --- erasure-createfile.go | 53 ++++++++------- erasure-readfile.go | 154 ++++++++++++++++++++++-------------------- fs-v1.go | 17 +---- main.go | 4 +- object-common.go | 11 +++ server-main.go | 7 +- xl-v1.go | 12 ---- 7 files changed, 126 insertions(+), 132 deletions(-) diff --git a/erasure-createfile.go b/erasure-createfile.go index 46aa99134..d2622a75c 100644 --- a/erasure-createfile.go +++ b/erasure-createfile.go @@ -25,32 +25,9 @@ import ( "github.com/klauspost/reedsolomon" ) -// encodeData - encodes incoming data buffer into -// dataBlocks+parityBlocks returns a 2 dimensional byte array. -func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) { - rs, err := reedsolomon.New(dataBlocks, parityBlocks) - if err != nil { - return nil, err - } - // Split the input buffer into data and parity blocks. - var blocks [][]byte - blocks, err = rs.Split(dataBuffer) - if err != nil { - return nil, err - } - - // Encode parity blocks using data blocks. - err = rs.Encode(blocks) - if err != nil { - return nil, err - } - - // Return encoded blocks. - return blocks, nil -} - -// erasureCreateFile - take a data stream, reads until io.EOF erasure -// code and writes to all the disks. +// erasureCreateFile - writes an entire stream by erasure coding to +// all the disks, writes also calculate individual block's checksum +// for future bit-rot protection. func erasureCreateFile(disks []StorageAPI, volume string, path string, partName string, data io.Reader, eInfos []erasureInfo) (newEInfos []erasureInfo, err error) { // Allocated blockSized buffer for reading. buf := make([]byte, blockSizeV1) @@ -104,6 +81,30 @@ func erasureCreateFile(disks []StorageAPI, volume string, path string, partName return newEInfos, nil } +// encodeData - encodes incoming data buffer into +// dataBlocks+parityBlocks returns a 2 dimensional byte array. +func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) { + rs, err := reedsolomon.New(dataBlocks, parityBlocks) + if err != nil { + return nil, err + } + // Split the input buffer into data and parity blocks. + var blocks [][]byte + blocks, err = rs.Split(dataBuffer) + if err != nil { + return nil, err + } + + // Encode parity blocks using data blocks. + err = rs.Encode(blocks) + if err != nil { + return nil, err + } + + // Return encoded blocks. + return blocks, nil +} + // appendFile - append data buffer at path. func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, distribution []int, hashWriters []hash.Hash) (err error) { var wg = &sync.WaitGroup{} diff --git a/erasure-readfile.go b/erasure-readfile.go index 27fed9eaf..fe573071f 100644 --- a/erasure-readfile.go +++ b/erasure-readfile.go @@ -23,81 +23,11 @@ import ( "github.com/klauspost/reedsolomon" ) -// PartObjectChecksum - returns the checksum for the part name from the checksum slice. -func (e erasureInfo) PartObjectChecksum(partName string) checkSumInfo { - for _, checksum := range e.Checksum { - if checksum.Name == partName { - return checksum - } - } - return checkSumInfo{} -} - -// xlMetaPartBlockChecksums - get block checksums for a given part. -func metaPartBlockChecksums(disks []StorageAPI, eInfos []erasureInfo, partName string) (blockCheckSums []checkSumInfo) { - for index := range disks { - // Save the read checksums for a given part. - blockCheckSums = append(blockCheckSums, eInfos[index].PartObjectChecksum(partName)) - } - return blockCheckSums -} - -// Takes block index and block distribution to get the disk index. -func toDiskIndex(blockIdx int, distribution []int) (diskIndex int) { - diskIndex = -1 - // Find out the right disk index for the input block index. - for index, blockIndex := range distribution { - if blockIndex == blockIdx { - diskIndex = index - } - } - return diskIndex -} - -// isValidBlock - calculates the checksum hash for the block and -// validates if its correct returns true for valid cases, false otherwise. -func isValidBlock(disks []StorageAPI, volume, path string, diskIndex int, blockCheckSums []checkSumInfo) bool { - // Unknown block index requested, treat it as error. - if diskIndex == -1 { - return false - } - // Disk is not present, treat entire block to be non existent. - if disks[diskIndex] == nil { - return false - } - // Read everything for a given block and calculate hash. - hashWriter := newHash(blockCheckSums[diskIndex].Algorithm) - hashBytes, err := hashSum(disks[diskIndex], volume, path, hashWriter) - if err != nil { - return false - } - return hex.EncodeToString(hashBytes) == blockCheckSums[diskIndex].Hash -} - -// decodeData - decode encoded blocks. -func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error { - rs, err := reedsolomon.New(dataBlocks, parityBlocks) - if err != nil { - return err - } - err = rs.Reconstruct(enBlocks) - if err != nil { - return err - } - // Verify reconstructed blocks (parity). - ok, err := rs.Verify(enBlocks) - if err != nil { - return err - } - if !ok { - // Blocks cannot be reconstructed, corrupted data. - err = errors.New("Verification failed after reconstruction, data likely corrupted.") - return err - } - return nil -} - -// ReadFile - decoded erasure coded file. +// erasureReadFile - read an entire erasure coded file at into a byte +// array. Erasure coded parts are often few mega bytes in size and it +// is convenient to return them as byte slice. This function also +// supports bit-rot detection by verifying checksum of individual +// block's checksum. func erasureReadFile(disks []StorageAPI, volume string, path string, partName string, size int64, eInfos []erasureInfo) ([]byte, error) { // Return data buffer. var buffer []byte @@ -195,3 +125,77 @@ func erasureReadFile(disks []StorageAPI, volume string, path string, partName st } return buffer, nil } + +// PartObjectChecksum - returns the checksum for the part name from the checksum slice. +func (e erasureInfo) PartObjectChecksum(partName string) checkSumInfo { + for _, checksum := range e.Checksum { + if checksum.Name == partName { + return checksum + } + } + return checkSumInfo{} +} + +// xlMetaPartBlockChecksums - get block checksums for a given part. +func metaPartBlockChecksums(disks []StorageAPI, eInfos []erasureInfo, partName string) (blockCheckSums []checkSumInfo) { + for index := range disks { + // Save the read checksums for a given part. + blockCheckSums = append(blockCheckSums, eInfos[index].PartObjectChecksum(partName)) + } + return blockCheckSums +} + +// Takes block index and block distribution to get the disk index. +func toDiskIndex(blockIdx int, distribution []int) (diskIndex int) { + diskIndex = -1 + // Find out the right disk index for the input block index. + for index, blockIndex := range distribution { + if blockIndex == blockIdx { + diskIndex = index + } + } + return diskIndex +} + +// isValidBlock - calculates the checksum hash for the block and +// validates if its correct returns true for valid cases, false otherwise. +func isValidBlock(disks []StorageAPI, volume, path string, diskIndex int, blockCheckSums []checkSumInfo) bool { + // Unknown block index requested, treat it as error. + if diskIndex == -1 { + return false + } + // Disk is not present, treat entire block to be non existent. + if disks[diskIndex] == nil { + return false + } + // Read everything for a given block and calculate hash. + hashWriter := newHash(blockCheckSums[diskIndex].Algorithm) + hashBytes, err := hashSum(disks[diskIndex], volume, path, hashWriter) + if err != nil { + return false + } + return hex.EncodeToString(hashBytes) == blockCheckSums[diskIndex].Hash +} + +// decodeData - decode encoded blocks. +func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error { + rs, err := reedsolomon.New(dataBlocks, parityBlocks) + if err != nil { + return err + } + err = rs.Reconstruct(enBlocks) + if err != nil { + return err + } + // Verify reconstructed blocks (parity). + ok, err := rs.Verify(enBlocks) + if err != nil { + return err + } + if !ok { + // Blocks cannot be reconstructed, corrupted data. + err = errors.New("Verification failed after reconstruction, data likely corrupted.") + return err + } + return nil +} diff --git a/fs-v1.go b/fs-v1.go index 87f383d58..4a9f8674d 100644 --- a/fs-v1.go +++ b/fs-v1.go @@ -41,20 +41,9 @@ type fsObjects struct { // newFSObjects - initialize new fs object layer. func newFSObjects(disk string) (ObjectLayer, error) { - var storage StorageAPI - var err error - if !strings.ContainsRune(disk, ':') || filepath.VolumeName(disk) != "" { - // Initialize filesystem storage API. - storage, err = newPosix(disk) - if err != nil { - return nil, err - } - } else { - // Initialize rpc client storage API. - storage, err = newRPCClient(disk) - if err != nil { - return nil, err - } + storage, err := newStorageAPI(disk) + if err != nil { + return nil, err } // Runs house keeping code, like creating minioMetaBucket, cleaning up tmp files etc. diff --git a/main.go b/main.go index 56e61245e..836895a10 100644 --- a/main.go +++ b/main.go @@ -118,8 +118,8 @@ func registerApp() *cli.App { app := cli.NewApp() app.Name = "Minio" app.Author = "Minio.io" - app.Usage = "Distributed Object Storage Server for Micro Services." - app.Description = `Micro services environment provisions one Minio server per application instance. Scalability is achieved through large number of smaller personalized instances. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.` + app.Usage = "Cloud Storage Server." + app.Description = `Minio is an Amazon S3 compatible object storage server. Use it to store photos, videos, VMs, containers, log files, or any blob of data as objects.` app.Flags = append(minioFlags, globalFlags...) app.Commands = commands app.CustomAppHelpTemplate = minioHelpTemplate diff --git a/object-common.go b/object-common.go index 979c3182b..9bad88337 100644 --- a/object-common.go +++ b/object-common.go @@ -17,6 +17,7 @@ package main import ( + "path/filepath" "strings" "sync" ) @@ -43,6 +44,16 @@ func fsHouseKeeping(storageDisk StorageAPI) error { return nil } +// Depending on the disk type network or local, initialize storage API. +func newStorageAPI(disk string) (storage StorageAPI, err error) { + if !strings.ContainsRune(disk, ':') || filepath.VolumeName(disk) != "" { + // Initialize filesystem storage API. + return newPosix(disk) + } + // Initialize rpc client storage API. + return newRPCClient(disk) +} + // House keeping code needed for XL. func xlHouseKeeping(storageDisks []StorageAPI) error { // This happens for the first time, but keep this here since this diff --git a/server-main.go b/server-main.go index e3b297399..fd72e2119 100644 --- a/server-main.go +++ b/server-main.go @@ -34,7 +34,7 @@ import ( var serverCmd = cli.Command{ Name: "server", - Usage: "Start Minio cloud storage server.", + Usage: "Start object storage server.", Flags: []cli.Flag{ cli.StringFlag{ Name: "address", @@ -65,9 +65,10 @@ EXAMPLES: 3. Start minio server on Windows. $ minio {{.Name}} C:\MyShare - 4. Start minio server 8 disks to enable erasure coded layer with 4 data and 4 parity. + 4. Start minio server 12 disks to enable erasure coded layer with 6 data and 6 parity. $ minio {{.Name}} /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend \ - /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend + /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend \ + /mnt/export10/backend /mnt/export11/backend /mnt/export12/backend `, } diff --git a/xl-v1.go b/xl-v1.go index 1d8005c9f..4287c8001 100644 --- a/xl-v1.go +++ b/xl-v1.go @@ -19,9 +19,7 @@ package main import ( "errors" "fmt" - "path/filepath" "sort" - "strings" "sync" "github.com/minio/minio/pkg/disk" @@ -92,16 +90,6 @@ func checkSufficientDisks(disks []string) error { return nil } -// Depending on the disk type network or local, initialize storage API. -func newStorageAPI(disk string) (storage StorageAPI, err error) { - if !strings.ContainsRune(disk, ':') || filepath.VolumeName(disk) != "" { - // Initialize filesystem storage API. - return newPosix(disk) - } - // Initialize rpc client storage API. - return newRPCClient(disk) -} - // newXLObjects - initialize new xl object layer. func newXLObjects(disks []string) (ObjectLayer, error) { // Validate if input disks are sufficient.