diff --git a/erasure-readfile.go b/erasure-readfile.go index 7f9a0389c..91f494db8 100644 --- a/erasure-readfile.go +++ b/erasure-readfile.go @@ -189,6 +189,7 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s } } + var outSize, outOffset int64 // enBlocks data can have 0-padding hence we need to figure the exact number // of bytes we want to read from enBlocks. blockSize := eInfo.BlockSize @@ -196,26 +197,24 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s // For the last block, the block size can be less than BlockSize. blockSize = totalLength % eInfo.BlockSize } - data, err := getDataBlocks(enBlocks, eInfo.DataBlocks, int(blockSize)) - if err != nil { - return bytesWritten, err - } // If this is start block, skip unwanted bytes. if block == startBlock { - data = data[bytesToSkip:] + outOffset = bytesToSkip } - if len(data) > int(length-bytesWritten) { + // Total data to be read. + outSize = blockSize + if length-bytesWritten < blockSize { // We should not send more data than what was requested. - data = data[:length-bytesWritten] + outSize = length - bytesWritten } - - _, err = writer.Write(data) + // Write data blocks. + n, err := writeDataBlocks(writer, enBlocks, eInfo.DataBlocks, outOffset, outSize) if err != nil { return bytesWritten, err } - bytesWritten += int64(len(data)) + bytesWritten += n } return bytesWritten, nil diff --git a/erasure-utils.go b/erasure-utils.go index 521bc7b21..9daf39325 100644 --- a/erasure-utils.go +++ b/erasure-utils.go @@ -17,6 +17,7 @@ package main import ( + "bytes" "crypto/sha512" "hash" "io" @@ -62,30 +63,74 @@ func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, er return writer.Sum(nil), nil } -// getDataBlocks - fetches the data block only part of the input encoded blocks. -func getDataBlocks(enBlocks [][]byte, dataBlocks int, curBlockSize int) (data []byte, err error) { - if len(enBlocks) < dataBlocks { - return nil, reedsolomon.ErrTooFewShards - } +// getDataBlockLen - get length of data blocks from encoded blocks. +func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int { size := 0 - blocks := enBlocks[:dataBlocks] - for _, block := range blocks { + // Figure out the data block length. + for _, block := range enBlocks[:dataBlocks] { size += len(block) } - if size < curBlockSize { - return nil, reedsolomon.ErrShortData + return size +} + +// Writes all the data blocks from encoded blocks until requested +// outSize length. Provides a way to skip bytes until the offset. +func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, outOffset int64, outSize int64) (int64, error) { + // Do we have enough blocks? + if len(enBlocks) < dataBlocks { + return 0, reedsolomon.ErrTooFewShards } - write := curBlockSize - for _, block := range blocks { - if write < len(block) { - data = append(data, block[:write]...) - return data, nil + // Do we have enough data? + if int64(getDataBlockLen(enBlocks, dataBlocks)) < outSize { + return 0, reedsolomon.ErrShortData + } + + // Counter to decrement total left to write. + write := outSize + + // Counter to increment total written. + totalWritten := int64(0) + + // Write all data blocks to dst. + for _, block := range enBlocks[:dataBlocks] { + // Skip blocks until we have reached our offset. + if outOffset >= int64(len(block)) { + // Decrement offset. + outOffset -= int64(len(block)) + continue + } else { + // Skip until offset. + block = block[outOffset:] + + // Reset the offset for next iteration to read everything + // from subsequent blocks. + outOffset = 0 + } + // We have written all the blocks, write the last remaining block. + if write < int64(len(block)) { + n, err := io.Copy(dst, bytes.NewReader(block[:write])) + if err != nil { + return 0, err + } + totalWritten += n + break + } + // Copy the block. + n, err := io.Copy(dst, bytes.NewReader(block)) + if err != nil { + return 0, err } - data = append(data, block...) - write -= len(block) + + // Decrement output size. + write -= n + + // Increment written. + totalWritten += n } - return data, nil + + // Success. + return totalWritten, nil } // getBlockInfo - find start/end block and bytes to skip for given offset, length and block size. diff --git a/server_xl_test.go b/server_xl_test.go index 4814cdc47..6e6be7de3 100644 --- a/server_xl_test.go +++ b/server_xl_test.go @@ -840,15 +840,17 @@ func (s *MyAPIXLSuite) TestPartialContent(c *C) { c.Assert(response.StatusCode, Equals, http.StatusOK) // Prepare request - var table = []struct { + var testCases = []struct { byteRange string expectedString string }{ - {"6-7", "Wo"}, + {"4-7", "o Wo"}, + {"1-", "ello World"}, {"6-", "World"}, + {"-2", "ld"}, {"-7", "o World"}, } - for _, t := range table { + for _, t := range testCases { request, err = newTestRequest("GET", s.testServer.Server.URL+"/partial-content/bar", 0, nil, s.testServer.AccessKey, s.testServer.SecretKey) c.Assert(err, IsNil)