Streaming bitrot verification support (#7004)
Krishna Srinivas
6 years ago
committed by
Harshavardhana
parent
94c52e3816
commit
98c950aacd
@ -0,0 +1,172 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2019 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/hex" |
||||
"hash" |
||||
"io" |
||||
|
||||
"github.com/minio/minio/cmd/logger" |
||||
) |
||||
|
||||
// Calculates bitrot in chunks and writes the hash into the stream.
|
||||
type streamingBitrotWriter struct { |
||||
iow *io.PipeWriter |
||||
h hash.Hash |
||||
shardSize int64 |
||||
canClose chan struct{} // Needed to avoid race explained in Close() call.
|
||||
|
||||
// Following two fields are used only to make sure that Write(p) is called such that
|
||||
// len(p) is always the block size except the last block, i.e prevent programmer errors.
|
||||
currentBlockIdx int |
||||
verifyTillIdx int |
||||
} |
||||
|
||||
func (b *streamingBitrotWriter) Write(p []byte) (int, error) { |
||||
if b.currentBlockIdx < b.verifyTillIdx && int64(len(p)) != b.shardSize { |
||||
// All blocks except last should be of the length b.shardSize
|
||||
logger.LogIf(context.Background(), errUnexpected) |
||||
return 0, errUnexpected |
||||
} |
||||
if len(p) == 0 { |
||||
return 0, nil |
||||
} |
||||
b.h.Reset() |
||||
b.h.Write(p) |
||||
hashBytes := b.h.Sum(nil) |
||||
n, err := b.iow.Write(hashBytes) |
||||
if n != len(hashBytes) { |
||||
logger.LogIf(context.Background(), err) |
||||
return 0, err |
||||
} |
||||
n, err = b.iow.Write(p) |
||||
b.currentBlockIdx++ |
||||
return n, err |
||||
} |
||||
|
||||
func (b *streamingBitrotWriter) Close() error { |
||||
err := b.iow.Close() |
||||
// Wait for all data to be written before returning else it causes race conditions.
|
||||
// Race condition is because of io.PipeWriter implementation. i.e consider the following
|
||||
// sequent of operations:
|
||||
// 1) pipe.Write()
|
||||
// 2) pipe.Close()
|
||||
// Now pipe.Close() can return before the data is read on the other end of the pipe and written to the disk
|
||||
// Hence an immediate Read() on the file can return incorrect data.
|
||||
<-b.canClose |
||||
return err |
||||
} |
||||
|
||||
// Returns streaming bitrot writer implementation.
|
||||
func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser { |
||||
r, w := io.Pipe() |
||||
h := algo.New() |
||||
bw := &streamingBitrotWriter{w, h, shardSize, make(chan struct{}), 0, int(length / shardSize)} |
||||
go func() { |
||||
bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
|
||||
totalFileSize := bitrotSumsTotalSize + length |
||||
err := disk.CreateFile(volume, filePath, totalFileSize, r) |
||||
if err != nil { |
||||
logger.LogIf(context.Background(), err) |
||||
r.CloseWithError(err) |
||||
} |
||||
close(bw.canClose) |
||||
}() |
||||
return bw |
||||
} |
||||
|
||||
// ReadAt() implementation which verifies the bitrot hash available as part of the stream.
|
||||
type streamingBitrotReader struct { |
||||
disk StorageAPI |
||||
rc io.ReadCloser |
||||
volume string |
||||
filePath string |
||||
tillOffset int64 |
||||
currOffset int64 |
||||
h hash.Hash |
||||
shardSize int64 |
||||
hashBytes []byte |
||||
} |
||||
|
||||
func (b *streamingBitrotReader) Close() error { |
||||
if b.rc == nil { |
||||
return nil |
||||
} |
||||
return b.rc.Close() |
||||
} |
||||
|
||||
func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) { |
||||
var err error |
||||
if offset%b.shardSize != 0 { |
||||
// Offset should always be aligned to b.shardSize
|
||||
logger.LogIf(context.Background(), errUnexpected) |
||||
return 0, errUnexpected |
||||
} |
||||
if b.rc == nil { |
||||
// For the first ReadAt() call we need to open the stream for reading.
|
||||
b.currOffset = offset |
||||
streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset |
||||
b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset) |
||||
if err != nil { |
||||
logger.LogIf(context.Background(), err) |
||||
return 0, err |
||||
} |
||||
} |
||||
if offset != b.currOffset { |
||||
logger.LogIf(context.Background(), errUnexpected) |
||||
return 0, errUnexpected |
||||
} |
||||
b.h.Reset() |
||||
_, err = io.ReadFull(b.rc, b.hashBytes) |
||||
if err != nil { |
||||
logger.LogIf(context.Background(), err) |
||||
return 0, err |
||||
} |
||||
_, err = io.ReadFull(b.rc, buf) |
||||
if err != nil { |
||||
logger.LogIf(context.Background(), err) |
||||
return 0, err |
||||
} |
||||
b.h.Write(buf) |
||||
|
||||
if bytes.Compare(b.h.Sum(nil), b.hashBytes) != 0 { |
||||
err = hashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))} |
||||
logger.LogIf(context.Background(), err) |
||||
return 0, err |
||||
} |
||||
b.currOffset += int64(len(buf)) |
||||
return len(buf), nil |
||||
} |
||||
|
||||
// Returns streaming bitrot reader implementation.
|
||||
func newStreamingBitrotReader(disk StorageAPI, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader { |
||||
h := algo.New() |
||||
return &streamingBitrotReader{ |
||||
disk, |
||||
nil, |
||||
volume, |
||||
filePath, |
||||
ceilFrac(tillOffset, shardSize)*int64(h.Size()) + tillOffset, |
||||
0, |
||||
h, |
||||
shardSize, |
||||
make([]byte, h.Size()), |
||||
} |
||||
} |
||||
@ -0,0 +1,109 @@ |
||||
/* |
||||
* Minio Cloud Storage, (C) 2019 Minio, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"context" |
||||
"hash" |
||||
"io" |
||||
|
||||
"github.com/minio/minio/cmd/logger" |
||||
) |
||||
|
||||
// Implementation to calculate bitrot for the whole file.
|
||||
type wholeBitrotWriter struct { |
||||
disk StorageAPI |
||||
volume string |
||||
filePath string |
||||
shardSize int64 // This is the shard size of the erasure logic
|
||||
hash.Hash // For bitrot hash
|
||||
|
||||
// Following two fields are used only to make sure that Write(p) is called such that
|
||||
// len(p) is always the block size except the last block and prevent programmer errors.
|
||||
currentBlockIdx int |
||||
lastBlockIdx int |
||||
} |
||||
|
||||
func (b *wholeBitrotWriter) Write(p []byte) (int, error) { |
||||
if b.currentBlockIdx < b.lastBlockIdx && int64(len(p)) != b.shardSize { |
||||
// All blocks except last should be of the length b.shardSize
|
||||
logger.LogIf(context.Background(), errUnexpected) |
||||
return 0, errUnexpected |
||||
} |
||||
err := b.disk.AppendFile(b.volume, b.filePath, p) |
||||
if err != nil { |
||||
logger.LogIf(context.Background(), err) |
||||
return 0, err |
||||
} |
||||
_, err = b.Hash.Write(p) |
||||
if err != nil { |
||||
logger.LogIf(context.Background(), err) |
||||
return 0, err |
||||
} |
||||
b.currentBlockIdx++ |
||||
return len(p), nil |
||||
} |
||||
|
||||
func (b *wholeBitrotWriter) Close() error { |
||||
return nil |
||||
} |
||||
|
||||
// Returns whole-file bitrot writer.
|
||||
func newWholeBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser { |
||||
return &wholeBitrotWriter{disk, volume, filePath, shardSize, algo.New(), 0, int(length / shardSize)} |
||||
} |
||||
|
||||
// Implementation to verify bitrot for the whole file.
|
||||
type wholeBitrotReader struct { |
||||
disk StorageAPI |
||||
volume string |
||||
filePath string |
||||
verifier *BitrotVerifier // Holds the bit-rot info
|
||||
tillOffset int64 // Affects the length of data requested in disk.ReadFile depending on Read()'s offset
|
||||
buf []byte // Holds bit-rot verified data
|
||||
} |
||||
|
||||
func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) { |
||||
if b.buf == nil { |
||||
b.buf = make([]byte, b.tillOffset-offset) |
||||
if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil { |
||||
ctx := context.Background() |
||||
logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String()) |
||||
logger.LogIf(ctx, err) |
||||
return 0, err |
||||
} |
||||
} |
||||
if len(b.buf) < len(buf) { |
||||
logger.LogIf(context.Background(), errLessData) |
||||
return 0, errLessData |
||||
} |
||||
n = copy(buf, b.buf) |
||||
b.buf = b.buf[n:] |
||||
return n, nil |
||||
} |
||||
|
||||
// Returns whole-file bitrot reader.
|
||||
func newWholeBitrotReader(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, tillOffset int64, sum []byte) *wholeBitrotReader { |
||||
return &wholeBitrotReader{ |
||||
disk: disk, |
||||
volume: volume, |
||||
filePath: filePath, |
||||
verifier: &BitrotVerifier{algo, sum}, |
||||
tillOffset: tillOffset, |
||||
buf: nil, |
||||
} |
||||
} |
||||
Loading…
Reference in new issue