Merge pull request #191 from harshavardhana/pr_out_add_proper_entries_for_gobheader_data_copy

master
Harshavardhana 10 years ago
commit 3d70b96ae9
  1. 1
      pkg/storage/donut/v1/.gitignore
  2. 115
      pkg/storage/donut/v1/donut.go
  3. 63
      pkg/storage/donut/v1/donut_gen.go
  4. 3
      pkg/storage/donut/v1/donut_test.go

@ -17,9 +17,14 @@
package v1 package v1
import ( import (
"bytes"
"encoding/binary" "encoding/binary"
"encoding/gob"
"errors" "errors"
"io" "io"
"sync"
"github.com/minio-io/minio/pkg/storage/erasure"
) )
/* /*
@ -55,7 +60,7 @@ type DonutFormat struct {
VersionReserved uint16 VersionReserved uint16
Reserved uint64 Reserved uint64
GobHeaderLen uint32 GobHeaderLen uint32
GobHeader GobHeader GobHeader []byte
BlockData uint32 // Magic="DATA"=1096040772 BlockData uint32 // Magic="DATA"=1096040772
Data io.Reader Data io.Reader
BlockLen uint64 BlockLen uint64
@ -68,36 +73,61 @@ type DonutFooter struct {
} }
type Donut struct { type Donut struct {
file io.Writer file io.Writer
// mutex mutex *sync.RWMutex
} }
type GobHeader struct{} type GobHeader struct {
Blocks []EncodedChunk
Md5sum []byte
EncoderParams erasure.EncoderParams
}
func (donut *Donut) Write(gobHeader GobHeader, object io.Reader) error { type EncodedChunk struct {
// TODO mutex Crc uint32
// Create bytes buffer representing the new object Length int
donutFormat := DonutFormat{ Offset int
BlockStart: MagicMINI, }
VersionMajor: 1,
VersionMinor: 0, func New(file io.Writer) *Donut {
VersionPatch: 0, donut := Donut{}
VersionReserved: 0, donut.mutex = new(sync.RWMutex)
Reserved: 0, donut.file = file
GobHeaderLen: 0, return &donut
GobHeader: gobHeader, }
BlockData: MagicDATA,
Data: object, func (donut *Donut) WriteGob(gobHeader GobHeader) (bytes.Buffer, error) {
BlockLen: 0, var gobBuffer bytes.Buffer
BlockEnd: MagicINIM, encoder := gob.NewEncoder(&gobBuffer)
err := encoder.Encode(gobHeader)
if err != nil {
return bytes.Buffer{}, err
} }
if err := donut.WriteFormat(donut.file, donutFormat); err != nil { return gobBuffer, nil
}
func (donut *Donut) WriteEnd(target io.Writer, donutFormat DonutFormat) error {
if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockLen); err != nil {
return err return err
} }
if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockEnd); err != nil {
return err
}
return nil
}
func (donut *Donut) WriteData(target io.Writer, donutFormat DonutFormat) error {
var b bytes.Buffer
if count, err := io.Copy(&b, donutFormat.Data); uint64(count) != donutFormat.BlockLen || err != nil {
if err == nil {
return binary.Write(target, binary.LittleEndian, b.Bytes())
}
return errors.New("Copy failed, count incorrect.")
}
return nil return nil
} }
func (donut *Donut) WriteFormat(target io.Writer, donutFormat DonutFormat) error { func (donut *Donut) WriteBegin(target io.Writer, donutFormat DonutFormat) error {
if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockStart); err != nil { if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockStart); err != nil {
return err return err
} }
@ -125,16 +155,43 @@ func (donut *Donut) WriteFormat(target io.Writer, donutFormat DonutFormat) error
if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockData); err != nil { if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockData); err != nil {
return err return err
} }
if count, err := io.Copy(target, donutFormat.Data); uint64(count) != donutFormat.BlockLen || err != nil { return nil
if err == nil { }
return err
} func (donut *Donut) Write(gobHeader GobHeader, object io.Reader) error {
return errors.New("Copy failed, count incorrect.") donut.mutex.Lock()
defer donut.mutex.Unlock()
gobBytes, err := donut.WriteGob(gobHeader)
if err != nil {
return err
} }
if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockLen); err != nil {
// Create bytes buffer representing the new object
donutFormat := DonutFormat{
BlockStart: MagicMINI,
VersionMajor: 1,
VersionMinor: 0,
VersionPatch: 0,
VersionReserved: 0,
Reserved: 0,
GobHeaderLen: uint32(gobBytes.Len()),
GobHeader: gobBytes.Bytes(),
BlockData: MagicDATA,
Data: object,
BlockLen: 0,
BlockEnd: MagicINIM,
}
if err := donut.WriteBegin(donut.file, donutFormat); err != nil {
return err return err
} }
if err := binary.Write(target, binary.LittleEndian, donutFormat.BlockEnd); err != nil {
if err := donut.WriteData(donut.file, donutFormat); err != nil {
return err
}
if err := donut.WriteEnd(donut.file, donutFormat); err != nil {
return err return err
} }

@ -0,0 +1,63 @@
// +build ignore
package main
import (
"bytes"
"fmt"
"os"
"reflect"
"github.com/minio-io/minio/pkg/storage/donut/v1"
)
func main() {
fmt.Println("--start")
file, err := os.OpenFile("newfile", os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
panic(err)
}
donut := v1.New(file)
gobHeader := v1.GobHeader{}
data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
dataBuffer := bytes.NewBuffer(data)
err = donut.Write(gobHeader, dataBuffer)
if err != nil {
panic(err)
}
file.Close()
fmt.Println("--closed")
fmt.Println("--verify")
stat, _ := os.Stat("newfile")
fileSize := stat.Size()
rfile, _ := os.OpenFile("newfile", os.O_RDONLY, 0666)
blockStart := make([]byte, 4)
blockStartCheck := []byte{'M', 'I', 'N', 'I'}
_, err = rfile.Read(blockStart)
if err != nil {
panic(err)
}
blockEnd := make([]byte, 4)
start := fileSize - 4
blockEndCheck := []byte{'I', 'N', 'I', 'M'}
rfile.ReadAt(blockEnd, start)
rfile.Close()
if !reflect.DeepEqual(blockStart, blockStartCheck) {
panic("Corrupted donut file")
}
if !reflect.DeepEqual(blockEnd, blockEndCheck) {
panic("Corrupted donut file")
}
fmt.Println("--verified")
fmt.Println("--end")
}

@ -32,11 +32,10 @@ func (s *MySuite) TestAPISuite(c *C) {
var b bytes.Buffer var b bytes.Buffer
var o bytes.Buffer var o bytes.Buffer
donut := Donut{&b} donut := New(&b)
gobheader := GobHeader{} gobheader := GobHeader{}
err := donut.Write(gobheader, &o) err := donut.Write(gobheader, &o)
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Log(b.Bytes())
blockStart := make([]byte, 4) blockStart := make([]byte, 4)
blockEnd := make([]byte, 4) blockEnd := make([]byte, 4)

Loading…
Cancel
Save