oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #128 from harshavardhana/pr_out_implement_commands

Browse Source
master
Harshavardhana 10 years ago
parent e3af681cc1 3ec5c2fb8f
commit 1f8806b44f
16 changed files with 312 additions and 479 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      Makefile
  2. 112
      cmd/erasure-demo/decode.go
  3. 83
      cmd/erasure-demo/encode.go
  4. 41
      cmd/erasure-demo/get.go
  5. 218
      cmd/erasure-demo/main.go
  6. 95
      cmd/minio/minio.go
  7. 103
      pkgs/minio/common.go
  8. 37
      pkgs/minio/encode.go
  9. 14
      pkgs/minio/erasure.go
  10. 30
      pkgs/minio/get.go
  11. 1
      pkgs/minio/init.go
  12. 24
      pkgs/minio/list.go
  13. 26
      pkgs/minio/put.go
  14. 1
      pkgs/minio/verify.go
  15. 2
      pkgs/storage/appendstorage/append_storage.go
  16. 2
      pkgs/storage/encodedstorage/encoded_storage.go

2
Makefile
Unescape View File

@ -44,7 +44,7 @@ cover: build-erasure build-signify build-split build-crc32c build-cpu build-sha1
@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway
install: build-erasure
@godep go install github.com/minio-io/minio/cmd/erasure-demo && echo "Installed erasure-demo into ${GOPATH}/bin"
@godep go install github.com/minio-io/minio/cmd/minio && echo "Installed minio into ${GOPATH}/bin"
save: restore
@godep save ./...

112
cmd/erasure-demo/decode.go
Unescape View File

@ -1,112 +0,0 @@
package main
import (
"io/ioutil"
"log"
"os"
"strconv"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/checksum/crc32c"
"github.com/minio-io/minio/pkgs/erasure"
)
func decode(c *cli.Context) {
// check if minio-encode called without parameters
if len(c.Args()) != 1 {
cli.ShowCommandHelp(c, "decode")
return
}
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
k := config.k
m := config.m
// check if output file exists, fail if so
if _, err := os.Stat(config.output); !os.IsNotExist(err) {
log.Fatal("Output file exists")
}
// get list of files
var inputFiles []string
if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) {
err = nil
chunkCount := 0
for !os.IsNotExist(err) {
_, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length")
chunkCount += 1
}
chunkCount = chunkCount - 1
inputFiles = make([]string, chunkCount)
for i := 0; i < chunkCount; i++ {
inputFiles[i] = config.input + "." + strconv.Itoa(i)
}
} else {
inputFiles = []string{config.input}
}
// open file to write
outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600)
defer outputFile.Close()
if err != nil {
log.Fatal(err)
}
for _, inputFile := range inputFiles {
// get chunks
chunks := make([][]byte, k+m)
// get checksum
cksums := make([][]byte, k+m)
for i := 0; i < k+m; i++ {
chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i))
cksums[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i) + ".cksum")
}
for i := 0; i < k+m; i++ {
crcChunk, err := crc32c.Crc32c(chunks[i])
if err != nil {
chunks[i] = nil
continue
}
crcChunkStr := strconv.Itoa(int(crcChunk))
if string(cksums[i]) != crcChunkStr {
chunks[i] = nil
}
}
// get length
lengthBytes, err := ioutil.ReadFile(inputFile + ".length")
if err != nil {
log.Fatal(err)
}
lengthString := string(lengthBytes)
length, err := strconv.Atoi(lengthString)
if err != nil {
log.Fatal(err)
}
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY)
// Get decoder
encoder := erasure.NewEncoder(erasureParameters)
// decode data
decodedData, err := encoder.Decode(chunks, length)
if err != nil {
log.Fatal(err)
}
// append decoded data
length, err = outputFile.Write(decodedData)
if err != nil {
log.Fatal(err)
}
}
}

83
cmd/erasure-demo/encode.go
Unescape View File

@ -1,83 +0,0 @@
package main
import (
"bytes"
"io/ioutil"
"log"
"os"
"strconv"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/checksum/crc32c"
"github.com/minio-io/minio/pkgs/erasure"
"github.com/minio-io/minio/pkgs/split"
)
func encode(c *cli.Context) {
// check if minio-encode called without parameters
if len(c.Args()) != 1 {
cli.ShowCommandHelp(c, "encode")
return
}
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
// get file
inputFile, err := os.Open(config.input)
defer inputFile.Close()
if err != nil {
log.Fatal(err)
}
// read file
input, err := ioutil.ReadAll(inputFile)
if err != nil {
log.Fatal(err)
}
// set up encoder
erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY)
// Init new encoder
encoder := erasure.NewEncoder(erasureParameters)
// encode data
if config.blockSize == 0 {
encodedData, length := encoder.Encode(input)
for key, data := range encodedData {
crc, err := crc32c.Crc32c(data)
if err != nil {
log.Fatal(err)
}
crcString := strconv.Itoa(int(crc))
ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600)
ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600)
}
} else {
chunkCount := 0
splitChannel := make(chan split.SplitMessage)
inputReader := bytes.NewReader(input)
go split.SplitStream(inputReader, config.blockSize, splitChannel)
for chunk := range splitChannel {
if chunk.Err != nil {
log.Fatal(chunk.Err)
}
encodedData, length := encoder.Encode(chunk.Data)
for key, data := range encodedData {
crc, err := crc32c.Crc32c(data)
if err != nil {
log.Fatal(err)
}
crcString := strconv.Itoa(int(crc))
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600)
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600)
}
ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600)
chunkCount += 1
}
}
}

41
cmd/erasure-demo/get.go
Unescape View File

@ -1,41 +0,0 @@
package main
import (
"io"
"log"
"os"
"github.com/codegangsta/cli"
)
func get(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
var objectReader io.Reader
objectName := c.Args().Get(0)
switch config.storageDriver {
case "erasure":
{
if len(objectName) == 0 {
if objectReader, err = erasureGetList(config, ""); err != nil {
log.Fatal(err)
}
} else {
if objectReader, err = erasureGet(config, objectName); err != nil {
log.Fatal(err)
}
}
}
default:
{
log.Fatal("Unknown driver")
}
}
if objectReader == nil {
log.Fatal("Object not found")
}
io.Copy(os.Stdout, objectReader)
}

218
cmd/erasure-demo/main.go
Unescape View File

@ -1,218 +0,0 @@
package main
import (
"errors"
"log"
"os"
"os/user"
"path"
"strconv"
"strings"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/strbyteconv"
)
func main() {
app := cli.NewApp()
app.Name = "erasure-demo"
app.Usage = "erasure encode a byte stream"
app.Commands = []cli.Command{
{
Name: "encode",
Usage: "erasure encode a byte stream",
Action: encode,
Flags: []cli.Flag{
cli.StringFlag{
Name: "output,o",
Value: "",
Usage: "Output file",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "decode",
Usage: "erasure decode a byte stream",
Action: decode,
Flags: []cli.Flag{
cli.StringFlag{
Name: "output,o",
Value: "",
Usage: "Output file",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
},
},
{
Name: "get",
Usage: "get an object",
Action: get,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: getMinioDir(),
Usage: "",
},
cli.StringFlag{
Name: "driver",
Value: "erasure",
Usage: "erasure",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "put",
Usage: "put an object",
Action: put,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: getMinioDir(),
Usage: "",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
cli.StringFlag{
Name: "driver",
Value: "erasure",
Usage: "erasure",
},
},
},
}
app.Run(os.Args)
}
// config representing cli input
type inputConfig struct {
input string
output string
k int
m int
blockSize uint64
rootDir string
storageDriver string
}
// parses input and returns an inputConfig with parsed input
func parseInput(c *cli.Context) (inputConfig, error) {
// get input path
inputFilePath := c.Args().Get(0)
// get output path
outputFilePath := inputFilePath
if c.String("output") != "" {
outputFilePath = c.String("output")
}
var k, m int
if c.String("protection-level") != "" {
protectionLevel := c.String("protection-level")
protectionLevelSplit := strings.Split(protectionLevel, ",")
if len(protectionLevelSplit) != 2 {
return inputConfig{}, errors.New("Malformed input for protection-level")
}
var err error
k, err = strconv.Atoi(protectionLevelSplit[0])
if err != nil {
return inputConfig{}, err
}
m, err = strconv.Atoi(protectionLevelSplit[1])
if err != nil {
return inputConfig{}, err
}
}
var blockSize uint64
blockSize = 0
if c.String("block-size") != "" {
if c.String("block-size") != "full" {
var err error
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
if err != nil {
return inputConfig{}, err
}
}
}
var rootDir string
if c.String("root") != "" {
rootDir = c.String("root")
}
var storageDriver string
if c.String("driver") != "" {
storageDriver = c.String("driver")
}
config := inputConfig{
input: inputFilePath,
output: outputFilePath,
k: k,
m: m,
blockSize: blockSize,
rootDir: rootDir,
storageDriver: storageDriver,
}
return config, nil
}
func getMinioDir() string {
user, err := user.Current()
if err != nil {
log.Fatal(err)
}
homePath := user.HomeDir
minioPath := path.Join(homePath, ".minio")
err = _initMinioDir(minioPath)
if err != nil {
log.Fatal(err)
}
return minioPath
}
func _initMinioDir(dirPath string) error {
_, err := os.Lstat(dirPath)
if err != nil {
log.Printf("%s not found, creating a new-one for the first time",
dirPath)
err = os.Mkdir(dirPath, 0700)
if err != nil {
return err
}
}
return nil
}

95
cmd/minio/minio.go
Unescape View File

@ -0,0 +1,95 @@
package main
import (
"os"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/minio"
)
func main() {
app := cli.NewApp()
app.Name = "minio"
app.Usage = "minio - object storage"
app.Commands = []cli.Command{
{
Name: "encode",
Usage: "erasure encode a byte stream",
Action: minio.Encode,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: minio.Getobjectdir(".minio/erasure"),
Usage: "",
},
cli.StringFlag{
Name: "staging",
Value: minio.Getobjectdir(".minio/staging"),
Usage: "",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "get",
Usage: "get an object",
Action: minio.Get,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: minio.Getobjectdir(".minio/erasure"),
Usage: "",
},
cli.StringFlag{
Name: "protection-level",
Value: "10,6",
Usage: "data,parity",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "put",
Usage: "put an object",
Action: minio.Put,
Flags: []cli.Flag{
cli.StringFlag{
Name: "staging",
Value: minio.Getobjectdir(".minio/staging"),
Usage: "",
},
cli.StringFlag{
Name: "block-size",
Value: "1M",
Usage: "Size of blocks. Examples: 1K, 1M, full",
},
},
},
{
Name: "list",
Usage: "list objects",
Action: minio.List,
Flags: []cli.Flag{
cli.StringFlag{
Name: "root",
Value: minio.Getobjectdir(".minio/erasure"),
Usage: "",
},
},
},
}
app.Run(os.Args)
}

103
pkgs/minio/common.go
Unescape View File

@ -0,0 +1,103 @@
package minio
import (
"errors"
"log"
"os"
"os/user"
"path"
"strconv"
"strings"
"github.com/codegangsta/cli"
"github.com/minio-io/minio/pkgs/strbyteconv"
)
// config representing cli input
type inputConfig struct {
k int
m int
blockSize uint64
rootDir string
stagingDir string
}
// parses input and returns an inputConfig with parsed input
func parseInput(c *cli.Context) (inputConfig, error) {
var k, m int
if c.String("protection-level") != "" {
protectionLevel := c.String("protection-level")
protectionLevelSplit := strings.Split(protectionLevel, ",")
if len(protectionLevelSplit) != 2 {
return inputConfig{}, errors.New("Malformed input for protection-level")
}
var err error
k, err = strconv.Atoi(protectionLevelSplit[0])
if err != nil {
return inputConfig{}, err
}
m, err = strconv.Atoi(protectionLevelSplit[1])
if err != nil {
return inputConfig{}, err
}
}
var blockSize uint64
blockSize = 0
if c.String("block-size") != "" {
if c.String("block-size") != "full" {
var err error
blockSize, err = strbyteconv.StringToBytes(c.String("block-size"))
if err != nil {
return inputConfig{}, err
}
}
}
var rootDir string
if c.String("root") != "" {
rootDir = c.String("root")
}
var stagingDir string
if c.String("staging") != "" {
stagingDir = c.String("staging")
}
config := inputConfig{
k: k,
m: m,
blockSize: blockSize,
rootDir: rootDir,
stagingDir: stagingDir,
}
return config, nil
}
func Getobjectdir(basename string) string {
user, err := user.Current()
if err != nil {
log.Fatal(err)
}
homePath := user.HomeDir
minioPath := path.Join(homePath, basename)
err = _initDir(minioPath)
if err != nil {
log.Fatal(err)
}
return minioPath
}
func _initDir(dirPath string) error {
_, err := os.Lstat(dirPath)
if err != nil {
log.Printf("%s not found, creating a new-one for the first time",
dirPath)
err = os.MkdirAll(dirPath, 0700)
if err != nil {
return err
}
}
return nil
}

37
pkgs/minio/encode.go
Unescape View File

@ -0,0 +1,37 @@
package minio
import (
"log"
"github.com/codegangsta/cli"
)
func Encode(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
var objectName string
switch len(c.Args()) {
case 1:
objectName = c.Args().Get(0)
default:
log.Fatal("Please specify a valid object name \n # erasure-demo encode [OBJECTNAME]")
}
// Get from staging area
stagingConfig := config
stagingConfig.k = 2
stagingConfig.m = 1
stagingConfig.rootDir = config.stagingDir
reader, err := erasureGet(stagingConfig, objectName)
if err != nil {
log.Fatal(err)
}
// Increase parity to user defined or default 10,6
err = erasurePut(config, objectName, reader)
if err != nil {
log.Fatal(err)
}
}

14
cmd/erasure-demo/erasure.go → pkgs/minio/erasure.go
Unescape View File

@ -1,11 +1,10 @@
package main
package minio
import (
"bytes"
"encoding/json"
"io"
"os"
"path"
"github.com/minio-io/minio/pkgs/storage"
es "github.com/minio-io/minio/pkgs/storage/encodedstorage"
@ -13,8 +12,7 @@ import (
func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) {
var objectStorage storage.ObjectStorage
rootDir := path.Join(config.rootDir, config.storageDriver)
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
objectStorage, err := es.NewStorage(config.rootDir, config.k, config.m, config.blockSize)
if err != nil {
return nil, err
}
@ -33,8 +31,7 @@ func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) {
func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {
var objectStorage storage.ObjectStorage
rootDir := path.Join(config.rootDir, config.storageDriver)
objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize)
objectStorage, err := es.NewStorage(config.rootDir, config.k, config.m, config.blockSize)
if err != nil {
return nil, err
}
@ -47,12 +44,11 @@ func erasureGet(config inputConfig, objectPath string) (io.Reader, error) {
func erasurePut(config inputConfig, objectPath string, reader io.Reader) error {
var err error
rootDir := path.Join(config.rootDir, config.storageDriver)
if err := os.MkdirAll(rootDir, 0700); err != nil {
if err := os.MkdirAll(config.rootDir, 0700); err != nil {
return err
}
var objectStorage storage.ObjectStorage
if objectStorage, err = es.NewStorage(rootDir, config.k, config.m, config.blockSize); err != nil {
if objectStorage, err = es.NewStorage(config.rootDir, config.k, config.m, config.blockSize); err != nil {
return err
}
if err = objectStorage.Put(objectPath, reader); err != nil {

30
pkgs/minio/get.go
Unescape View File

@ -0,0 +1,30 @@
package minio
import (
"io"
"log"
"os"
"github.com/codegangsta/cli"
)
func Get(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
var objectName string
var objectReader io.Reader
switch len(c.Args()) {
case 1:
objectName = c.Args().Get(0)
default:
log.Fatal("Please specify a valid object name \n # erasure-demo get [OBJECTNAME]")
}
if objectReader, err = erasureGet(config, objectName); err != nil {
log.Fatal(err)
}
io.Copy(os.Stdout, objectReader)
}

1
pkgs/minio/init.go
Unescape View File

@ -0,0 +1 @@
package minio

24
pkgs/minio/list.go
Unescape View File

@ -0,0 +1,24 @@
package minio
import (
"io"
"log"
"os"
"github.com/codegangsta/cli"
)
func List(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
}
config.k = 10
config.m = 6
reader, err := erasureGetList(config, "")
if err != nil {
log.Fatal(err)
}
io.Copy(os.Stdout, reader)
}

26
cmd/erasure-demo/put.go → pkgs/minio/put.go
Unescape View File

@ -1,4 +1,4 @@
package main
package minio
import (
"log"
@ -8,7 +8,7 @@ import (
"github.com/codegangsta/cli"
)
func put(c *cli.Context) {
func Put(c *cli.Context) {
config, err := parseInput(c)
if err != nil {
log.Fatal(err)
@ -25,19 +25,19 @@ func put(c *cli.Context) {
log.Fatal("Please specify a valid object name \n # erasure-demo put [OBJECTNAME] [FILENAME]")
}
inputFile, err := os.Open(filePath)
defer inputFile.Close()
if err != nil {
log.Fatal(err)
}
switch config.storageDriver {
case "erasure":
{
if err := erasurePut(config, objectName, inputFile); err != nil {
log.Fatal(err)
}
}
default:
{
log.Fatal("Unknown driver")
}
// Staging parity
stagingConfig := config
stagingConfig.k = 2
stagingConfig.m = 1
stagingConfig.rootDir = config.stagingDir
if err := erasurePut(stagingConfig, objectName, inputFile); err != nil {
log.Fatal(err)
}
}

1
pkgs/minio/verify.go
Unescape View File

@ -0,0 +1 @@
package minio

2
pkgs/storage/appendstorage/append_storage.go
Unescape View File

@ -80,7 +80,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) {
func (storage *appendStorage) Get(objectPath string) (io.Reader, error) {
header, ok := storage.objects[objectPath]
if ok == false {
return nil, nil
return nil, errors.New("Object not found")
}
offset := header.Offset

2
pkgs/storage/encodedstorage/encoded_storage.go
Unescape View File

@ -110,7 +110,7 @@ func NewStorage(rootDir string, k, m int, blockSize uint64) (storage.ObjectStora
func (eStorage *encodedStorage) Get(objectPath string) (io.Reader, error) {
entry, ok := eStorage.objects[objectPath]
if ok == false {
return nil, nil
return nil, errors.New("Object not found")
}
reader, writer := io.Pipe()
go eStorage.readObject(objectPath, entry, writer)

Write Preview
Loading…
Cancel
Save