diff --git a/Makefile b/Makefile index 9916e574d..ea162b95d 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ cover: build-erasure build-signify build-split build-crc32c build-cpu build-sha1 @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway install: build-erasure - @godep go install github.com/minio-io/minio/cmd/erasure-demo && echo "Installed erasure-demo into ${GOPATH}/bin" + @godep go install github.com/minio-io/minio/cmd/minio && echo "Installed minio into ${GOPATH}/bin" save: restore @godep save ./... diff --git a/cmd/erasure-demo/decode.go b/cmd/erasure-demo/decode.go deleted file mode 100644 index a5bf57677..000000000 --- a/cmd/erasure-demo/decode.go +++ /dev/null @@ -1,112 +0,0 @@ -package main - -import ( - "io/ioutil" - "log" - "os" - "strconv" - - "github.com/codegangsta/cli" - "github.com/minio-io/minio/pkgs/checksum/crc32c" - "github.com/minio-io/minio/pkgs/erasure" -) - -func decode(c *cli.Context) { - // check if minio-encode called without parameters - if len(c.Args()) != 1 { - cli.ShowCommandHelp(c, "decode") - return - } - - config, err := parseInput(c) - if err != nil { - log.Fatal(err) - } - - k := config.k - m := config.m - - // check if output file exists, fail if so - if _, err := os.Stat(config.output); !os.IsNotExist(err) { - log.Fatal("Output file exists") - } - - // get list of files - var inputFiles []string - if _, err := os.Stat(config.input + ".length"); os.IsNotExist(err) { - err = nil - chunkCount := 0 - for !os.IsNotExist(err) { - _, err = os.Stat(config.input + "." + strconv.Itoa(chunkCount) + ".length") - chunkCount += 1 - } - chunkCount = chunkCount - 1 - inputFiles = make([]string, chunkCount) - for i := 0; i < chunkCount; i++ { - inputFiles[i] = config.input + "." + strconv.Itoa(i) - } - } else { - inputFiles = []string{config.input} - } - - // open file to write - outputFile, err := os.OpenFile(config.output, os.O_CREATE|os.O_WRONLY, 0600) - defer outputFile.Close() - if err != nil { - log.Fatal(err) - } - - for _, inputFile := range inputFiles { - // get chunks - chunks := make([][]byte, k+m) - // get checksum - cksums := make([][]byte, k+m) - for i := 0; i < k+m; i++ { - chunks[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i)) - cksums[i], _ = ioutil.ReadFile(inputFile + "." + strconv.Itoa(i) + ".cksum") - } - - for i := 0; i < k+m; i++ { - crcChunk, err := crc32c.Crc32c(chunks[i]) - if err != nil { - chunks[i] = nil - continue - } - crcChunkStr := strconv.Itoa(int(crcChunk)) - if string(cksums[i]) != crcChunkStr { - chunks[i] = nil - } - } - - // get length - lengthBytes, err := ioutil.ReadFile(inputFile + ".length") - if err != nil { - log.Fatal(err) - } - - lengthString := string(lengthBytes) - length, err := strconv.Atoi(lengthString) - if err != nil { - log.Fatal(err) - } - - // set up encoder - erasureParameters, _ := erasure.ParseEncoderParams(k, m, erasure.CAUCHY) - - // Get decoder - encoder := erasure.NewEncoder(erasureParameters) - - // decode data - decodedData, err := encoder.Decode(chunks, length) - if err != nil { - log.Fatal(err) - } - - // append decoded data - length, err = outputFile.Write(decodedData) - if err != nil { - - log.Fatal(err) - } - } -} diff --git a/cmd/erasure-demo/encode.go b/cmd/erasure-demo/encode.go deleted file mode 100644 index e799447f0..000000000 --- a/cmd/erasure-demo/encode.go +++ /dev/null @@ -1,83 +0,0 @@ -package main - -import ( - "bytes" - "io/ioutil" - "log" - "os" - "strconv" - - "github.com/codegangsta/cli" - "github.com/minio-io/minio/pkgs/checksum/crc32c" - "github.com/minio-io/minio/pkgs/erasure" - "github.com/minio-io/minio/pkgs/split" -) - -func encode(c *cli.Context) { - // check if minio-encode called without parameters - if len(c.Args()) != 1 { - cli.ShowCommandHelp(c, "encode") - return - } - - config, err := parseInput(c) - if err != nil { - log.Fatal(err) - } - - // get file - inputFile, err := os.Open(config.input) - defer inputFile.Close() - if err != nil { - log.Fatal(err) - } - - // read file - input, err := ioutil.ReadAll(inputFile) - if err != nil { - log.Fatal(err) - } - - // set up encoder - erasureParameters, _ := erasure.ParseEncoderParams(config.k, config.m, erasure.CAUCHY) - - // Init new encoder - encoder := erasure.NewEncoder(erasureParameters) - - // encode data - if config.blockSize == 0 { - encodedData, length := encoder.Encode(input) - for key, data := range encodedData { - crc, err := crc32c.Crc32c(data) - if err != nil { - log.Fatal(err) - } - crcString := strconv.Itoa(int(crc)) - ioutil.WriteFile(config.output+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600) - ioutil.WriteFile(config.output+"."+strconv.Itoa(key), data, 0600) - ioutil.WriteFile(config.output+".length", []byte(strconv.Itoa(length)), 0600) - } - } else { - chunkCount := 0 - splitChannel := make(chan split.SplitMessage) - inputReader := bytes.NewReader(input) - go split.SplitStream(inputReader, config.blockSize, splitChannel) - for chunk := range splitChannel { - if chunk.Err != nil { - log.Fatal(chunk.Err) - } - encodedData, length := encoder.Encode(chunk.Data) - for key, data := range encodedData { - crc, err := crc32c.Crc32c(data) - if err != nil { - log.Fatal(err) - } - crcString := strconv.Itoa(int(crc)) - ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key)+".cksum", []byte(crcString), 0600) - ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+"."+strconv.Itoa(key), data, 0600) - } - ioutil.WriteFile(config.output+"."+strconv.Itoa(chunkCount)+".length", []byte(strconv.Itoa(length)), 0600) - chunkCount += 1 - } - } -} diff --git a/cmd/erasure-demo/get.go b/cmd/erasure-demo/get.go deleted file mode 100644 index 991c5a09e..000000000 --- a/cmd/erasure-demo/get.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "io" - "log" - "os" - - "github.com/codegangsta/cli" -) - -func get(c *cli.Context) { - config, err := parseInput(c) - if err != nil { - log.Fatal(err) - } - var objectReader io.Reader - objectName := c.Args().Get(0) - - switch config.storageDriver { - case "erasure": - { - if len(objectName) == 0 { - if objectReader, err = erasureGetList(config, ""); err != nil { - log.Fatal(err) - } - } else { - if objectReader, err = erasureGet(config, objectName); err != nil { - log.Fatal(err) - } - } - } - default: - { - log.Fatal("Unknown driver") - } - } - if objectReader == nil { - log.Fatal("Object not found") - } - io.Copy(os.Stdout, objectReader) -} diff --git a/cmd/erasure-demo/main.go b/cmd/erasure-demo/main.go deleted file mode 100644 index 59e91565c..000000000 --- a/cmd/erasure-demo/main.go +++ /dev/null @@ -1,218 +0,0 @@ -package main - -import ( - "errors" - "log" - "os" - "os/user" - "path" - "strconv" - "strings" - - "github.com/codegangsta/cli" - "github.com/minio-io/minio/pkgs/strbyteconv" -) - -func main() { - app := cli.NewApp() - app.Name = "erasure-demo" - app.Usage = "erasure encode a byte stream" - app.Commands = []cli.Command{ - { - Name: "encode", - Usage: "erasure encode a byte stream", - Action: encode, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "output,o", - Value: "", - Usage: "Output file", - }, - cli.StringFlag{ - Name: "protection-level", - Value: "10,6", - Usage: "data,parity", - }, - cli.StringFlag{ - Name: "block-size", - Value: "1M", - Usage: "Size of blocks. Examples: 1K, 1M, full", - }, - }, - }, - { - Name: "decode", - Usage: "erasure decode a byte stream", - Action: decode, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "output,o", - Value: "", - Usage: "Output file", - }, - cli.StringFlag{ - Name: "protection-level", - Value: "10,6", - Usage: "data,parity", - }, - }, - }, - { - Name: "get", - Usage: "get an object", - Action: get, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "root", - Value: getMinioDir(), - Usage: "", - }, - cli.StringFlag{ - Name: "driver", - Value: "erasure", - Usage: "erasure", - }, - cli.StringFlag{ - Name: "protection-level", - Value: "10,6", - Usage: "data,parity", - }, - cli.StringFlag{ - Name: "block-size", - Value: "1M", - Usage: "Size of blocks. Examples: 1K, 1M, full", - }, - }, - }, - { - Name: "put", - Usage: "put an object", - Action: put, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "root", - Value: getMinioDir(), - Usage: "", - }, - cli.StringFlag{ - Name: "protection-level", - Value: "10,6", - Usage: "data,parity", - }, - cli.StringFlag{ - Name: "block-size", - Value: "1M", - Usage: "Size of blocks. Examples: 1K, 1M, full", - }, - cli.StringFlag{ - Name: "driver", - Value: "erasure", - Usage: "erasure", - }, - }, - }, - } - app.Run(os.Args) -} - -// config representing cli input -type inputConfig struct { - input string - output string - k int - m int - blockSize uint64 - rootDir string - storageDriver string -} - -// parses input and returns an inputConfig with parsed input -func parseInput(c *cli.Context) (inputConfig, error) { - // get input path - inputFilePath := c.Args().Get(0) - - // get output path - outputFilePath := inputFilePath - if c.String("output") != "" { - outputFilePath = c.String("output") - } - - var k, m int - if c.String("protection-level") != "" { - protectionLevel := c.String("protection-level") - protectionLevelSplit := strings.Split(protectionLevel, ",") - if len(protectionLevelSplit) != 2 { - return inputConfig{}, errors.New("Malformed input for protection-level") - } - - var err error - k, err = strconv.Atoi(protectionLevelSplit[0]) - if err != nil { - return inputConfig{}, err - } - m, err = strconv.Atoi(protectionLevelSplit[1]) - if err != nil { - return inputConfig{}, err - } - } - - var blockSize uint64 - blockSize = 0 - if c.String("block-size") != "" { - if c.String("block-size") != "full" { - var err error - blockSize, err = strbyteconv.StringToBytes(c.String("block-size")) - if err != nil { - return inputConfig{}, err - } - } - } - - var rootDir string - if c.String("root") != "" { - rootDir = c.String("root") - } - - var storageDriver string - if c.String("driver") != "" { - storageDriver = c.String("driver") - } - - config := inputConfig{ - input: inputFilePath, - output: outputFilePath, - k: k, - m: m, - blockSize: blockSize, - rootDir: rootDir, - storageDriver: storageDriver, - } - return config, nil -} - -func getMinioDir() string { - user, err := user.Current() - if err != nil { - log.Fatal(err) - } - homePath := user.HomeDir - minioPath := path.Join(homePath, ".minio") - err = _initMinioDir(minioPath) - if err != nil { - log.Fatal(err) - } - return minioPath -} - -func _initMinioDir(dirPath string) error { - _, err := os.Lstat(dirPath) - if err != nil { - log.Printf("%s not found, creating a new-one for the first time", - dirPath) - err = os.Mkdir(dirPath, 0700) - if err != nil { - return err - } - } - return nil -} diff --git a/cmd/minio/minio.go b/cmd/minio/minio.go new file mode 100644 index 000000000..9fe2a5f82 --- /dev/null +++ b/cmd/minio/minio.go @@ -0,0 +1,95 @@ +package main + +import ( + "os" + + "github.com/codegangsta/cli" + "github.com/minio-io/minio/pkgs/minio" +) + +func main() { + app := cli.NewApp() + app.Name = "minio" + app.Usage = "minio - object storage" + app.Commands = []cli.Command{ + { + Name: "encode", + Usage: "erasure encode a byte stream", + Action: minio.Encode, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "root", + Value: minio.Getobjectdir(".minio/erasure"), + Usage: "", + }, + cli.StringFlag{ + Name: "staging", + Value: minio.Getobjectdir(".minio/staging"), + Usage: "", + }, + cli.StringFlag{ + Name: "protection-level", + Value: "10,6", + Usage: "data,parity", + }, + cli.StringFlag{ + Name: "block-size", + Value: "1M", + Usage: "Size of blocks. Examples: 1K, 1M, full", + }, + }, + }, + { + Name: "get", + Usage: "get an object", + Action: minio.Get, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "root", + Value: minio.Getobjectdir(".minio/erasure"), + Usage: "", + }, + cli.StringFlag{ + Name: "protection-level", + Value: "10,6", + Usage: "data,parity", + }, + cli.StringFlag{ + Name: "block-size", + Value: "1M", + Usage: "Size of blocks. Examples: 1K, 1M, full", + }, + }, + }, + { + Name: "put", + Usage: "put an object", + Action: minio.Put, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "staging", + Value: minio.Getobjectdir(".minio/staging"), + Usage: "", + }, + cli.StringFlag{ + Name: "block-size", + Value: "1M", + Usage: "Size of blocks. Examples: 1K, 1M, full", + }, + }, + }, + { + Name: "list", + Usage: "list objects", + Action: minio.List, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "root", + Value: minio.Getobjectdir(".minio/erasure"), + Usage: "", + }, + }, + }, + } + app.Run(os.Args) +} diff --git a/pkgs/minio/common.go b/pkgs/minio/common.go new file mode 100644 index 000000000..a799016bd --- /dev/null +++ b/pkgs/minio/common.go @@ -0,0 +1,103 @@ +package minio + +import ( + "errors" + "log" + "os" + "os/user" + "path" + "strconv" + "strings" + + "github.com/codegangsta/cli" + "github.com/minio-io/minio/pkgs/strbyteconv" +) + +// config representing cli input +type inputConfig struct { + k int + m int + blockSize uint64 + rootDir string + stagingDir string +} + +// parses input and returns an inputConfig with parsed input +func parseInput(c *cli.Context) (inputConfig, error) { + var k, m int + if c.String("protection-level") != "" { + protectionLevel := c.String("protection-level") + protectionLevelSplit := strings.Split(protectionLevel, ",") + if len(protectionLevelSplit) != 2 { + return inputConfig{}, errors.New("Malformed input for protection-level") + } + + var err error + k, err = strconv.Atoi(protectionLevelSplit[0]) + if err != nil { + return inputConfig{}, err + } + m, err = strconv.Atoi(protectionLevelSplit[1]) + if err != nil { + return inputConfig{}, err + } + } + + var blockSize uint64 + blockSize = 0 + if c.String("block-size") != "" { + if c.String("block-size") != "full" { + var err error + blockSize, err = strbyteconv.StringToBytes(c.String("block-size")) + if err != nil { + return inputConfig{}, err + } + } + } + + var rootDir string + if c.String("root") != "" { + rootDir = c.String("root") + } + + var stagingDir string + if c.String("staging") != "" { + stagingDir = c.String("staging") + } + + config := inputConfig{ + k: k, + m: m, + blockSize: blockSize, + rootDir: rootDir, + stagingDir: stagingDir, + } + return config, nil +} + +func Getobjectdir(basename string) string { + user, err := user.Current() + if err != nil { + log.Fatal(err) + } + homePath := user.HomeDir + minioPath := path.Join(homePath, basename) + err = _initDir(minioPath) + if err != nil { + log.Fatal(err) + } + return minioPath +} + +func _initDir(dirPath string) error { + _, err := os.Lstat(dirPath) + if err != nil { + log.Printf("%s not found, creating a new-one for the first time", + dirPath) + err = os.MkdirAll(dirPath, 0700) + if err != nil { + return err + } + } + return nil +} diff --git a/pkgs/minio/encode.go b/pkgs/minio/encode.go new file mode 100644 index 000000000..f28605b1d --- /dev/null +++ b/pkgs/minio/encode.go @@ -0,0 +1,37 @@ +package minio + +import ( + "log" + + "github.com/codegangsta/cli" +) + +func Encode(c *cli.Context) { + config, err := parseInput(c) + if err != nil { + log.Fatal(err) + } + var objectName string + switch len(c.Args()) { + case 1: + objectName = c.Args().Get(0) + default: + log.Fatal("Please specify a valid object name \n # erasure-demo encode [OBJECTNAME]") + } + + // Get from staging area + stagingConfig := config + stagingConfig.k = 2 + stagingConfig.m = 1 + stagingConfig.rootDir = config.stagingDir + reader, err := erasureGet(stagingConfig, objectName) + if err != nil { + log.Fatal(err) + } + + // Increase parity to user defined or default 10,6 + err = erasurePut(config, objectName, reader) + if err != nil { + log.Fatal(err) + } +} diff --git a/cmd/erasure-demo/erasure.go b/pkgs/minio/erasure.go similarity index 68% rename from cmd/erasure-demo/erasure.go rename to pkgs/minio/erasure.go index 18acc8c5b..64d2cbb8f 100644 --- a/cmd/erasure-demo/erasure.go +++ b/pkgs/minio/erasure.go @@ -1,11 +1,10 @@ -package main +package minio import ( "bytes" "encoding/json" "io" "os" - "path" "github.com/minio-io/minio/pkgs/storage" es "github.com/minio-io/minio/pkgs/storage/encodedstorage" @@ -13,8 +12,7 @@ import ( func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) { var objectStorage storage.ObjectStorage - rootDir := path.Join(config.rootDir, config.storageDriver) - objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize) + objectStorage, err := es.NewStorage(config.rootDir, config.k, config.m, config.blockSize) if err != nil { return nil, err } @@ -33,8 +31,7 @@ func erasureGetList(config inputConfig, objectPath string) (io.Reader, error) { func erasureGet(config inputConfig, objectPath string) (io.Reader, error) { var objectStorage storage.ObjectStorage - rootDir := path.Join(config.rootDir, config.storageDriver) - objectStorage, err := es.NewStorage(rootDir, config.k, config.m, config.blockSize) + objectStorage, err := es.NewStorage(config.rootDir, config.k, config.m, config.blockSize) if err != nil { return nil, err } @@ -47,12 +44,11 @@ func erasureGet(config inputConfig, objectPath string) (io.Reader, error) { func erasurePut(config inputConfig, objectPath string, reader io.Reader) error { var err error - rootDir := path.Join(config.rootDir, config.storageDriver) - if err := os.MkdirAll(rootDir, 0700); err != nil { + if err := os.MkdirAll(config.rootDir, 0700); err != nil { return err } var objectStorage storage.ObjectStorage - if objectStorage, err = es.NewStorage(rootDir, config.k, config.m, config.blockSize); err != nil { + if objectStorage, err = es.NewStorage(config.rootDir, config.k, config.m, config.blockSize); err != nil { return err } if err = objectStorage.Put(objectPath, reader); err != nil { diff --git a/pkgs/minio/get.go b/pkgs/minio/get.go new file mode 100644 index 000000000..68f271965 --- /dev/null +++ b/pkgs/minio/get.go @@ -0,0 +1,30 @@ +package minio + +import ( + "io" + "log" + "os" + + "github.com/codegangsta/cli" +) + +func Get(c *cli.Context) { + config, err := parseInput(c) + if err != nil { + log.Fatal(err) + } + var objectName string + var objectReader io.Reader + switch len(c.Args()) { + case 1: + objectName = c.Args().Get(0) + default: + log.Fatal("Please specify a valid object name \n # erasure-demo get [OBJECTNAME]") + } + + if objectReader, err = erasureGet(config, objectName); err != nil { + log.Fatal(err) + } + + io.Copy(os.Stdout, objectReader) +} diff --git a/pkgs/minio/init.go b/pkgs/minio/init.go new file mode 100644 index 000000000..c5576b1fd --- /dev/null +++ b/pkgs/minio/init.go @@ -0,0 +1 @@ +package minio diff --git a/pkgs/minio/list.go b/pkgs/minio/list.go new file mode 100644 index 000000000..59d3769ad --- /dev/null +++ b/pkgs/minio/list.go @@ -0,0 +1,24 @@ +package minio + +import ( + "io" + "log" + "os" + + "github.com/codegangsta/cli" +) + +func List(c *cli.Context) { + config, err := parseInput(c) + if err != nil { + log.Fatal(err) + } + + config.k = 10 + config.m = 6 + reader, err := erasureGetList(config, "") + if err != nil { + log.Fatal(err) + } + io.Copy(os.Stdout, reader) +} diff --git a/cmd/erasure-demo/put.go b/pkgs/minio/put.go similarity index 64% rename from cmd/erasure-demo/put.go rename to pkgs/minio/put.go index 5e660a3d6..021d49556 100644 --- a/cmd/erasure-demo/put.go +++ b/pkgs/minio/put.go @@ -1,4 +1,4 @@ -package main +package minio import ( "log" @@ -8,7 +8,7 @@ import ( "github.com/codegangsta/cli" ) -func put(c *cli.Context) { +func Put(c *cli.Context) { config, err := parseInput(c) if err != nil { log.Fatal(err) @@ -25,19 +25,19 @@ func put(c *cli.Context) { log.Fatal("Please specify a valid object name \n # erasure-demo put [OBJECTNAME] [FILENAME]") } inputFile, err := os.Open(filePath) + defer inputFile.Close() if err != nil { log.Fatal(err) } - switch config.storageDriver { - case "erasure": - { - if err := erasurePut(config, objectName, inputFile); err != nil { - log.Fatal(err) - } - } - default: - { - log.Fatal("Unknown driver") - } + + // Staging parity + stagingConfig := config + stagingConfig.k = 2 + stagingConfig.m = 1 + stagingConfig.rootDir = config.stagingDir + + if err := erasurePut(stagingConfig, objectName, inputFile); err != nil { + log.Fatal(err) } + } diff --git a/pkgs/minio/verify.go b/pkgs/minio/verify.go new file mode 100644 index 000000000..c5576b1fd --- /dev/null +++ b/pkgs/minio/verify.go @@ -0,0 +1 @@ +package minio diff --git a/pkgs/storage/appendstorage/append_storage.go b/pkgs/storage/appendstorage/append_storage.go index 97f15e43c..27fc32f03 100644 --- a/pkgs/storage/appendstorage/append_storage.go +++ b/pkgs/storage/appendstorage/append_storage.go @@ -80,7 +80,7 @@ func NewStorage(rootDir string, slice int) (storage.ObjectStorage, error) { func (storage *appendStorage) Get(objectPath string) (io.Reader, error) { header, ok := storage.objects[objectPath] if ok == false { - return nil, nil + return nil, errors.New("Object not found") } offset := header.Offset diff --git a/pkgs/storage/encodedstorage/encoded_storage.go b/pkgs/storage/encodedstorage/encoded_storage.go index cc00f60f1..a58dc8b37 100644 --- a/pkgs/storage/encodedstorage/encoded_storage.go +++ b/pkgs/storage/encodedstorage/encoded_storage.go @@ -110,7 +110,7 @@ func NewStorage(rootDir string, k, m int, blockSize uint64) (storage.ObjectStora func (eStorage *encodedStorage) Get(objectPath string) (io.Reader, error) { entry, ok := eStorage.objects[objectPath] if ok == false { - return nil, nil + return nil, errors.New("Object not found") } reader, writer := io.Pipe() go eStorage.readObject(objectPath, entry, writer)