diff --git a/pkgs/split/Makefile b/pkgs/split/Makefile index 5e12bca9b..0f1ee2600 100644 --- a/pkgs/split/Makefile +++ b/pkgs/split/Makefile @@ -8,4 +8,4 @@ test: build @godep go test -race -coverprofile=cover.out clean: - @rm -v TESTFILE.* cover.out + @rm -fv TESTFILE.* TESTPREFIX.* cover.out diff --git a/pkgs/split/split.c b/pkgs/split/split.c deleted file mode 100644 index 0a3cef41a..000000000 --- a/pkgs/split/split.c +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Mini Object Storage, (C) 2014 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "split.h" - -static -size_t _get_filesize(int fd) -{ - struct stat st; - assert(fstat(fd, &st) != -1); - return (ssize_t) st.st_size; -} - -static -ssize_t read_write_chunk(int in, int out, ssize_t bytes) -{ - ssize_t n, m; - char *buf = NULL; - - buf = calloc(bytes, 1); - assert(buf != NULL); - - n = read(in, buf, bytes); - if (n < 0) - return -1; - - m = write(out, buf, n); - if (m < 0) - return -1; - - if (buf) - free(buf); - return m; -} - -static int -_allocate_newchunk(char *chunkname) -{ - - int chunk = -1; - - if (!chunkname) - return -1; - - if ((strlen(chunkname)) >= MAXPATHLEN) { - fprintf (stderr, "chunkname + suffix too long"); - return -1; - } - - chunk = open (chunkname, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR); - return chunk; -} - -/* - * Generate chunks by chunking at input bytes. - */ -int -minio_split(char *filename, ssize_t bytecnt) -{ - ssize_t fsize; - int output = -1; - int input = -1; - int remainder = 0; - char newchunk[MAXPATHLEN] = {0,}; - int chunk_size = 0; - int i = 0; - - if (bytecnt < 0) - return -1; - - if ((input = open(filename, O_RDONLY)) < 0) - return -1; - - fsize = _get_filesize(input); - remainder = fsize % bytecnt; - - if (remainder == 0) - chunk_size = fsize / bytecnt; - else - chunk_size = (fsize + (bytecnt - remainder)) / bytecnt; - - if (chunk_size == 0) - return -1; - - for (i = 0; i < chunk_size; i++) { - snprintf (newchunk, sizeof(newchunk)-1, "%s.%d", filename, i); - - if ((output = _allocate_newchunk(newchunk)) < 0) - return -1; - - if (read_write_chunk(input, output, bytecnt) < 0) - return -1; - } - return 0; -} diff --git a/pkgs/split/split.go b/pkgs/split/split.go index fd50359b3..f0e996252 100644 --- a/pkgs/split/split.go +++ b/pkgs/split/split.go @@ -19,11 +19,6 @@ package split -// #include -// #include -// -// #include "split.h" -import "C" import ( "bufio" "bytes" @@ -32,38 +27,21 @@ import ( "io/ioutil" "os" "strconv" - "unsafe" "github.com/minio-io/minio/pkgs/strbyteconv" ) type Split struct { - bytecnt C.ssize_t - bname *C.char -} - -func (b *Split) GenChunks(bname string, bytestr string) error { - bytecnt, err := strbyteconv.StringToBytes(bytestr) - if err != nil { - return err - } - - b.bytecnt = C.ssize_t(bytecnt) - b.bname = C.CString(bname) - defer C.free(unsafe.Pointer(b.bname)) - - value := C.minio_split(b.bname, b.bytecnt) - if value < 0 { - return errors.New("File split failed") - } - return nil -} - -type GoSplit struct { file string offset uint64 } +// Message structure for results from the SplitStream goroutine +type ByteMessage struct { + Data []byte + Err error +} + // SplitStream reads from io.Reader, splits the data into chunks, and sends // each chunk to the channel. This method runs until an EOF or error occurs. If // an error occurs, the method sends the error over the channel and returns. @@ -117,20 +95,24 @@ func SplitStream(reader io.Reader, chunkSize uint64, ch chan ByteMessage) { close(ch) } -// Message structure for results from the SplitStream goroutine -type ByteMessage struct { - Data []byte - Err error -} - // Takes a file and splits it into chunks with size chunkSize. The output // filename is given with outputPrefix. -func SplitFilesWithPrefix(filename string, chunkSize uint64, outputPrefix string) error { +func SplitFilesWithPrefix(filename string, chunkstr string, outputPrefix string) error { // open file file, err := os.Open(filename) if err != nil { return err } + + if outputPrefix == "" { + return errors.New("Invalid argument outputPrefix cannot be empty string") + } + + chunkSize, err := strbyteconv.StringToBytes(chunkstr) + if err != nil { + return err + } + // start stream splitting goroutine ch := make(chan ByteMessage) go SplitStream(file, chunkSize, ch) diff --git a/pkgs/split/split.h b/pkgs/split/split.h deleted file mode 100644 index f8ccca479..000000000 --- a/pkgs/split/split.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Mini Object Storage, (C) 2014 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __SPLIT_H__ -#define __SPLIT_H__ - -int minio_split(char *filename, ssize_t bytecnt); - -#endif /* __SPLIT_H__ */ diff --git a/pkgs/split/split_test.go b/pkgs/split/split_test.go index 33b5dc658..37324463f 100644 --- a/pkgs/split/split_test.go +++ b/pkgs/split/split_test.go @@ -32,12 +32,6 @@ var _ = Suite(&MySuite{}) func Test(t *testing.T) { TestingT(t) } -func (s *MySuite) TestFileSplit(c *C) { - b := Split{} - err := b.GenChunks("TESTFILE", "1KB") - c.Assert(err, IsNil) -} - func (s *MySuite) TestSplitStream(c *C) { var bytesBuffer bytes.Buffer bytesWriter := bufio.NewWriter(&bytesBuffer) @@ -59,6 +53,8 @@ func (s *MySuite) TestSplitStream(c *C) { } func (s *MySuite) TestFileSplit2(c *C) { - err := SplitFilesWithPrefix("TESTFILE", 1024, "TESTPREFIX") + err := SplitFilesWithPrefix("TESTFILE", "1KB", "TESTPREFIX") c.Assert(err, IsNil) + err = SplitFilesWithPrefix("TESTFILE", "1KB", "") + c.Assert(err, Not(IsNil)) }