Merge pull request #28 from harshavardhana/pr_out_enable_way_to_split_files_at_input_bytes

Enable way to split files at input bytes

Makefile

@@ -13,7 +13,10 @@ build-erasure:
 build-signify:
 	@cd pkgs/signify && ${MAKE} ${MAKE_OPTIONS}
 
-cover: build-erasure build-signify
+build-split:
+	@cd pkgs/split && ${MAKE} ${MAKE_OPTIONS}
+
+cover: build-erasure build-signify build-split
 	@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/storage
 	@godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway
 

pkgs/split/.gitignore

@@ -0,0 +1 @@
+TESTFILE.*

pkgs/split/Makefile

@@ -0,0 +1,11 @@
+all: build test
+.PHONY: all
+
+build:
+	@godep go build
+
+test: build
+	@godep go test -race -coverprofile=cover.out
+
+clean:
+	@rm -v TESTFILE.* cover.out

pkgs/split/split.c

@@ -0,0 +1,120 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <ctype.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+#include "split.h"
+
+static
+size_t _get_filesize(int fd)
+{
+        struct stat st;
+        assert(fstat(fd, &st) != -1);
+        return (ssize_t) st.st_size;
+}
+
+static
+ssize_t read_write_chunk(int in, int out, ssize_t bytes)
+{
+        ssize_t n, m;
+        char *buf = NULL;
+
+        buf = calloc(bytes, 1);
+        assert(buf != NULL);
+
+        n = read(in, buf, bytes);
+        if (n < 0)
+                return -1;
+
+        m = write(out, buf, n);
+        if (m < 0)
+                return -1;
+
+        if (buf)
+                free(buf);
+        return m;
+}
+
+static int
+_allocate_newchunk(char *chunkname)
+{
+
+        int chunk = -1;
+
+        if (!chunkname)
+                return -1;
+
+        if ((strlen(chunkname)) >= MAXPATHLEN) {
+                fprintf (stderr, "chunkname + suffix too long");
+                return -1;
+        }
+
+        chunk = open (chunkname, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR);
+        return chunk;
+}
+
+/*
+ * Generate chunks by chunking at input bytes.
+ */
+int
+minio_split(char *filename, ssize_t bytecnt)
+{
+	ssize_t fsize;
+        int output = -1;
+        int input = -1;
+        int remainder = 0;
+        char newchunk[MAXPATHLEN] = {0,};
+        int chunk_size = 0;
+        int i = 0;
+
+        if (bytecnt < 0)
+                return -1;
+
+        if ((input = open(filename, O_RDONLY)) < 0)
+                return -1;
+
+        fsize = _get_filesize(input);
+        remainder = fsize % bytecnt;
+
+        if (remainder == 0)
+                chunk_size = fsize / bytecnt;
+        else
+                chunk_size = (fsize + (bytecnt - remainder)) / bytecnt;
+
+        if (chunk_size == 0)
+                return -1;
+
+        for (i = 0; i < chunk_size; i++) {
+                snprintf (newchunk, sizeof(newchunk)-1, "%s.%d", filename, i);
+
+                if ((output = _allocate_newchunk(newchunk)) < 0)
+                        return -1;
+
+                if (read_write_chunk(input, output, bytecnt) < 0)
+                        return -1;
+        }
+        return 0;
+}

pkgs/split/split.go

@@ -0,0 +1,53 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// +build linux
+// amd64
+
+package split
+
+// #include <stdlib.h>
+// #include <stdlib.h>
+//
+// #include "split.h"
+import "C"
+import (
+	"errors"
+	"github.com/minio-io/minio/pkgs/strbyteconv"
+	"unsafe"
+)
+
+type Split struct {
+	bytecnt C.ssize_t
+	bname   *C.char
+}
+
+func (b *Split) GenChunks(bname string, bytestr string) error {
+	bytecnt, err := strbyteconv.StringToBytes(bytestr)
+	if err != nil {
+		return err
+	}
+
+	b.bytecnt = C.ssize_t(bytecnt)
+	b.bname = C.CString(bname)
+	defer C.free(unsafe.Pointer(b.bname))
+
+	value := C.minio_split(b.bname, b.bytecnt)
+	if value < 0 {
+		return errors.New("File split failed")
+	}
+	return nil
+}

pkgs/split/split.h

@@ -0,0 +1,22 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __SPLIT_H__
+#define __SPLIT_H__
+
+int minio_split(char *filename, ssize_t bytecnt);
+
+#endif  /* __SPLIT_H__ */

pkgs/split/split_test.go

@@ -0,0 +1,34 @@
+/*
+ * Mini Object Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package split
+
+import (
+	. "gopkg.in/check.v1"
+	"testing"
+)
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func Test(t *testing.T) { TestingT(t) }
+
+func (s *MySuite) TestFileSplit(c *C) {
+	b := Split{}
+	err := b.GenChunks("TESTFILE", "20KB")
+	c.Assert(err, IsNil)
+}