diff --git a/Makefile b/Makefile index 748535dbf..c3a0170e6 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,10 @@ build-erasure: build-signify: @cd pkgs/signify && ${MAKE} ${MAKE_OPTIONS} -cover: build-erasure build-signify +build-split: + @cd pkgs/split && ${MAKE} ${MAKE_OPTIONS} + +cover: build-erasure build-signify build-split @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/storage @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway diff --git a/pkgs/split/.gitignore b/pkgs/split/.gitignore new file mode 100644 index 000000000..411512991 --- /dev/null +++ b/pkgs/split/.gitignore @@ -0,0 +1 @@ +TESTFILE.* \ No newline at end of file diff --git a/pkgs/split/Makefile b/pkgs/split/Makefile new file mode 100644 index 000000000..5e12bca9b --- /dev/null +++ b/pkgs/split/Makefile @@ -0,0 +1,11 @@ +all: build test +.PHONY: all + +build: + @godep go build + +test: build + @godep go test -race -coverprofile=cover.out + +clean: + @rm -v TESTFILE.* cover.out diff --git a/pkgs/split/TESTFILE b/pkgs/split/TESTFILE new file mode 100644 index 000000000..6d23118f0 Binary files /dev/null and b/pkgs/split/TESTFILE differ diff --git a/pkgs/split/split.c b/pkgs/split/split.c new file mode 100644 index 000000000..0a3cef41a --- /dev/null +++ b/pkgs/split/split.c @@ -0,0 +1,120 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "split.h" + +static +size_t _get_filesize(int fd) +{ + struct stat st; + assert(fstat(fd, &st) != -1); + return (ssize_t) st.st_size; +} + +static +ssize_t read_write_chunk(int in, int out, ssize_t bytes) +{ + ssize_t n, m; + char *buf = NULL; + + buf = calloc(bytes, 1); + assert(buf != NULL); + + n = read(in, buf, bytes); + if (n < 0) + return -1; + + m = write(out, buf, n); + if (m < 0) + return -1; + + if (buf) + free(buf); + return m; +} + +static int +_allocate_newchunk(char *chunkname) +{ + + int chunk = -1; + + if (!chunkname) + return -1; + + if ((strlen(chunkname)) >= MAXPATHLEN) { + fprintf (stderr, "chunkname + suffix too long"); + return -1; + } + + chunk = open (chunkname, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR); + return chunk; +} + +/* + * Generate chunks by chunking at input bytes. + */ +int +minio_split(char *filename, ssize_t bytecnt) +{ + ssize_t fsize; + int output = -1; + int input = -1; + int remainder = 0; + char newchunk[MAXPATHLEN] = {0,}; + int chunk_size = 0; + int i = 0; + + if (bytecnt < 0) + return -1; + + if ((input = open(filename, O_RDONLY)) < 0) + return -1; + + fsize = _get_filesize(input); + remainder = fsize % bytecnt; + + if (remainder == 0) + chunk_size = fsize / bytecnt; + else + chunk_size = (fsize + (bytecnt - remainder)) / bytecnt; + + if (chunk_size == 0) + return -1; + + for (i = 0; i < chunk_size; i++) { + snprintf (newchunk, sizeof(newchunk)-1, "%s.%d", filename, i); + + if ((output = _allocate_newchunk(newchunk)) < 0) + return -1; + + if (read_write_chunk(input, output, bytecnt) < 0) + return -1; + } + return 0; +} diff --git a/pkgs/split/split.go b/pkgs/split/split.go new file mode 100644 index 000000000..64dc7cae2 --- /dev/null +++ b/pkgs/split/split.go @@ -0,0 +1,53 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// +build linux +// amd64 + +package split + +// #include +// #include +// +// #include "split.h" +import "C" +import ( + "errors" + "github.com/minio-io/minio/pkgs/strbyteconv" + "unsafe" +) + +type Split struct { + bytecnt C.ssize_t + bname *C.char +} + +func (b *Split) GenChunks(bname string, bytestr string) error { + bytecnt, err := strbyteconv.StringToBytes(bytestr) + if err != nil { + return err + } + + b.bytecnt = C.ssize_t(bytecnt) + b.bname = C.CString(bname) + defer C.free(unsafe.Pointer(b.bname)) + + value := C.minio_split(b.bname, b.bytecnt) + if value < 0 { + return errors.New("File split failed") + } + return nil +} diff --git a/pkgs/split/split.h b/pkgs/split/split.h new file mode 100644 index 000000000..f8ccca479 --- /dev/null +++ b/pkgs/split/split.h @@ -0,0 +1,22 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SPLIT_H__ +#define __SPLIT_H__ + +int minio_split(char *filename, ssize_t bytecnt); + +#endif /* __SPLIT_H__ */ diff --git a/pkgs/split/split_test.go b/pkgs/split/split_test.go new file mode 100644 index 000000000..62dc28ebd --- /dev/null +++ b/pkgs/split/split_test.go @@ -0,0 +1,34 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package split + +import ( + . "gopkg.in/check.v1" + "testing" +) + +type MySuite struct{} + +var _ = Suite(&MySuite{}) + +func Test(t *testing.T) { TestingT(t) } + +func (s *MySuite) TestFileSplit(c *C) { + b := Split{} + err := b.GenChunks("TESTFILE", "20KB") + c.Assert(err, IsNil) +}