oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #75 from harshavardhana/pr_out_remove_unnecessary_c_code_and_use_everything_from_golang

Browse Source
master
Frederick F. Kautz IV 10 years ago
parent 59c1197f47 3a6cac8ada
commit dac14cd7a1
5 changed files with 21 additions and 185 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      pkgs/split/Makefile
  2. 120
      pkgs/split/split.c
  3. 52
      pkgs/split/split.go
  4. 22
      pkgs/split/split.h
  5. 10
      pkgs/split/split_test.go

2
pkgs/split/Makefile
Unescape View File

@ -8,4 +8,4 @@ test: build
@godep go test -race -coverprofile=cover.out
clean:
@rm -v TESTFILE.* cover.out
@rm -fv TESTFILE.* TESTPREFIX.* cover.out

120
pkgs/split/split.c
Unescape View File

@ -1,120 +0,0 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <sys/param.h>
#include <sys/types.h>
#include <ctype.h>
#include <fcntl.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <assert.h>
#include "split.h"
static
size_t _get_filesize(int fd)
{
struct stat st;
assert(fstat(fd, &st) != -1);
return (ssize_t) st.st_size;
}
static
ssize_t read_write_chunk(int in, int out, ssize_t bytes)
{
ssize_t n, m;
char *buf = NULL;
buf = calloc(bytes, 1);
assert(buf != NULL);
n = read(in, buf, bytes);
if (n < 0)
return -1;
m = write(out, buf, n);
if (m < 0)
return -1;
if (buf)
free(buf);
return m;
}
static int
_allocate_newchunk(char *chunkname)
{
int chunk = -1;
if (!chunkname)
return -1;
if ((strlen(chunkname)) >= MAXPATHLEN) {
fprintf (stderr, "chunkname + suffix too long");
return -1;
}
chunk = open (chunkname, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR);
return chunk;
}
/*
* Generate chunks by chunking at input bytes.
*/
int
minio_split(char *filename, ssize_t bytecnt)
{
ssize_t fsize;
int output = -1;
int input = -1;
int remainder = 0;
char newchunk[MAXPATHLEN] = {0,};
int chunk_size = 0;
int i = 0;
if (bytecnt < 0)
return -1;
if ((input = open(filename, O_RDONLY)) < 0)
return -1;
fsize = _get_filesize(input);
remainder = fsize % bytecnt;
if (remainder == 0)
chunk_size = fsize / bytecnt;
else
chunk_size = (fsize + (bytecnt - remainder)) / bytecnt;
if (chunk_size == 0)
return -1;
for (i = 0; i < chunk_size; i++) {
snprintf (newchunk, sizeof(newchunk)-1, "%s.%d", filename, i);
if ((output = _allocate_newchunk(newchunk)) < 0)
return -1;
if (read_write_chunk(input, output, bytecnt) < 0)
return -1;
}
return 0;
}

52
pkgs/split/split.go
Unescape View File

@ -19,11 +19,6 @@
package split
// #include <stdlib.h>
// #include <stdlib.h>
//
// #include "split.h"
import "C"
import (
"bufio"
"bytes"
@ -32,38 +27,21 @@ import (
"io/ioutil"
"os"
"strconv"
"unsafe"
"github.com/minio-io/minio/pkgs/strbyteconv"
)
type Split struct {
bytecnt C.ssize_t
bname *C.char
}
func (b *Split) GenChunks(bname string, bytestr string) error {
bytecnt, err := strbyteconv.StringToBytes(bytestr)
if err != nil {
return err
}
b.bytecnt = C.ssize_t(bytecnt)
b.bname = C.CString(bname)
defer C.free(unsafe.Pointer(b.bname))
value := C.minio_split(b.bname, b.bytecnt)
if value < 0 {
return errors.New("File split failed")
}
return nil
}
type GoSplit struct {
file string
offset uint64
}
// Message structure for results from the SplitStream goroutine
type ByteMessage struct {
Data []byte
Err error
}
// SplitStream reads from io.Reader, splits the data into chunks, and sends
// each chunk to the channel. This method runs until an EOF or error occurs. If
// an error occurs, the method sends the error over the channel and returns.
@ -117,20 +95,24 @@ func SplitStream(reader io.Reader, chunkSize uint64, ch chan ByteMessage) {
close(ch)
}
// Message structure for results from the SplitStream goroutine
type ByteMessage struct {
Data []byte
Err error
}
// Takes a file and splits it into chunks with size chunkSize. The output
// filename is given with outputPrefix.
func SplitFilesWithPrefix(filename string, chunkSize uint64, outputPrefix string) error {
func SplitFilesWithPrefix(filename string, chunkstr string, outputPrefix string) error {
// open file
file, err := os.Open(filename)
if err != nil {
return err
}
if outputPrefix == "" {
return errors.New("Invalid argument outputPrefix cannot be empty string")
}
chunkSize, err := strbyteconv.StringToBytes(chunkstr)
if err != nil {
return err
}
// start stream splitting goroutine
ch := make(chan ByteMessage)
go SplitStream(file, chunkSize, ch)

22
pkgs/split/split.h
Unescape View File

@ -1,22 +0,0 @@
/*
* Mini Object Storage, (C) 2014 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __SPLIT_H__
#define __SPLIT_H__
int minio_split(char *filename, ssize_t bytecnt);
#endif /* __SPLIT_H__ */

10
pkgs/split/split_test.go
Unescape View File

@ -32,12 +32,6 @@ var _ = Suite(&MySuite{})
func Test(t *testing.T) { TestingT(t) }
func (s *MySuite) TestFileSplit(c *C) {
b := Split{}
err := b.GenChunks("TESTFILE", "1KB")
c.Assert(err, IsNil)
}
func (s *MySuite) TestSplitStream(c *C) {
var bytesBuffer bytes.Buffer
bytesWriter := bufio.NewWriter(&bytesBuffer)
@ -59,6 +53,8 @@ func (s *MySuite) TestSplitStream(c *C) {
}
func (s *MySuite) TestFileSplit2(c *C) {
err := SplitFilesWithPrefix("TESTFILE", 1024, "TESTPREFIX")
err := SplitFilesWithPrefix("TESTFILE", "1KB", "TESTPREFIX")
c.Assert(err, IsNil)
err = SplitFilesWithPrefix("TESTFILE", "1KB", "")
c.Assert(err, Not(IsNil))
}

Write Preview
Loading…
Cancel
Save