From 3622fbc87daaac4ec9215653f392351429a11886 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 6 Jul 2015 21:54:00 -0700 Subject: [PATCH] Across donut, split, nimble some code cleanup --- commands.go | 16 +++++ pkg/donut/.gitignore | 2 - pkg/donut/acl.go | 47 +++++++++++++++ pkg/donut/bucket.go | 4 +- pkg/donut/common.go | 42 +++++++++++++ pkg/donut/date.go | 78 ------------------------ pkg/donut/donut-v2.go | 4 +- pkg/donut/split/split.go | 27 ++++----- pkg/donut/split/split_test.go | 8 +-- pkg/donut/utils.go | 109 ---------------------------------- pkg/server/nimble/http.go | 16 +++++ pkg/server/nimble/net.go | 16 +++++ 12 files changed, 158 insertions(+), 211 deletions(-) delete mode 100644 pkg/donut/.gitignore create mode 100644 pkg/donut/acl.go delete mode 100644 pkg/donut/date.go delete mode 100644 pkg/donut/utils.go diff --git a/commands.go b/commands.go index da07fb6b5..743f1f951 100644 --- a/commands.go +++ b/commands.go @@ -1,3 +1,19 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package main import ( diff --git a/pkg/donut/.gitignore b/pkg/donut/.gitignore deleted file mode 100644 index 144114ecd..000000000 --- a/pkg/donut/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -donut -build-constants.go \ No newline at end of file diff --git a/pkg/donut/acl.go b/pkg/donut/acl.go new file mode 100644 index 000000000..cc62cae9b --- /dev/null +++ b/pkg/donut/acl.go @@ -0,0 +1,47 @@ +package donut + +// BucketACL - bucket level access control +type BucketACL string + +// different types of ACL's currently supported for buckets +const ( + BucketPrivate = BucketACL("private") + BucketPublicRead = BucketACL("public-read") + BucketPublicReadWrite = BucketACL("public-read-write") +) + +func (b BucketACL) String() string { + return string(b) +} + +// IsPrivate - is acl Private +func (b BucketACL) IsPrivate() bool { + return b == BucketACL("private") +} + +// IsPublicRead - is acl PublicRead +func (b BucketACL) IsPublicRead() bool { + return b == BucketACL("public-read") +} + +// IsPublicReadWrite - is acl PublicReadWrite +func (b BucketACL) IsPublicReadWrite() bool { + return b == BucketACL("public-read-write") +} + +// IsValidBucketACL - is provided acl string supported +func IsValidBucketACL(acl string) bool { + switch acl { + case "private": + fallthrough + case "public-read": + fallthrough + case "public-read-write": + return true + case "": + // by default its "private" + return true + default: + return false + } +} diff --git a/pkg/donut/bucket.go b/pkg/donut/bucket.go index afb30505e..7c5113779 100644 --- a/pkg/donut/bucket.go +++ b/pkg/donut/bucket.go @@ -444,7 +444,7 @@ func (b bucket) readObjectData(objectName string, writer *io.PipeWriter, objMeta writer.CloseWithError(iodine.New(err, nil)) return } - _, err = io.Copy(mwriter, bytes.NewBuffer(decodedData)) + _, err = io.Copy(mwriter, bytes.NewReader(decodedData)) if err != nil { writer.CloseWithError(iodine.New(err, nil)) return @@ -473,7 +473,7 @@ func (b bucket) decodeEncodedData(totalLeft, blockSize int64, readers []io.ReadC if blockSize < totalLeft { curBlockSize = blockSize } else { - curBlockSize = totalLeft // cast is safe, blockSize in if protects + curBlockSize = totalLeft } curChunkSize, err := encoder.GetEncodedBlockLen(int(curBlockSize)) if err != nil { diff --git a/pkg/donut/common.go b/pkg/donut/common.go index b91fd3b1b..9f71665de 100644 --- a/pkg/donut/common.go +++ b/pkg/donut/common.go @@ -20,10 +20,52 @@ import ( "bufio" "bytes" "io" + "regexp" "sort" "strings" + "unicode/utf8" ) +// IsValidBucket - verify bucket name in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html +func IsValidBucket(bucket string) bool { + if len(bucket) < 3 || len(bucket) > 63 { + return false + } + if bucket[0] == '.' || bucket[len(bucket)-1] == '.' { + return false + } + if match, _ := regexp.MatchString("\\.\\.", bucket); match == true { + return false + } + // We don't support buckets with '.' in them + match, _ := regexp.MatchString("^[a-zA-Z][a-zA-Z0-9\\-]+[a-zA-Z0-9]$", bucket) + return match +} + +// IsValidObjectName - verify object name in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +func IsValidObjectName(object string) bool { + if strings.TrimSpace(object) == "" { + return false + } + if len(object) > 1024 || len(object) == 0 { + return false + } + if !utf8.ValidString(object) { + return false + } + return true +} + +// IsValidPrefix - verify prefix name is correct, an empty prefix is valid +func IsValidPrefix(prefix string) bool { + if strings.TrimSpace(prefix) == "" { + return true + } + return IsValidObjectName(prefix) +} + // ProxyWriter implements io.Writer to trap written bytes type ProxyWriter struct { writer io.Writer diff --git a/pkg/donut/date.go b/pkg/donut/date.go deleted file mode 100644 index e91ff5aff..000000000 --- a/pkg/donut/date.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package donut - -import ( - "errors" - "fmt" - "strconv" - "strings" -) - -// Date - [0000-00-00] -type Date struct { - Year int16 - Month byte - Day byte -} - -// String output in yyyy-mm-dd format -func (d Date) String() string { - return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day) -} - -// IsZero true if date is 0000-00-00 -func (d Date) IsZero() bool { - return d.Day == 0 && d.Month == 0 && d.Year == 0 -} - -// Convert string date in format YYYY-MM-DD to Date. -// Leading and trailing spaces are ignored. If format is invalid returns zero. -func parseDate(str string) (d Date, err error) { - str = strings.TrimSpace(str) - if str == "0000-00-00" { - return - } - var ( - y, m, n int - ) - if len(str) != 10 || str[4] != '-' || str[7] != '-' { - err = errors.New("Invalid 0000-00-000 style DATE string: " + str) - return - } - if y, err = strconv.Atoi(str[0:4]); err != nil { - return - } - if m, err = strconv.Atoi(str[5:7]); err != nil { - return - } - if m < 1 || m > 12 { - err = errors.New("Invalid 0000-00-000 style DATE string: " + str) - return - } - if n, err = strconv.Atoi(str[8:10]); err != nil { - return - } - if n < 1 || n > 31 { - err = errors.New("Invalid 0000-00-000 style DATE string: " + str) - return - } - d.Year = int16(y) - d.Month = byte(m) - d.Day = byte(n) - return -} diff --git a/pkg/donut/donut-v2.go b/pkg/donut/donut-v2.go index a9687a4a9..655057ab3 100644 --- a/pkg/donut/donut-v2.go +++ b/pkg/donut/donut-v2.go @@ -152,7 +152,7 @@ func (donut API) GetObject(w io.Writer, bucket string, object string) (int64, er return 0, iodine.New(err, nil) } /// cache object read from disk - ok := donut.objects.Set(objectKey, pw.writtenBytes) + ok := donut.objects.Append(objectKey, pw.writtenBytes) pw.writtenBytes = nil go debug.FreeOSMemory() if !ok { @@ -208,7 +208,7 @@ func (donut API) GetPartialObject(w io.Writer, bucket, object string, start, len if err != nil { return 0, iodine.New(err, nil) } - ok := donut.objects.Set(objectKey, pw.writtenBytes) + ok := donut.objects.Append(objectKey, pw.writtenBytes) pw.writtenBytes = nil go debug.FreeOSMemory() if !ok { diff --git a/pkg/donut/split/split.go b/pkg/donut/split/split.go index ba83d3240..40ef19232 100644 --- a/pkg/donut/split/split.go +++ b/pkg/donut/split/split.go @@ -25,6 +25,8 @@ import ( "os" "strconv" "strings" + + "github.com/minio/minio/pkg/iodine" ) // Message - message structure for results from the Stream goroutine @@ -52,7 +54,9 @@ func Stream(reader io.Reader, chunkSize uint64) <-chan Message { return ch } -func splitStreamGoRoutine(reader io.Reader, chunkSize uint64, ch chan Message) { +func splitStreamGoRoutine(reader io.Reader, chunkSize uint64, ch chan<- Message) { + defer close(ch) + // we read until EOF or another error var readError error @@ -88,8 +92,6 @@ func splitStreamGoRoutine(reader io.Reader, chunkSize uint64, ch chan Message) { if readError != io.EOF { ch <- Message{nil, readError} } - // close the channel, signaling the channel reader that the stream is complete - close(ch) } // JoinFiles reads from a given directory, joins data in chunks with prefix and sends @@ -103,12 +105,12 @@ func splitStreamGoRoutine(reader io.Reader, chunkSize uint64, ch chan Message) { // fmt.Println(buf) // } // -func JoinFiles(dirname string, inputPrefix string) (io.Reader, error) { +func JoinFiles(dirname string, inputPrefix string) io.Reader { reader, writer := io.Pipe() fileInfos, readError := ioutil.ReadDir(dirname) if readError != nil { writer.CloseWithError(readError) - return nil, readError + return nil } var newfileInfos []os.FileInfo @@ -119,16 +121,16 @@ func JoinFiles(dirname string, inputPrefix string) (io.Reader, error) { } if len(newfileInfos) == 0 { - nofilesError := errors.New("no files found for given prefix " + inputPrefix) + nofilesError := iodine.New(errors.New("no files found for given prefix "+inputPrefix), nil) writer.CloseWithError(nofilesError) - return nil, nofilesError + return nil } - go joinFilesGoRoutine(newfileInfos, writer) - return reader, nil + go joinFilesInGoRoutine(newfileInfos, writer) + return reader } -func joinFilesGoRoutine(fileInfos []os.FileInfo, writer *io.PipeWriter) { +func joinFilesInGoRoutine(fileInfos []os.FileInfo, writer *io.PipeWriter) { for _, fileInfo := range fileInfos { file, err := os.Open(fileInfo.Name()) defer file.Close() @@ -159,14 +161,11 @@ func FileWithPrefix(filename string, chunkSize uint64, outputPrefix string) erro return errors.New("Invalid argument outputPrefix cannot be empty string") } - // start stream splitting goroutine - ch := Stream(file, chunkSize) - // used to write each chunk out as a separate file. {{outputPrefix}}.{{i}} i := 0 // write each chunk out to a separate file - for chunk := range ch { + for chunk := range Stream(file, chunkSize) { if chunk.Err != nil { return chunk.Err } diff --git a/pkg/donut/split/split_test.go b/pkg/donut/split/split_test.go index 569f6d02a..94413a988 100644 --- a/pkg/donut/split/split_test.go +++ b/pkg/donut/split/split_test.go @@ -62,11 +62,11 @@ func (s *MySuite) TestFileSplitJoin(c *C) { defer devnull.Close() var reader io.Reader - reader, err = split.JoinFiles(".", "ERROR") - c.Assert(err, Not(IsNil)) + reader = split.JoinFiles(".", "ERROR") + c.Assert(reader, IsNil) - reader, err = split.JoinFiles(".", "TESTPREFIX") - c.Assert(err, IsNil) + reader = split.JoinFiles(".", "TESTPREFIX") + c.Assert(reader, Not(IsNil)) _, err = io.Copy(devnull, reader) c.Assert(err, IsNil) } diff --git a/pkg/donut/utils.go b/pkg/donut/utils.go deleted file mode 100644 index 3d8e9403f..000000000 --- a/pkg/donut/utils.go +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package donut - -import ( - "regexp" - "strings" - "unicode/utf8" -) - -// BucketACL - bucket level access control -type BucketACL string - -// different types of ACL's currently supported for buckets -const ( - BucketPrivate = BucketACL("private") - BucketPublicRead = BucketACL("public-read") - BucketPublicReadWrite = BucketACL("public-read-write") -) - -func (b BucketACL) String() string { - return string(b) -} - -// IsPrivate - is acl Private -func (b BucketACL) IsPrivate() bool { - return b == BucketACL("private") -} - -// IsPublicRead - is acl PublicRead -func (b BucketACL) IsPublicRead() bool { - return b == BucketACL("public-read") -} - -// IsPublicReadWrite - is acl PublicReadWrite -func (b BucketACL) IsPublicReadWrite() bool { - return b == BucketACL("public-read-write") -} - -// IsValidBucketACL - is provided acl string supported -func IsValidBucketACL(acl string) bool { - switch acl { - case "private": - fallthrough - case "public-read": - fallthrough - case "public-read-write": - return true - case "": - // by default its "private" - return true - default: - return false - } -} - -// IsValidBucket - verify bucket name in accordance with -// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html -func IsValidBucket(bucket string) bool { - if len(bucket) < 3 || len(bucket) > 63 { - return false - } - if bucket[0] == '.' || bucket[len(bucket)-1] == '.' { - return false - } - if match, _ := regexp.MatchString("\\.\\.", bucket); match == true { - return false - } - // We don't support buckets with '.' in them - match, _ := regexp.MatchString("^[a-zA-Z][a-zA-Z0-9\\-]+[a-zA-Z0-9]$", bucket) - return match -} - -// IsValidObjectName - verify object name in accordance with -// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html -func IsValidObjectName(object string) bool { - if strings.TrimSpace(object) == "" { - return false - } - if len(object) > 1024 || len(object) == 0 { - return false - } - if !utf8.ValidString(object) { - return false - } - return true -} - -// IsValidPrefix - verify prefix name is correct, an empty prefix is valid -func IsValidPrefix(prefix string) bool { - if strings.TrimSpace(prefix) == "" { - return true - } - return IsValidObjectName(prefix) -} diff --git a/pkg/server/nimble/http.go b/pkg/server/nimble/http.go index 4caa2fabe..98b7660e7 100644 --- a/pkg/server/nimble/http.go +++ b/pkg/server/nimble/http.go @@ -1,3 +1,19 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // Package nimble provides easy to use graceful restart for a set of HTTP services // // This package originally from https://github.com/facebookgo/grace diff --git a/pkg/server/nimble/net.go b/pkg/server/nimble/net.go index eeaabae0d..89d595306 100644 --- a/pkg/server/nimble/net.go +++ b/pkg/server/nimble/net.go @@ -1,3 +1,19 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package nimble import (