From 800b19d8e5a014d28f83fa5f7bd6147a770bd33a Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 19 Feb 2016 16:04:29 -0800 Subject: [PATCH] cleanup: Remove definitions and move them to its relative places accordingly - Move fs-definitions.go and break them into fs-datatypes.go, fs-bucket-acl.go and fs-utils.go - Move api-definitions.go to api-response.go, where they should be. - Move web-definitions to its related handlers. --- api-definitions.go | 225 ------------------ api-response.go | 230 +++++++++++++++++-- generic-handlers.go | 28 ++- pkg/fs/{config.go => fs-backend-metadata.go} | 2 +- pkg/fs/{acl.go => fs-bucket-acl.go} | 48 +++- pkg/fs/fs-bucket.go | 23 +- pkg/fs/fs-common.go | 94 -------- pkg/fs/{definitions.go => fs-datatypes.go} | 83 +------ pkg/fs/{errors.go => fs-errors.go} | 0 pkg/fs/fs-multipart.go | 7 + pkg/fs/fs-object.go | 2 +- pkg/fs/fs-utils.go | 55 +++++ pkg/probe/probe_test.go | 1 + web-definitions.go | 17 -- 14 files changed, 358 insertions(+), 457 deletions(-) delete mode 100644 api-definitions.go rename pkg/fs/{config.go => fs-backend-metadata.go} (97%) rename pkg/fs/{acl.go => fs-bucket-acl.go} (56%) delete mode 100644 pkg/fs/fs-common.go rename pkg/fs/{definitions.go => fs-datatypes.go} (60%) rename pkg/fs/{errors.go => fs-errors.go} (100%) create mode 100644 pkg/fs/fs-utils.go delete mode 100644 web-definitions.go diff --git a/api-definitions.go b/api-definitions.go deleted file mode 100644 index f5ccc0de5..000000000 --- a/api-definitions.go +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import "encoding/xml" - -// Limit number of objects in a given response. -const ( - maxObjectList = 1000 -) - -// LocationResponse - format for location response. -type LocationResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"` - Location string `xml:",chardata"` -} - -// AccessControlPolicyResponse - format for get bucket acl response. -type AccessControlPolicyResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"` - - AccessControlList struct { - Grant []Grant - } - Owner Owner -} - -// Grant container for grantee and permission. -type Grant struct { - Grantee struct { - ID string - DisplayName string - EmailAddress string - Type string - URI string - } - Permission string -} - -// ListObjectsResponse - format for list objects response. -type ListObjectsResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` - - CommonPrefixes []*CommonPrefix - Contents []*Object - - Delimiter string - - // Encoding type used to encode object keys in the response. - EncodingType string - - // A flag that indicates whether or not ListObjects returned all of the results - // that satisfied the search criteria. - IsTruncated bool - Marker string - MaxKeys int - Name string - - // When response is truncated (the IsTruncated element value in the response - // is true), you can use the key name in this field as marker in the subsequent - // request to get next set of objects. Server lists objects in alphabetical - // order Note: This element is returned only if you have delimiter request parameter - // specified. If response does not include the NextMaker and it is truncated, - // you can use the value of the last Key in the response as the marker in the - // subsequent request to get the next set of object keys. - NextMarker string - Prefix string -} - -// Part container for part metadata. -type Part struct { - PartNumber int - ETag string - LastModified string - Size int64 -} - -// ListPartsResponse - format for list parts response. -type ListPartsResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"` - - Bucket string - Key string - UploadID string `xml:"UploadId"` - - Initiator Initiator - Owner Owner - - // The class of storage used to store the object. - StorageClass string - - PartNumberMarker int - NextPartNumberMarker int - MaxParts int - IsTruncated bool - - // List of parts. - Part []*Part -} - -// ListMultipartUploadsResponse - format for list multipart uploads response. -type ListMultipartUploadsResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"` - - Bucket string - KeyMarker string - UploadIDMarker string `xml:"UploadIdMarker"` - NextKeyMarker string - NextUploadIDMarker string `xml:"NextUploadIdMarker"` - EncodingType string - MaxUploads int - IsTruncated bool - Upload []*Upload - Prefix string - Delimiter string - CommonPrefixes []*CommonPrefix -} - -// ListBucketsResponse - format for list buckets response -type ListBucketsResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"` - // Container for one or more buckets. - Buckets struct { - Bucket []*Bucket - } // Buckets are nested - Owner Owner -} - -// Upload container for in progress multipart upload -type Upload struct { - Key string - UploadID string `xml:"UploadId"` - Initiator Initiator - Owner Owner - StorageClass string - Initiated string -} - -// CommonPrefix container for prefix response in ListObjectsResponse -type CommonPrefix struct { - Prefix string -} - -// Bucket container for bucket metadata -type Bucket struct { - Name string - CreationDate string -} - -// Object container for object metadata -type Object struct { - ETag string - Key string - LastModified string - Size int64 - - Owner Owner - - // The class of storage used to store the object. - StorageClass string -} - -// Initiator inherit from Owner struct, fields are same -type Initiator Owner - -// Owner - bucket owner/principal -type Owner struct { - ID string - DisplayName string -} - -// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload -type InitiateMultipartUploadResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"` - - Bucket string - Key string - UploadID string `xml:"UploadId"` -} - -// CompleteMultipartUploadResponse container for completed multipart upload response -type CompleteMultipartUploadResponse struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"` - - Location string - Bucket string - Key string - ETag string -} - -// List of not implemented bucket queries -var notimplementedBucketResourceNames = map[string]bool{ - "policy": true, - "cors": true, - "lifecycle": true, - "logging": true, - "notification": true, - "replication": true, - "tagging": true, - "versions": true, - "requestPayment": true, - "versioning": true, - "website": true, -} - -// List of not implemented object queries -var notimplementedObjectResourceNames = map[string]bool{ - "torrent": true, - "acl": true, - "policy": true, -} diff --git a/api-response.go b/api-response.go index e033668da..c3cebc2a6 100644 --- a/api-response.go +++ b/api-response.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015 Minio, Inc. + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,198 @@ package main import ( + "encoding/xml" "net/http" "github.com/minio/minio/pkg/fs" ) -// Reply date format const ( - rfcFormat = "2006-01-02T15:04:05.000Z" + // Reply date format + timeFormatAMZ = "2006-01-02T15:04:05.000Z" + // Limit number of objects in a given response. + maxObjectList = 1000 ) +// LocationResponse - format for location response. +type LocationResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"` + Location string `xml:",chardata"` +} + +// AccessControlPolicyResponse - format for get bucket acl response. +type AccessControlPolicyResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"` + + AccessControlList struct { + Grants []Grant `xml:"Grant"` + } + Owner Owner +} + +// Grant container for grantee and permission. +type Grant struct { + Grantee struct { + ID string + DisplayName string + EmailAddress string + Type string + URI string + } + Permission string +} + +// ListObjectsResponse - format for list objects response. +type ListObjectsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + + CommonPrefixes []CommonPrefix + Contents []Object + + Delimiter string + + // Encoding type used to encode object keys in the response. + EncodingType string + + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + Marker string + MaxKeys int + Name string + + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Server lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + NextMarker string + Prefix string +} + +// Part container for part metadata. +type Part struct { + PartNumber int + ETag string + LastModified string + Size int64 +} + +// ListPartsResponse - format for list parts response. +type ListPartsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"` + + Bucket string + Key string + UploadID string `xml:"UploadId"` + + Initiator Initiator + Owner Owner + + // The class of storage used to store the object. + StorageClass string + + PartNumberMarker int + NextPartNumberMarker int + MaxParts int + IsTruncated bool + + // List of parts. + Parts []Part `xml:"Part"` +} + +// ListMultipartUploadsResponse - format for list multipart uploads response. +type ListMultipartUploadsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"` + + Bucket string + KeyMarker string + UploadIDMarker string `xml:"UploadIdMarker"` + NextKeyMarker string + NextUploadIDMarker string `xml:"NextUploadIdMarker"` + EncodingType string + MaxUploads int + IsTruncated bool + Uploads []Upload `xml:"Upload"` + Prefix string + Delimiter string + CommonPrefixes []CommonPrefix +} + +// ListBucketsResponse - format for list buckets response +type ListBucketsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"` + // Container for one or more buckets. + Buckets struct { + Buckets []Bucket `xml:"Bucket"` + } // Buckets are nested + Owner Owner +} + +// Upload container for in progress multipart upload +type Upload struct { + Key string + UploadID string `xml:"UploadId"` + Initiator Initiator + Owner Owner + StorageClass string + Initiated string +} + +// CommonPrefix container for prefix response in ListObjectsResponse +type CommonPrefix struct { + Prefix string +} + +// Bucket container for bucket metadata +type Bucket struct { + Name string + CreationDate string // time string of format "2006-01-02T15:04:05.000Z" +} + +// Object container for object metadata +type Object struct { + ETag string + Key string + LastModified string // time string of format "2006-01-02T15:04:05.000Z" + Size int64 + + Owner Owner + + // The class of storage used to store the object. + StorageClass string +} + +// Initiator inherit from Owner struct, fields are same +type Initiator Owner + +// Owner - bucket owner/principal +type Owner struct { + ID string + DisplayName string +} + +// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload +type InitiateMultipartUploadResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"` + + Bucket string + Key string + UploadID string `xml:"UploadId"` +} + +// CompleteMultipartUploadResponse container for completed multipart upload response +type CompleteMultipartUploadResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"` + + Location string + Bucket string + Key string + ETag string +} + // takes an array of Bucketmetadata information for serialization // input: // array of bucket metadata @@ -34,7 +216,7 @@ const ( // output: // populated struct that can be serialized to match xml and json api spec output func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsResponse { - var listbuckets []*Bucket + var listbuckets []Bucket var data = ListBucketsResponse{} var owner = Owner{} @@ -42,14 +224,14 @@ func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsRespons owner.DisplayName = "minio" for _, bucket := range buckets { - var listbucket = &Bucket{} + var listbucket = Bucket{} listbucket.Name = bucket.Name - listbucket.CreationDate = bucket.Created.Format(rfcFormat) + listbucket.CreationDate = bucket.Created.Format(timeFormatAMZ) listbuckets = append(listbuckets, listbucket) } data.Owner = owner - data.Buckets.Bucket = listbuckets + data.Buckets.Buckets = listbuckets return data } @@ -65,7 +247,7 @@ func generateAccessControlPolicyResponse(acl fs.BucketACL) AccessControlPolicyRe defaultGrant.Grantee.ID = "minio" defaultGrant.Grantee.DisplayName = "minio" defaultGrant.Permission = "FULL_CONTROL" - accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, defaultGrant) + accessCtrlPolicyResponse.AccessControlList.Grants = append(accessCtrlPolicyResponse.AccessControlList.Grants, defaultGrant) switch { case acl.IsPublicRead(): publicReadGrant := Grant{} @@ -73,7 +255,7 @@ func generateAccessControlPolicyResponse(acl fs.BucketACL) AccessControlPolicyRe publicReadGrant.Grantee.DisplayName = "minio" publicReadGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers" publicReadGrant.Permission = "READ" - accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadGrant) + accessCtrlPolicyResponse.AccessControlList.Grants = append(accessCtrlPolicyResponse.AccessControlList.Grants, publicReadGrant) case acl.IsPublicReadWrite(): publicReadGrant := Grant{} publicReadGrant.Grantee.ID = "minio" @@ -85,16 +267,16 @@ func generateAccessControlPolicyResponse(acl fs.BucketACL) AccessControlPolicyRe publicReadWriteGrant.Grantee.DisplayName = "minio" publicReadWriteGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers" publicReadWriteGrant.Permission = "WRITE" - accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadGrant) - accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadWriteGrant) + accessCtrlPolicyResponse.AccessControlList.Grants = append(accessCtrlPolicyResponse.AccessControlList.Grants, publicReadGrant) + accessCtrlPolicyResponse.AccessControlList.Grants = append(accessCtrlPolicyResponse.AccessControlList.Grants, publicReadWriteGrant) } return accessCtrlPolicyResponse } // generates an ListObjects response for the said bucket with other enumerated options. func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse { - var contents []*Object - var prefixes []*CommonPrefix + var contents []Object + var prefixes []CommonPrefix var owner = Owner{} var data = ListObjectsResponse{} @@ -102,12 +284,12 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe owner.DisplayName = "minio" for _, object := range resp.Objects { - var content = &Object{} + var content = Object{} if object.Object == "" { continue } content.Key = object.Object - content.LastModified = object.Created.Format(rfcFormat) + content.LastModified = object.Created.Format(timeFormatAMZ) if object.MD5 != "" { content.ETag = "\"" + object.MD5 + "\"" } @@ -128,7 +310,7 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe data.NextMarker = resp.NextMarker data.IsTruncated = resp.IsTruncated for _, prefix := range resp.Prefixes { - var prefixItem = &CommonPrefix{} + var prefixItem = CommonPrefix{} prefixItem.Prefix = prefix prefixes = append(prefixes, prefixItem) } @@ -173,14 +355,14 @@ func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPa listPartsResponse.IsTruncated = objectMetadata.IsTruncated listPartsResponse.NextPartNumberMarker = objectMetadata.NextPartNumberMarker - listPartsResponse.Part = make([]*Part, len(objectMetadata.Part)) + listPartsResponse.Parts = make([]Part, len(objectMetadata.Part)) for _, part := range objectMetadata.Part { - newPart := &Part{} + newPart := Part{} newPart.PartNumber = part.PartNumber newPart.ETag = "\"" + part.ETag + "\"" newPart.Size = part.Size - newPart.LastModified = part.LastModified.Format(rfcFormat) - listPartsResponse.Part = append(listPartsResponse.Part, newPart) + newPart.LastModified = part.LastModified.Format(timeFormatAMZ) + listPartsResponse.Parts = append(listPartsResponse.Parts, newPart) } return listPartsResponse } @@ -199,13 +381,13 @@ func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMulti listMultipartUploadsResponse.NextUploadIDMarker = metadata.NextUploadIDMarker listMultipartUploadsResponse.UploadIDMarker = metadata.UploadIDMarker - listMultipartUploadsResponse.Upload = make([]*Upload, len(metadata.Upload)) + listMultipartUploadsResponse.Uploads = make([]Upload, len(metadata.Upload)) for _, upload := range metadata.Upload { - newUpload := &Upload{} + newUpload := Upload{} newUpload.UploadID = upload.UploadID newUpload.Key = upload.Object - newUpload.Initiated = upload.Initiated.Format(rfcFormat) - listMultipartUploadsResponse.Upload = append(listMultipartUploadsResponse.Upload, newUpload) + newUpload.Initiated = upload.Initiated.Format(timeFormatAMZ) + listMultipartUploadsResponse.Uploads = append(listMultipartUploadsResponse.Uploads, newUpload) } return listMultipartUploadsResponse } diff --git a/generic-handlers.go b/generic-handlers.go index 8d8f96d84..60a65ec60 100644 --- a/generic-handlers.go +++ b/generic-handlers.go @@ -248,8 +248,7 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Checks requests for not implemented Bucket resources func ignoreNotImplementedBucketResources(req *http.Request) bool { - q := req.URL.Query() - for name := range q { + for name := range req.URL.Query() { if notimplementedBucketResourceNames[name] { return true } @@ -259,11 +258,32 @@ func ignoreNotImplementedBucketResources(req *http.Request) bool { // Checks requests for not implemented Object resources func ignoreNotImplementedObjectResources(req *http.Request) bool { - q := req.URL.Query() - for name := range q { + for name := range req.URL.Query() { if notimplementedObjectResourceNames[name] { return true } } return false } + +// List of not implemented bucket queries +var notimplementedBucketResourceNames = map[string]bool{ + "policy": true, + "cors": true, + "lifecycle": true, + "logging": true, + "notification": true, + "replication": true, + "tagging": true, + "versions": true, + "requestPayment": true, + "versioning": true, + "website": true, +} + +// List of not implemented object queries +var notimplementedObjectResourceNames = map[string]bool{ + "torrent": true, + "acl": true, + "policy": true, +} diff --git a/pkg/fs/config.go b/pkg/fs/fs-backend-metadata.go similarity index 97% rename from pkg/fs/config.go rename to pkg/fs/fs-backend-metadata.go index 8c6440043..6f28965d9 100644 --- a/pkg/fs/config.go +++ b/pkg/fs/fs-backend-metadata.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015 Minio, Inc. + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/fs/acl.go b/pkg/fs/fs-bucket-acl.go similarity index 56% rename from pkg/fs/acl.go rename to pkg/fs/fs-bucket-acl.go index 85e63dfaf..46a61020e 100644 --- a/pkg/fs/acl.go +++ b/pkg/fs/fs-bucket-acl.go @@ -1,5 +1,5 @@ /* - * Minio Cloud Storage, (C) 2015 Minio, Inc. + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,3 +48,49 @@ func (fs Filesystem) IsReadOnlyBucket(bucket string) bool { } return bucketMetadata.ACL.IsPublicRead() } + +// BucketACL - bucket level access control +type BucketACL string + +// different types of ACL's currently supported for buckets +const ( + BucketPrivate = BucketACL("private") + BucketPublicRead = BucketACL("public-read") + BucketPublicReadWrite = BucketACL("public-read-write") +) + +func (b BucketACL) String() string { + return string(b) +} + +// IsPrivate - is acl Private +func (b BucketACL) IsPrivate() bool { + return b == BucketACL("private") +} + +// IsPublicRead - is acl PublicRead +func (b BucketACL) IsPublicRead() bool { + return b == BucketACL("public-read") +} + +// IsPublicReadWrite - is acl PublicReadWrite +func (b BucketACL) IsPublicReadWrite() bool { + return b == BucketACL("public-read-write") +} + +// IsValidBucketACL - is provided acl string supported +func IsValidBucketACL(acl string) bool { + switch acl { + case "private": + fallthrough + case "public-read": + fallthrough + case "public-read-write": + return true + case "": + // by default its "private" + return true + default: + return false + } +} diff --git a/pkg/fs/fs-bucket.go b/pkg/fs/fs-bucket.go index 45720d738..f366e286d 100644 --- a/pkg/fs/fs-bucket.go +++ b/pkg/fs/fs-bucket.go @@ -96,16 +96,23 @@ func (fs Filesystem) ListBuckets() ([]BucketMetadata, *probe.Error) { } // removeDuplicateBuckets - remove duplicate buckets. -func removeDuplicateBuckets(elements []BucketMetadata) (result []BucketMetadata) { - // Use map to record duplicates as we find them. - duplicates := make(map[string]struct{}) - for _, element := range elements { - if _, ok := duplicates[element.Name]; !ok { - duplicates[element.Name] = struct{}{} - result = append(result, element) +func removeDuplicateBuckets(buckets []BucketMetadata) []BucketMetadata { + length := len(buckets) - 1 + for i := 0; i < length; i++ { + for j := i + 1; j <= length; j++ { + if buckets[i].Name == buckets[j].Name { + if buckets[i].Created.Sub(buckets[j].Created) > 0 { + buckets[i] = buckets[length] + } else { + buckets[j] = buckets[length] + } + buckets = buckets[0:length] + length-- + j-- + } } } - return result + return buckets } // MakeBucket - PUT Bucket. diff --git a/pkg/fs/fs-common.go b/pkg/fs/fs-common.go deleted file mode 100644 index 71d2d53a1..000000000 --- a/pkg/fs/fs-common.go +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package fs - -import ( - "bufio" - "bytes" - "os" - "sort" - "strings" - "time" -) - -// Metadata - carries metadata about object -type Metadata struct { - MD5sum []byte - ContentType string -} - -// sanitizeWindowsPath - sanitize a path -func sanitizeWindowsPath(path string) string { - return strings.Replace(path, "\\", "/", -1) -} - -// sanitizeWindowsPaths - sanitize some windows paths -func sanitizeWindowsPaths(paths ...string) []string { - var results []string - for _, path := range paths { - results = append(results, sanitizeWindowsPath(path)) - } - return results -} - -// sortUnique returns n, the number of distinct elements in data in sorted order. -func sortUnique(data sort.Interface) (n int) { - if n = data.Len(); n < 2 { - return n - } - sort.Sort(data) - a, b := 0, 1 - for b < n { - if data.Less(a, b) { - a++ - if a != b { - data.Swap(a, b) - } - } - b++ - } - return a + 1 -} - -type contentInfo struct { - os.FileInfo - Prefix string - Size int64 - Mode os.FileMode - ModTime time.Time -} - -type bucketDir struct { - files []contentInfo - root string -} - -func delimiter(object, delimiter string) string { - readBuffer := bytes.NewBufferString(object) - reader := bufio.NewReader(readBuffer) - stringReader := strings.NewReader(delimiter) - delimited, _ := stringReader.ReadByte() - delimitedStr, _ := reader.ReadString(delimited) - return delimitedStr -} - -// byObjectMetadataKey is a sortable interface for UploadMetadata slice -type byUploadMetadataKey []*UploadMetadata - -func (b byUploadMetadataKey) Len() int { return len(b) } -func (b byUploadMetadataKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } -func (b byUploadMetadataKey) Less(i, j int) bool { return b[i].Object < b[j].Object } diff --git a/pkg/fs/definitions.go b/pkg/fs/fs-datatypes.go similarity index 60% rename from pkg/fs/definitions.go rename to pkg/fs/fs-datatypes.go index 69fd681b7..c26e0bc71 100644 --- a/pkg/fs/definitions.go +++ b/pkg/fs/fs-datatypes.go @@ -1,5 +1,5 @@ /* - * Minimalist Object Storage, (C) 2015 Minio, Inc. + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,40 +18,9 @@ package fs import ( "os" - "regexp" "time" - "unicode/utf8" ) -// BucketACL - bucket level access control -type BucketACL string - -// different types of ACL's currently supported for buckets -const ( - BucketPrivate = BucketACL("private") - BucketPublicRead = BucketACL("public-read") - BucketPublicReadWrite = BucketACL("public-read-write") -) - -func (b BucketACL) String() string { - return string(b) -} - -// IsPrivate - is acl Private -func (b BucketACL) IsPrivate() bool { - return b == BucketACL("private") -} - -// IsPublicRead - is acl PublicRead -func (b BucketACL) IsPublicRead() bool { - return b == BucketACL("public-read") -} - -// IsPublicReadWrite - is acl PublicReadWrite -func (b BucketACL) IsPublicReadWrite() bool { - return b == BucketACL("public-read-write") -} - // BucketMetadata - name and create date type BucketMetadata struct { Name string @@ -142,53 +111,3 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part type CompleteMultipartUpload struct { Part []CompletePart } - -// IsValidBucketACL - is provided acl string supported -func IsValidBucketACL(acl string) bool { - switch acl { - case "private": - fallthrough - case "public-read": - fallthrough - case "public-read-write": - return true - case "": - // by default its "private" - return true - default: - return false - } -} - -// validBucket regexp. -var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`) - -// IsValidBucketName - verify bucket name in accordance with -// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html -func IsValidBucketName(bucket string) bool { - if bucket == "" { - return false - } - if len(bucket) < 3 || len(bucket) > 63 { - return false - } - if bucket[0] == '.' || bucket[len(bucket)-1] == '.' { - return false - } - return validBucket.MatchString(bucket) -} - -// IsValidObjectName - verify object name in accordance with -// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html -func IsValidObjectName(object string) bool { - if object == "" { - return true - } - if len(object) > 1024 || len(object) == 0 { - return false - } - if !utf8.ValidString(object) { - return false - } - return true -} diff --git a/pkg/fs/errors.go b/pkg/fs/fs-errors.go similarity index 100% rename from pkg/fs/errors.go rename to pkg/fs/fs-errors.go diff --git a/pkg/fs/fs-multipart.go b/pkg/fs/fs-multipart.go index a7f51631b..0d3db0ba4 100644 --- a/pkg/fs/fs-multipart.go +++ b/pkg/fs/fs-multipart.go @@ -53,6 +53,13 @@ func (fs Filesystem) isValidUploadID(object, uploadID string) (ok bool) { return } +// byObjectMetadataKey is a sortable interface for UploadMetadata slice +type byUploadMetadataKey []*UploadMetadata + +func (b byUploadMetadataKey) Len() int { return len(b) } +func (b byUploadMetadataKey) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byUploadMetadataKey) Less(i, j int) bool { return b[i].Object < b[j].Object } + // ListMultipartUploads - list incomplete multipart sessions for a given BucketMultipartResourcesMetadata func (fs Filesystem) ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error) { // Input validation. diff --git a/pkg/fs/fs-object.go b/pkg/fs/fs-object.go index bb9cc123b..5e5d6ae61 100644 --- a/pkg/fs/fs-object.go +++ b/pkg/fs/fs-object.go @@ -155,7 +155,7 @@ func getMetadata(rootPath, bucket, object string) (ObjectMetadata, *probe.Error) } contentType := "application/octet-stream" if runtime.GOOS == "windows" { - object = sanitizeWindowsPath(object) + object = filepath.ToSlash(object) } if objectExt := filepath.Ext(object); objectExt != "" { diff --git a/pkg/fs/fs-utils.go b/pkg/fs/fs-utils.go new file mode 100644 index 000000000..309431592 --- /dev/null +++ b/pkg/fs/fs-utils.go @@ -0,0 +1,55 @@ +/* + * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs + +import ( + "regexp" + "unicode/utf8" +) + +// validBucket regexp. +var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`) + +// IsValidBucketName - verify bucket name in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html +func IsValidBucketName(bucket string) bool { + if bucket == "" { + return false + } + if len(bucket) < 3 || len(bucket) > 63 { + return false + } + if bucket[0] == '.' || bucket[len(bucket)-1] == '.' { + return false + } + return validBucket.MatchString(bucket) +} + +// IsValidObjectName - verify object name in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +func IsValidObjectName(object string) bool { + if object == "" { + return true + } + if len(object) > 1024 || len(object) == 0 { + return false + } + if !utf8.ValidString(object) { + return false + } + return true +} diff --git a/pkg/probe/probe_test.go b/pkg/probe/probe_test.go index 6f064843a..9c215412d 100644 --- a/pkg/probe/probe_test.go +++ b/pkg/probe/probe_test.go @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package probe_test import ( diff --git a/web-definitions.go b/web-definitions.go deleted file mode 100644 index 4e7aad742..000000000 --- a/web-definitions.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main