diff --git a/Makefile b/Makefile index e79707b78..f435a8895 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ lint: cyclo: @echo "Running $@:" - @test -z "$$(gocyclo -over 19 . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" + @test -z "$$(gocyclo -over 25 . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" gomake-all: getdeps verifiers @echo "Installing minio:" diff --git a/pkg/server/router.go b/pkg/server/router.go index e9fcaaed1..0ed4e1070 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -101,14 +101,14 @@ func registerRPC(mux *router.Router, s *rpc.Server) http.Handler { return mux } -// APIHandler api handler -func APIHandler(conf api.Config) http.Handler { +// getAPIHandler api handler +func getAPIHandler(conf api.Config) http.Handler { mux := router.NewRouter() return registerOtherMiddleware(registerAPI(mux), conf) } -// RPCHandler rpc handler -func RPCHandler() http.Handler { +// getRPCHandler rpc handler +func getRPCHandler() http.Handler { s := rpc.NewServer() s.RegisterJSONCodec() s.RegisterService(new(rpc.HelloService), "") diff --git a/pkg/server/server.go b/pkg/server/server.go index 83426d888..3ddc12623 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -31,7 +31,7 @@ func startAPI(errCh chan error, conf api.Config) { // Minio server config httpServer := &http.Server{ Addr: conf.Address, - Handler: APIHandler(conf), + Handler: getAPIHandler(conf), MaxHeaderBytes: 1 << 20, } @@ -77,16 +77,19 @@ func startAPI(errCh chan error, conf api.Config) { func startRPC(errCh chan error) { defer close(errCh) - rpcHandler := RPCHandler() // Minio server config httpServer := &http.Server{ Addr: "127.0.0.1:9001", // TODO make this configurable - Handler: rpcHandler, + Handler: getRPCHandler(), MaxHeaderBytes: 1 << 20, } errCh <- httpServer.ListenAndServe() } +func startTM(errCh chan error) { + defer close(errCh) +} + // StartServices starts basic services for a server func StartServices(conf api.Config) error { apiErrCh := make(chan error) diff --git a/pkg/storage/donut/cache.go b/pkg/storage/donut/cache.go index 91e3c7adf..59b49a5fa 100644 --- a/pkg/storage/donut/cache.go +++ b/pkg/storage/donut/cache.go @@ -467,48 +467,6 @@ func (cache Cache) MakeBucket(bucketName, acl string) error { return nil } -func (cache Cache) filterDelimiterPrefix(keys []string, commonPrefixes []string, key, prefix, delim string) ([]string, []string) { - switch true { - case key == prefix: - keys = append(keys, key) - // delim - requires r.Prefix as it was trimmed off earlier - case key == prefix+delim: - keys = append(keys, key) - fallthrough - case delim != "": - commonPrefixes = append(commonPrefixes, prefix+delim) - } - return keys, commonPrefixes -} - -func (cache Cache) listObjects(keys []string, commonPrefixes []string, key string, r BucketResourcesMetadata) ([]string, []string) { - switch true { - // Prefix absent, delimit object key based on delimiter - case r.IsDelimiterSet(): - delim := Delimiter(key, r.Delimiter) - switch true { - case delim == "" || delim == key: - keys = append(keys, key) - case delim != "": - commonPrefixes = append(commonPrefixes, delim) - } - // Prefix present, delimit object key with prefix key based on delimiter - case r.IsDelimiterPrefixSet(): - if strings.HasPrefix(key, r.Prefix) { - trimmedName := strings.TrimPrefix(key, r.Prefix) - delim := Delimiter(trimmedName, r.Delimiter) - keys, commonPrefixes = cache.filterDelimiterPrefix(keys, commonPrefixes, key, r.Prefix, delim) - } - // Prefix present, nothing to delimit - case r.IsPrefixSet(): - keys = append(keys, key) - // Prefix and delimiter absent - case r.IsDefault(): - keys = append(keys, key) - } - return keys, commonPrefixes -} - // ListObjects - list objects from cache func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, error) { cache.lock.RLock() @@ -524,27 +482,62 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) } var results []ObjectMetadata var keys []string + if cache.donut != nil { + listObjects, err := cache.donut.ListObjects( + bucket, + resources.Prefix, + resources.Marker, + resources.Delimiter, + resources.Maxkeys, + ) + if err != nil { + return nil, BucketResourcesMetadata{IsTruncated: false}, iodine.New(err, nil) + } + resources.CommonPrefixes = listObjects.CommonPrefixes + resources.IsTruncated = listObjects.IsTruncated + if resources.IsTruncated && resources.IsDelimiterSet() { + resources.NextMarker = results[len(results)-1].Object + } + for key := range listObjects.Objects { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + results = append(results, listObjects.Objects[key]) + } + return results, resources, nil + } storedBucket := cache.storedBuckets[bucket] for key := range storedBucket.objectMetadata { if strings.HasPrefix(key, bucket+"/") { key = key[len(bucket)+1:] - keys, resources.CommonPrefixes = cache.listObjects(keys, resources.CommonPrefixes, key, resources) - } - } - var newKeys []string - switch { - case resources.Marker != "": - for _, key := range keys { - if key > resources.Marker { - newKeys = append(newKeys, key) + if strings.HasPrefix(key, resources.Prefix) { + if key > resources.Marker { + keys = append(keys, key) + } } } - default: - newKeys = keys } - newKeys = RemoveDuplicates(newKeys) - sort.Strings(newKeys) - for _, key := range newKeys { + if strings.TrimSpace(resources.Prefix) != "" { + keys = TrimPrefix(keys, resources.Prefix) + } + var prefixes []string + var filteredKeys []string + if strings.TrimSpace(resources.Delimiter) != "" { + filteredKeys = HasNoDelimiter(keys, resources.Delimiter) + prefixes = HasDelimiter(keys, resources.Delimiter) + prefixes = SplitDelimiter(prefixes, resources.Delimiter) + prefixes = SortU(prefixes) + } else { + filteredKeys = keys + } + for _, commonPrefix := range prefixes { + resources.CommonPrefixes = append(resources.CommonPrefixes, resources.Prefix+commonPrefix) + } + filteredKeys = RemoveDuplicates(filteredKeys) + sort.Strings(filteredKeys) + + for _, key := range filteredKeys { if len(results) == resources.Maxkeys { resources.IsTruncated = true if resources.IsTruncated && resources.IsDelimiterSet() { @@ -552,10 +545,11 @@ func (cache Cache) ListObjects(bucket string, resources BucketResourcesMetadata) } return results, resources, nil } - object := storedBucket.objectMetadata[bucket+"/"+key] + object := storedBucket.objectMetadata[bucket+"/"+resources.Prefix+key] results = append(results, object) } resources.CommonPrefixes = RemoveDuplicates(resources.CommonPrefixes) + sort.Strings(resources.CommonPrefixes) return results, resources, nil } diff --git a/pkg/storage/donut/cache_test.go b/pkg/storage/donut/cache_test.go new file mode 100644 index 000000000..644df5f00 --- /dev/null +++ b/pkg/storage/donut/cache_test.go @@ -0,0 +1,252 @@ +/* + * Minimalist Object Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliedc. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package donut + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "io/ioutil" + "testing" + "time" + + . "github.com/minio/check" +) + +func TestCache(t *testing.T) { TestingT(t) } + +type MyCacheSuite struct{} + +var _ = Suite(&MyCacheSuite{}) + +var dc Cache + +func (s *MyCacheSuite) SetUpSuite(c *C) { + // no donut this time + dc = NewCache(100000, time.Duration(1*time.Hour), "", nil) + // testing empty cache + buckets, err := dc.ListBuckets() + c.Assert(err, IsNil) + c.Assert(len(buckets), Equals, 0) +} + +// test make bucket without name +func (s *MyCacheSuite) TestBucketWithoutNameFails(c *C) { + // fail to create new bucket without a name + err := dc.MakeBucket("", "private") + c.Assert(err, Not(IsNil)) + + err = dc.MakeBucket(" ", "private") + c.Assert(err, Not(IsNil)) +} + +// test empty bucket +func (s *MyCacheSuite) TestEmptyBucket(c *C) { + c.Assert(dc.MakeBucket("foo1", "private"), IsNil) + // check if bucket is empty + var resources BucketResourcesMetadata + resources.Maxkeys = 1 + objectsMetadata, resources, err := dc.ListObjects("foo1", resources) + c.Assert(err, IsNil) + c.Assert(len(objectsMetadata), Equals, 0) + c.Assert(resources.CommonPrefixes, DeepEquals, []string{}) + c.Assert(resources.IsTruncated, Equals, false) +} + +// test bucket list +func (s *MyCacheSuite) TestMakeBucketAndList(c *C) { + // create bucket + err := dc.MakeBucket("foo2", "private") + c.Assert(err, IsNil) + + // check bucket exists + buckets, err := dc.ListBuckets() + c.Assert(err, IsNil) + c.Assert(len(buckets), Equals, 5) + c.Assert(buckets[0].ACL, Equals, BucketACL("private")) +} + +// test re-create bucket +func (s *MyCacheSuite) TestMakeBucketWithSameNameFails(c *C) { + err := dc.MakeBucket("foo3", "private") + c.Assert(err, IsNil) + + err = dc.MakeBucket("foo3", "private") + c.Assert(err, Not(IsNil)) +} + +// test make multiple buckets +func (s *MyCacheSuite) TestCreateMultipleBucketsAndList(c *C) { + // add a second bucket + err := dc.MakeBucket("foo4", "private") + c.Assert(err, IsNil) + + err = dc.MakeBucket("bar1", "private") + c.Assert(err, IsNil) + + buckets, err := dc.ListBuckets() + c.Assert(err, IsNil) + + c.Assert(len(buckets), Equals, 2) + c.Assert(buckets[0].Name, Equals, "bar1") + c.Assert(buckets[1].Name, Equals, "foo4") + + err = dc.MakeBucket("foobar1", "private") + c.Assert(err, IsNil) + + buckets, err = dc.ListBuckets() + c.Assert(err, IsNil) + + c.Assert(len(buckets), Equals, 3) + c.Assert(buckets[2].Name, Equals, "foobar1") +} + +// test object create without bucket +func (s *MyCacheSuite) TestNewObjectFailsWithoutBucket(c *C) { + _, err := dc.CreateObject("unknown", "obj", "", "", 0, nil) + c.Assert(err, Not(IsNil)) +} + +// test create object metadata +func (s *MyCacheSuite) TestNewObjectMetadata(c *C) { + data := "Hello World" + hasher := md5.New() + hasher.Write([]byte(data)) + expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + reader := ioutil.NopCloser(bytes.NewReader([]byte(data))) + + err := dc.MakeBucket("foo6", "private") + c.Assert(err, IsNil) + + objectMetadata, err := dc.CreateObject("foo6", "obj", "application/json", expectedMd5Sum, int64(len(data)), reader) + c.Assert(err, IsNil) + c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil))) + c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json") +} + +// test create object fails without name +func (s *MyCacheSuite) TestNewObjectFailsWithEmptyName(c *C) { + _, err := dc.CreateObject("foo", "", "", "", 0, nil) + c.Assert(err, Not(IsNil)) +} + +// test create object +func (s *MyCacheSuite) TestNewObjectCanBeWritten(c *C) { + err := dc.MakeBucket("foo", "private") + c.Assert(err, IsNil) + + data := "Hello World" + + hasher := md5.New() + hasher.Write([]byte(data)) + expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + reader := ioutil.NopCloser(bytes.NewReader([]byte(data))) + + actualMetadata, err := dc.CreateObject("foo", "obj", "application/octet-stream", expectedMd5Sum, int64(len(data)), reader) + c.Assert(err, IsNil) + c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil))) + + var buffer bytes.Buffer + size, err := dc.GetObject(&buffer, "foo", "obj") + c.Assert(err, IsNil) + c.Assert(size, Equals, int64(len(data))) + c.Assert(buffer.Bytes(), DeepEquals, []byte(data)) + + actualMetadata, err = dc.GetObjectMetadata("foo", "obj") + c.Assert(err, IsNil) + c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum) + c.Assert(int64(len(data)), Equals, actualMetadata.Size) +} + +// test list objects +func (s *MyCacheSuite) TestMultipleNewObjects(c *C) { + c.Assert(dc.MakeBucket("foo5", "private"), IsNil) + + one := ioutil.NopCloser(bytes.NewReader([]byte("one"))) + + _, err := dc.CreateObject("foo5", "obj1", "", "", int64(len("one")), one) + c.Assert(err, IsNil) + + two := ioutil.NopCloser(bytes.NewReader([]byte("two"))) + _, err = dc.CreateObject("foo5", "obj2", "", "", int64(len("two")), two) + c.Assert(err, IsNil) + + var buffer1 bytes.Buffer + size, err := dc.GetObject(&buffer1, "foo5", "obj1") + c.Assert(err, IsNil) + c.Assert(size, Equals, int64(len([]byte("one")))) + c.Assert(buffer1.Bytes(), DeepEquals, []byte("one")) + + var buffer2 bytes.Buffer + size, err = dc.GetObject(&buffer2, "foo5", "obj2") + c.Assert(err, IsNil) + c.Assert(size, Equals, int64(len([]byte("two")))) + + c.Assert(buffer2.Bytes(), DeepEquals, []byte("two")) + + /// test list of objects + + // test list objects with prefix and delimiter + var resources BucketResourcesMetadata + resources.Prefix = "o" + resources.Delimiter = "1" + resources.Maxkeys = 10 + objectsMetadata, resources, err := dc.ListObjects("foo5", resources) + c.Assert(err, IsNil) + c.Assert(resources.IsTruncated, Equals, false) + c.Assert(resources.CommonPrefixes[0], Equals, "obj1") + + // test list objects with only delimiter + resources.Prefix = "" + resources.Delimiter = "1" + resources.Maxkeys = 10 + objectsMetadata, resources, err = dc.ListObjects("foo5", resources) + c.Assert(err, IsNil) + c.Assert(objectsMetadata[0].Object, Equals, "obj2") + c.Assert(resources.IsTruncated, Equals, false) + c.Assert(resources.CommonPrefixes[0], Equals, "obj1") + + // test list objects with only prefix + resources.Prefix = "o" + resources.Delimiter = "" + resources.Maxkeys = 10 + objectsMetadata, resources, err = dc.ListObjects("foo5", resources) + c.Assert(err, IsNil) + c.Assert(resources.IsTruncated, Equals, false) + c.Assert(objectsMetadata[0].Object, Equals, "obj1") + c.Assert(objectsMetadata[1].Object, Equals, "obj2") + + three := ioutil.NopCloser(bytes.NewReader([]byte("three"))) + _, err = dc.CreateObject("foo5", "obj3", "", "", int64(len("three")), three) + c.Assert(err, IsNil) + + var buffer bytes.Buffer + size, err = dc.GetObject(&buffer, "foo5", "obj3") + c.Assert(err, IsNil) + c.Assert(size, Equals, int64(len([]byte("three")))) + c.Assert(buffer.Bytes(), DeepEquals, []byte("three")) + + // test list objects with maxkeys + resources.Prefix = "o" + resources.Delimiter = "" + resources.Maxkeys = 2 + objectsMetadata, resources, err = dc.ListObjects("foo5", resources) + c.Assert(err, IsNil) + c.Assert(resources.IsTruncated, Equals, true) + c.Assert(len(objectsMetadata), Equals, 2) +} diff --git a/pkg/storage/donut/donut_test.go b/pkg/storage/donut/donut_test.go index 6b7c949bd..e4f56fec4 100644 --- a/pkg/storage/donut/donut_test.go +++ b/pkg/storage/donut/donut_test.go @@ -9,7 +9,7 @@ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliedd. * See the License for the specific language governing permissions and * limitations under the License. */ @@ -31,13 +31,13 @@ import ( . "github.com/minio/check" ) -func Test(t *testing.T) { TestingT(t) } +func TestDonut(t *testing.T) { TestingT(t) } -type MySuite struct { +type MyDonutSuite struct { root string } -var _ = Suite(&MySuite{}) +var _ = Suite(&MyDonutSuite{}) // create a dummy TestNodeDiskMap func createTestNodeDiskMap(p string) map[string][]string { @@ -55,39 +55,40 @@ func createTestNodeDiskMap(p string) map[string][]string { return nodes } -var d Cache +var dd Cache -func (s *MySuite) SetUpSuite(c *C) { +func (s *MyDonutSuite) SetUpSuite(c *C) { root, err := ioutil.TempDir(os.TempDir(), "donut-") c.Assert(err, IsNil) s.root = root - d = NewCache(100000, time.Duration(1*time.Hour), "test", createTestNodeDiskMap(root)) - buckets, err := d.ListBuckets() + dd = NewCache(100000, time.Duration(1*time.Hour), "test", createTestNodeDiskMap(root)) + // testing empty donut + buckets, err := dd.ListBuckets() c.Assert(err, IsNil) c.Assert(len(buckets), Equals, 0) } -func (s *MySuite) TearDownSuite(c *C) { +func (s *MyDonutSuite) TearDownSuite(c *C) { os.RemoveAll(s.root) } // test make bucket without name -func (s *MySuite) TestBucketWithoutNameFails(c *C) { +func (s *MyDonutSuite) TestBucketWithoutNameFails(c *C) { // fail to create new bucket without a name - err := d.MakeBucket("", "private") + err := dd.MakeBucket("", "private") c.Assert(err, Not(IsNil)) - err = d.MakeBucket(" ", "private") + err = dd.MakeBucket(" ", "private") c.Assert(err, Not(IsNil)) } // test empty bucket -func (s *MySuite) TestEmptyBucket(c *C) { - c.Assert(d.MakeBucket("foo1", "private"), IsNil) +func (s *MyDonutSuite) TestEmptyBucket(c *C) { + c.Assert(dd.MakeBucket("foo1", "private"), IsNil) // check if bucket is empty var resources BucketResourcesMetadata resources.Maxkeys = 1 - objectsMetadata, resources, err := d.ListObjects("foo1", resources) + objectsMetadata, resources, err := dd.ListObjects("foo1", resources) c.Assert(err, IsNil) c.Assert(len(objectsMetadata), Equals, 0) c.Assert(resources.CommonPrefixes, DeepEquals, []string{}) @@ -95,47 +96,47 @@ func (s *MySuite) TestEmptyBucket(c *C) { } // test bucket list -func (s *MySuite) TestMakeBucketAndList(c *C) { +func (s *MyDonutSuite) TestMakeBucketAndList(c *C) { // create bucket - err := d.MakeBucket("foo2", "private") + err := dd.MakeBucket("foo2", "private") c.Assert(err, IsNil) // check bucket exists - buckets, err := d.ListBuckets() + buckets, err := dd.ListBuckets() c.Assert(err, IsNil) c.Assert(len(buckets), Equals, 5) c.Assert(buckets[0].ACL, Equals, BucketACL("private")) } // test re-create bucket -func (s *MySuite) TestMakeBucketWithSameNameFails(c *C) { - err := d.MakeBucket("foo3", "private") +func (s *MyDonutSuite) TestMakeBucketWithSameNameFails(c *C) { + err := dd.MakeBucket("foo3", "private") c.Assert(err, IsNil) - err = d.MakeBucket("foo3", "private") + err = dd.MakeBucket("foo3", "private") c.Assert(err, Not(IsNil)) } // test make multiple buckets -func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) { +func (s *MyDonutSuite) TestCreateMultipleBucketsAndList(c *C) { // add a second bucket - err := d.MakeBucket("foo4", "private") + err := dd.MakeBucket("foo4", "private") c.Assert(err, IsNil) - err = d.MakeBucket("bar1", "private") + err = dd.MakeBucket("bar1", "private") c.Assert(err, IsNil) - buckets, err := d.ListBuckets() + buckets, err := dd.ListBuckets() c.Assert(err, IsNil) c.Assert(len(buckets), Equals, 2) c.Assert(buckets[0].Name, Equals, "bar1") c.Assert(buckets[1].Name, Equals, "foo4") - err = d.MakeBucket("foobar1", "private") + err = dd.MakeBucket("foobar1", "private") c.Assert(err, IsNil) - buckets, err = d.ListBuckets() + buckets, err = dd.ListBuckets() c.Assert(err, IsNil) c.Assert(len(buckets), Equals, 3) @@ -143,37 +144,37 @@ func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) { } // test object create without bucket -func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) { - _, err := d.CreateObject("unknown", "obj", "", "", 0, nil) +func (s *MyDonutSuite) TestNewObjectFailsWithoutBucket(c *C) { + _, err := dd.CreateObject("unknown", "obj", "", "", 0, nil) c.Assert(err, Not(IsNil)) } // test create object metadata -func (s *MySuite) TestNewObjectMetadata(c *C) { +func (s *MyDonutSuite) TestNewObjectMetadata(c *C) { data := "Hello World" hasher := md5.New() hasher.Write([]byte(data)) expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) reader := ioutil.NopCloser(bytes.NewReader([]byte(data))) - err := d.MakeBucket("foo6", "private") + err := dd.MakeBucket("foo6", "private") c.Assert(err, IsNil) - objectMetadata, err := d.CreateObject("foo6", "obj", "application/json", expectedMd5Sum, int64(len(data)), reader) + objectMetadata, err := dd.CreateObject("foo6", "obj", "application/json", expectedMd5Sum, int64(len(data)), reader) c.Assert(err, IsNil) c.Assert(objectMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil))) c.Assert(objectMetadata.Metadata["contentType"], Equals, "application/json") } // test create object fails without name -func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) { - _, err := d.CreateObject("foo", "", "", "", 0, nil) +func (s *MyDonutSuite) TestNewObjectFailsWithEmptyName(c *C) { + _, err := dd.CreateObject("foo", "", "", "", 0, nil) c.Assert(err, Not(IsNil)) } // test create object -func (s *MySuite) TestNewObjectCanBeWritten(c *C) { - err := d.MakeBucket("foo", "private") +func (s *MyDonutSuite) TestNewObjectCanBeWritten(c *C) { + err := dd.MakeBucket("foo", "private") c.Assert(err, IsNil) data := "Hello World" @@ -183,43 +184,43 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) { expectedMd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) reader := ioutil.NopCloser(bytes.NewReader([]byte(data))) - actualMetadata, err := d.CreateObject("foo", "obj", "application/octet-stream", expectedMd5Sum, int64(len(data)), reader) + actualMetadata, err := dd.CreateObject("foo", "obj", "application/octet-stream", expectedMd5Sum, int64(len(data)), reader) c.Assert(err, IsNil) c.Assert(actualMetadata.MD5Sum, Equals, hex.EncodeToString(hasher.Sum(nil))) var buffer bytes.Buffer - size, err := d.GetObject(&buffer, "foo", "obj") + size, err := dd.GetObject(&buffer, "foo", "obj") c.Assert(err, IsNil) c.Assert(size, Equals, int64(len(data))) c.Assert(buffer.Bytes(), DeepEquals, []byte(data)) - actualMetadata, err = d.GetObjectMetadata("foo", "obj") + actualMetadata, err = dd.GetObjectMetadata("foo", "obj") c.Assert(err, IsNil) c.Assert(hex.EncodeToString(hasher.Sum(nil)), Equals, actualMetadata.MD5Sum) c.Assert(int64(len(data)), Equals, actualMetadata.Size) } // test list objects -func (s *MySuite) TestMultipleNewObjects(c *C) { - c.Assert(d.MakeBucket("foo5", "private"), IsNil) +func (s *MyDonutSuite) TestMultipleNewObjects(c *C) { + c.Assert(dd.MakeBucket("foo5", "private"), IsNil) one := ioutil.NopCloser(bytes.NewReader([]byte("one"))) - _, err := d.CreateObject("foo5", "obj1", "", "", int64(len("one")), one) + _, err := dd.CreateObject("foo5", "obj1", "", "", int64(len("one")), one) c.Assert(err, IsNil) two := ioutil.NopCloser(bytes.NewReader([]byte("two"))) - _, err = d.CreateObject("foo5", "obj2", "", "", int64(len("two")), two) + _, err = dd.CreateObject("foo5", "obj2", "", "", int64(len("two")), two) c.Assert(err, IsNil) var buffer1 bytes.Buffer - size, err := d.GetObject(&buffer1, "foo5", "obj1") + size, err := dd.GetObject(&buffer1, "foo5", "obj1") c.Assert(err, IsNil) c.Assert(size, Equals, int64(len([]byte("one")))) c.Assert(buffer1.Bytes(), DeepEquals, []byte("one")) var buffer2 bytes.Buffer - size, err = d.GetObject(&buffer2, "foo5", "obj2") + size, err = dd.GetObject(&buffer2, "foo5", "obj2") c.Assert(err, IsNil) c.Assert(size, Equals, int64(len([]byte("two")))) @@ -232,7 +233,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) { resources.Prefix = "o" resources.Delimiter = "1" resources.Maxkeys = 10 - objectsMetadata, resources, err := d.ListObjects("foo5", resources) + objectsMetadata, resources, err := dd.ListObjects("foo5", resources) c.Assert(err, IsNil) c.Assert(resources.IsTruncated, Equals, false) c.Assert(resources.CommonPrefixes[0], Equals, "obj1") @@ -241,9 +242,9 @@ func (s *MySuite) TestMultipleNewObjects(c *C) { resources.Prefix = "" resources.Delimiter = "1" resources.Maxkeys = 10 - objectsMetadata, resources, err = d.ListObjects("foo5", resources) + objectsMetadata, resources, err = dd.ListObjects("foo5", resources) c.Assert(err, IsNil) - c.Assert(objectsMetadata[0].Object, Equals, "obj1") + c.Assert(objectsMetadata[0].Object, Equals, "obj2") c.Assert(resources.IsTruncated, Equals, false) c.Assert(resources.CommonPrefixes[0], Equals, "obj1") @@ -251,18 +252,18 @@ func (s *MySuite) TestMultipleNewObjects(c *C) { resources.Prefix = "o" resources.Delimiter = "" resources.Maxkeys = 10 - objectsMetadata, resources, err = d.ListObjects("foo5", resources) + objectsMetadata, resources, err = dd.ListObjects("foo5", resources) c.Assert(err, IsNil) c.Assert(resources.IsTruncated, Equals, false) c.Assert(objectsMetadata[0].Object, Equals, "obj1") c.Assert(objectsMetadata[1].Object, Equals, "obj2") three := ioutil.NopCloser(bytes.NewReader([]byte("three"))) - _, err = d.CreateObject("foo5", "obj3", "", "", int64(len("three")), three) + _, err = dd.CreateObject("foo5", "obj3", "", "", int64(len("three")), three) c.Assert(err, IsNil) var buffer bytes.Buffer - size, err = d.GetObject(&buffer, "foo5", "obj3") + size, err = dd.GetObject(&buffer, "foo5", "obj3") c.Assert(err, IsNil) c.Assert(size, Equals, int64(len([]byte("three")))) c.Assert(buffer.Bytes(), DeepEquals, []byte("three")) @@ -271,7 +272,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) { resources.Prefix = "o" resources.Delimiter = "" resources.Maxkeys = 2 - objectsMetadata, resources, err = d.ListObjects("foo5", resources) + objectsMetadata, resources, err = dd.ListObjects("foo5", resources) c.Assert(err, IsNil) c.Assert(resources.IsTruncated, Equals, true) c.Assert(len(objectsMetadata), Equals, 2) diff --git a/pkg/storage/pq/pq.go b/pkg/storage/pq/pq.go deleted file mode 100644 index c491f4bb6..000000000 --- a/pkg/storage/pq/pq.go +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package pq - -import "container/heap" - -// Item container for tasks in priority queue -type Item struct { - task Task // task - - // The index is needed by Fix and is maintained by the heap.Interface methods. - index int // The index of the item in the heap. -} - -// A PriorityQueue implements heap.Interface and holds Items. -type PriorityQueue []*Item - -// Len length of current priority queue -func (pq PriorityQueue) Len() int { return len(pq) } - -// Less used internally by heap.Interface to arrange items in order -func (pq PriorityQueue) Less(i, j int) bool { - // We want Pop to give us the highest, not lowest, priority so we use greater than here. - return pq[i].task.GetPriority() > pq[j].task.GetPriority() -} - -// Swap used internally by heap.Interface to arrange incoming items -func (pq PriorityQueue) Swap(i, j int) { - pq[i], pq[j] = pq[j], pq[i] - pq[i].index = i - pq[j].index = j -} - -// Push push items onto priority queue -func (pq *PriorityQueue) Push(x interface{}) { - n := len(*pq) - item := x.(*Item) - item.index = n - *pq = append(*pq, item) -} - -// Pop pop items with highest priority -func (pq *PriorityQueue) Pop() interface{} { - old := *pq - n := len(old) - item := old[n-1] - item.index = -1 // for safety - *pq = old[0 : n-1] - return item -} - -// Fix modifies an item in-place on the queue -func (pq *PriorityQueue) Fix(item *Item, task Task) { - item.task = task - heap.Fix(pq, item.index) -} diff --git a/pkg/storage/pq/pq_test.go b/pkg/storage/pq/pq_test.go deleted file mode 100644 index b070324fa..000000000 --- a/pkg/storage/pq/pq_test.go +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package pq - -import ( - "container/heap" - "fmt" - "testing" - - . "github.com/minio/check" -) - -func Test(t *testing.T) { TestingT(t) } - -type MySuite struct{} - -var _ = Suite(&MySuite{}) - -func helloTask1() <-chan error { - errCh := make(chan error) - go func() { - defer close(errCh) - println("Hello task1") - errCh <- nil - }() - return errCh -} - -func helloTask2() <-chan error { - errCh := make(chan error) - go func() { - defer close(errCh) - println("Hello task2") - errCh <- nil - }() - return errCh -} - -func newJob1() <-chan error { - errCh := make(chan error) - go func() { - defer close(errCh) - println("New Job1") - errCh <- nil - }() - return errCh -} - -func newJob2() <-chan error { - errCh := make(chan error) - go func() { - defer close(errCh) - println("New Job2") - errCh <- nil - }() - return errCh -} - -func (s *MySuite) TestPQ(c *C) { - // Create a priority queue, put the items in it, and - // establish the priority queue (heap) invariants. - pq := make(PriorityQueue, 2) - pq[0] = &Item{ - task: Task{job: helloTask1, priority: 2}, - index: 0, - } - pq[1] = &Item{ - task: Task{job: helloTask2, priority: 1}, - index: 1, - } - heap.Init(&pq) - - // Insert a new item and then modify its priority. - item := &Item{ - task: Task{job: newJob1, priority: 5}, - } - heap.Push(&pq, item) - newTask := Task{job: newJob2, priority: 6} - pq.Fix(item, newTask) - - // Take the items out; they arrive in decreasing priority order. - for pq.Len() > 0 { - item := heap.Pop(&pq).(*Item) - fmt.Printf("%.2d", item.task.GetPriority()) - item.task.Execute() - } -} diff --git a/pkg/storage/pq/task.go b/pkg/storage/pq/task.go deleted file mode 100644 index 9a1f3e84d..000000000 --- a/pkg/storage/pq/task.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Minimalist Object Storage, (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package pq - -// Task container for any generic tasks -type Task struct { - job func() <-chan error - priority int -} - -// GetPriority get current task priority -func (t Task) GetPriority() int { - return t.priority -} - -// UpdatePriority update current task priority -func (t Task) UpdatePriority(p int) { - t.priority = p -} - -// Execute execute current task -func (t Task) Execute() error { - return <-t.job() -}