From 5c0b98abf0917ed2e096c1f5410d61511b116ffd Mon Sep 17 00:00:00 2001 From: poornas Date: Mon, 10 Sep 2018 09:42:43 -0700 Subject: [PATCH] Add ObjectOptions to ObjectLayer calls (#6382) --- cmd/acl-handlers.go | 2 +- cmd/admin-handlers_test.go | 6 +- cmd/admin-heal-ops.go | 2 +- cmd/benchmark-utils_test.go | 17 +- cmd/bucket-handlers.go | 2 +- cmd/bucket-handlers_test.go | 2 +- cmd/config.go | 6 +- cmd/disk-cache-fs.go | 14 +- cmd/disk-cache.go | 110 +- cmd/disk-cache_test.go | 16 +- cmd/dummy-object-layer_test.go | 14 +- cmd/fs-v1-metadata_test.go | 4 +- cmd/fs-v1-multipart.go | 10 +- cmd/fs-v1-multipart_test.go | 20 +- cmd/fs-v1.go | 8 +- cmd/fs-v1_test.go | 16 +- cmd/gateway-unsupported.go | 8 +- cmd/gateway/azure/gateway-azure.go | 18 +- cmd/gateway/b2/gateway-b2.go | 12 +- cmd/gateway/gcs/gateway-gcs.go | 12 +- cmd/gateway/manta/gateway-manta.go | 12 +- cmd/gateway/oss/gateway-oss.go | 18 +- cmd/gateway/s3/gateway-s3.go | 28 +- cmd/gateway/sia/gateway-sia.go | 6 +- cmd/object-api-deleteobject_test.go | 2 +- cmd/object-api-getobject_test.go | 12 +- cmd/object-api-getobjectinfo_test.go | 9 +- cmd/object-api-interface.go | 21 +- cmd/object-api-listobjects_test.go | 4 +- cmd/object-api-multipart_test.go | 54 +- cmd/object-api-putobject_test.go | 16 +- cmd/object-handlers.go | 61 +- cmd/object-handlers_test.go | 59 +- cmd/object_api_suite_test.go | 66 +- cmd/post-policy_test.go | 7 +- cmd/storage-class_test.go | 15 +- cmd/web-handlers.go | 19 +- cmd/web-handlers_test.go | 23 +- cmd/xl-sets.go | 34 +- cmd/xl-v1-common_test.go | 2 +- cmd/xl-v1-healing-common_test.go | 4 +- cmd/xl-v1-healing_test.go | 5 +- cmd/xl-v1-metadata_test.go | 7 +- cmd/xl-v1-multipart.go | 10 +- cmd/xl-v1-multipart_test.go | 3 +- cmd/xl-v1-object.go | 20 +- cmd/xl-v1-object_test.go | 27 +- vendor/github.com/minio/minio-go/README.md | 6 +- .../minio/minio-go/api-compose-object.go | 33 +- .../minio/minio-go/api-get-lifecycle.go | 77 + .../minio/minio-go/api-get-object-acl.go | 136 + .../minio/minio-go/api-get-options.go | 2 +- vendor/github.com/minio/minio-go/api-list.go | 15 +- .../minio/minio-go/api-put-bucket.go | 81 + .../minio-go/api-put-object-multipart.go | 2 +- .../github.com/minio/minio-go/api-remove.go | 10 +- vendor/github.com/minio/minio-go/api.go | 25 +- vendor/github.com/minio/minio-go/core.go | 9 +- .../minio/minio-go/functional_tests.go | 7499 ----------------- vendor/github.com/minio/minio-go/retry.go | 46 +- vendor/github.com/minio/minio-go/utils.go | 3 +- vendor/vendor.json | 6 +- 62 files changed, 812 insertions(+), 7981 deletions(-) create mode 100644 vendor/github.com/minio/minio-go/api-get-lifecycle.go create mode 100644 vendor/github.com/minio/minio-go/api-get-object-acl.go delete mode 100644 vendor/github.com/minio/minio-go/functional_tests.go diff --git a/cmd/acl-handlers.go b/cmd/acl-handlers.go index f871a1db8..718531c1b 100644 --- a/cmd/acl-handlers.go +++ b/cmd/acl-handlers.go @@ -121,7 +121,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http. } // Before proceeding validate if object exists. - _, err := objAPI.GetObjectInfo(ctx, bucket, object) + _, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go index aedd9cf70..5ed2f095b 100644 --- a/cmd/admin-handlers_test.go +++ b/cmd/admin-handlers_test.go @@ -284,7 +284,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) { objectName := fmt.Sprintf("%s-%d", objName, i) _, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("hello")), - int64(len("hello")), "", ""), nil) + int64(len("hello")), "", ""), nil, ObjectOptions{}) if err != nil { t.Fatalf("Failed to create %s - %v", objectName, err) @@ -296,14 +296,14 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) { { objName := "mpObject" uploadID, err := atb.objLayer.NewMultipartUpload(context.Background(), bucketName, - objName, nil) + objName, nil, ObjectOptions{}) if err != nil { t.Fatalf("mp new error: %v", err) } _, err = atb.objLayer.PutObjectPart(context.Background(), bucketName, objName, uploadID, 3, mustGetHashReader(t, bytes.NewReader( - []byte("hello")), int64(len("hello")), "", "")) + []byte("hello")), int64(len("hello")), "", ""), ObjectOptions{}) if err != nil { t.Fatalf("mp put error: %v", err) } diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go index b39b2ecf1..65c9752b0 100644 --- a/cmd/admin-heal-ops.go +++ b/cmd/admin-heal-ops.go @@ -610,7 +610,7 @@ func (h *healSequence) healBucket(bucket string) error { if h.objPrefix != "" { // Check if an object named as the objPrefix exists, // and if so heal it. - _, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix) + _, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{}) if err == nil { err = h.healObject(bucket, h.objPrefix) if err != nil { diff --git a/cmd/benchmark-utils_test.go b/cmd/benchmark-utils_test.go index 70456c1d4..e516b780d 100644 --- a/cmd/benchmark-utils_test.go +++ b/cmd/benchmark-utils_test.go @@ -61,7 +61,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { for i := 0; i < b.N; i++ { // insert the object. objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i), - mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata) + mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{}) if err != nil { b.Fatal(err) } @@ -97,7 +97,8 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) { // generate md5sum for the generated data. // md5sum of the data to written is required as input for NewMultipartUpload. metadata := make(map[string]string) - uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata) + opts := ObjectOptions{} + uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata, opts) if err != nil { b.Fatal(err) } @@ -122,7 +123,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) { md5hex = getMD5Hash([]byte(textPartData)) var partInfo PartInfo partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j, - mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex)) + mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts) if err != nil { b.Fatal(err) } @@ -203,7 +204,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { // insert the object. var objInfo ObjectInfo objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i), - mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata) + mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{}) if err != nil { b.Fatal(err) } @@ -218,7 +219,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { b.ResetTimer() for i := 0; i < b.N; i++ { var buffer = new(bytes.Buffer) - err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "") + err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "", ObjectOptions{}) if err != nil { b.Error(err) } @@ -303,7 +304,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { for pb.Next() { // insert the object. objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i), - mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata) + mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{}) if err != nil { b.Fatal(err) } @@ -343,7 +344,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { // insert the object. var objInfo ObjectInfo objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i), - mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata) + mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{}) if err != nil { b.Fatal(err) } @@ -359,7 +360,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { b.RunParallel(func(pb *testing.PB) { i := 0 for pb.Next() { - err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "") + err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "", ObjectOptions{}) if err != nil { b.Error(err) } diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 17e74264d..7262bf9a3 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -622,7 +622,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h } } - objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata) + objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata, ObjectOptions{}) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return diff --git a/cmd/bucket-handlers_test.go b/cmd/bucket-handlers_test.go index 1b88dfdb3..f55a26f2d 100644 --- a/cmd/bucket-handlers_test.go +++ b/cmd/bucket-handlers_test.go @@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa for i := 0; i < 10; i++ { objectName := "test-object-" + strconv.Itoa(i) // uploading the object. - _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil) + _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{}) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object %d: Error uploading object: %v", i, err) diff --git a/cmd/config.go b/cmd/config.go index af8cef7ef..fa8467817 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -158,7 +158,7 @@ func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error { return checkServerConfigEtcd(configFile) } - if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile); err != nil { + if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil { // Convert ObjectNotFound, Quorum errors into errConfigNotFound if isErrObjectNotFound(err) || isInsufficientReadQuorum(err) { return errConfigNotFound @@ -176,7 +176,7 @@ func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error { return err } - _, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil) + _, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil, ObjectOptions{}) return err } @@ -185,7 +185,7 @@ var errConfigNotFound = errors.New("config file not found") func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) { var buffer bytes.Buffer // Read entire content by setting size to -1 - if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, ""); err != nil { + if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil { // Convert ObjectNotFound, IncompleteBody and Quorum errors into errConfigNotFound if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) { return nil, errConfigNotFound diff --git a/cmd/disk-cache-fs.go b/cmd/disk-cache-fs.go index 4650d9fdd..b92a85dea 100644 --- a/cmd/disk-cache-fs.go +++ b/cmd/disk-cache-fs.go @@ -258,7 +258,7 @@ func (cfs *cacheFSObjects) IsOnline() bool { } // Caches the object to disk -func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) error { +func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) error { if cfs.diskUsageHigh() { select { case cfs.purgeChan <- struct{}{}: @@ -275,7 +275,7 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data return pErr } } - _, err := cfs.PutObject(ctx, bucket, object, data, metadata) + _, err := cfs.PutObject(ctx, bucket, object, data, metadata, opts) // if err is due to disk being offline , mark cache drive as offline if IsErr(err, baseErrs...) { cfs.setOnline(false) @@ -284,8 +284,8 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data } // Returns the handle for the cached object -func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) { - return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag) +func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) { + return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts) } // Deletes the cached object @@ -295,13 +295,13 @@ func (cfs *cacheFSObjects) Delete(ctx context.Context, bucket, object string) (e // convenience function to check if object is cached on this cacheFSObjects func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bool { - _, err := cfs.GetObjectInfo(ctx, bucket, object) + _, err := cfs.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) return err == nil } // Identical to fs PutObject operation except that it uses ETag in metadata // headers. -func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { +func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) { fs := cfs.FSObjects // Lock the object. objectLock := fs.nsMutex.NewNSLock(bucket, object) @@ -438,7 +438,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object // Implements S3 compatible initiate multipart API. Operation here is identical // to fs backend implementation - with the exception that cache FS uses the uploadID // generated on the backend -func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string) (string, error) { +func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string, opts ObjectOptions) (string, error) { if cfs.diskUsageHigh() { select { case cfs.purgeChan <- struct{}{}: diff --git a/cmd/disk-cache.go b/cmd/disk-cache.go index f44ef7375..358a62493 100644 --- a/cmd/disk-cache.go +++ b/cmd/disk-cache.go @@ -57,16 +57,16 @@ type cacheObjects struct { // file path patterns to exclude from cache exclude []string // Object functions pointing to the corresponding functions of backend implementation. - GetObjectFn func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) - GetObjectInfoFn func(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) - PutObjectFn func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) + GetObjectFn func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) + GetObjectInfoFn func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) + PutObjectFn func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) DeleteObjectFn func(ctx context.Context, bucket, object string) error ListObjectsFn func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) ListObjectsV2Fn func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) ListBucketsFn func(ctx context.Context) (buckets []BucketInfo, err error) GetBucketInfoFn func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) - NewMultipartUploadFn func(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) - PutObjectPartFn func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) + NewMultipartUploadFn func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) + PutObjectPartFn func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) AbortMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string) error CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) DeleteBucketFn func(ctx context.Context, bucket string) error @@ -88,14 +88,14 @@ type CacheObjectLayer interface { ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) DeleteBucket(ctx context.Context, bucket string) error // Object operations. - GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) - GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) - PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) + GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) + GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) + PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) DeleteObject(ctx context.Context, bucket, object string) error // Multipart operations. - NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) - PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) + NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) + PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) @@ -177,20 +177,20 @@ func (c cacheObjects) getMetadata(objInfo ObjectInfo) map[string]string { // Uses cached-object to serve the request. If object is not cached it serves the request from the backend and also // stores it in the cache for serving subsequent requests. -func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) { +func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) { GetObjectFn := c.GetObjectFn GetObjectInfoFn := c.GetObjectInfoFn if c.isCacheExclude(bucket, object) { - return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag) + return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts) } // fetch cacheFSObjects if object is currently cached or nearest available cache drive dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object) if err != nil { - return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag) + return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts) } // stat object on backend - objInfo, err := GetObjectInfoFn(ctx, bucket, object) + objInfo, err := GetObjectInfoFn(ctx, bucket, object, opts) backendDown := backendDownError(err) if err != nil && !backendDown { if _, ok := err.(ObjectNotFound); ok { @@ -201,27 +201,27 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star } if !backendDown && filterFromCache(objInfo.UserDefined) { - return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag) + return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts) } - cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object) + cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object, opts) if err == nil { if backendDown { // If the backend is down, serve the request from cache. - return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag) + return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag, opts) } if cachedObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) { - return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag) + return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag, opts) } dcache.Delete(ctx, bucket, object) } if startOffset != 0 || length != objInfo.Size { // We don't cache partial objects. - return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag) + return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts) } if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) { // cache only objects < 1/100th of disk capacity - return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag) + return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts) } // Initialize pipe. pipeReader, pipeWriter := io.Pipe() @@ -230,13 +230,13 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star return err } go func() { - if err = GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag); err != nil { + if err = GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts); err != nil { pipeWriter.CloseWithError(err) return } pipeWriter.Close() // Close writer explicitly signaling we wrote all data. }() - err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo)) + err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo), opts) if err != nil { return err } @@ -245,17 +245,17 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star } // Returns ObjectInfo from cache if available. -func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string) (ObjectInfo, error) { +func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { getObjectInfoFn := c.GetObjectInfoFn if c.isCacheExclude(bucket, object) { - return getObjectInfoFn(ctx, bucket, object) + return getObjectInfoFn(ctx, bucket, object, opts) } // fetch cacheFSObjects if object is currently cached or nearest available cache drive dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object) if err != nil { - return getObjectInfoFn(ctx, bucket, object) + return getObjectInfoFn(ctx, bucket, object, opts) } - objInfo, err := getObjectInfoFn(ctx, bucket, object) + objInfo, err := getObjectInfoFn(ctx, bucket, object, opts) if err != nil { if _, ok := err.(ObjectNotFound); ok { // Delete the cached entry if backend object was deleted. @@ -266,14 +266,14 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string) return ObjectInfo{}, err } // when backend is down, serve from cache. - cachedObjInfo, cerr := dcache.GetObjectInfo(ctx, bucket, object) + cachedObjInfo, cerr := dcache.GetObjectInfo(ctx, bucket, object, opts) if cerr == nil { return cachedObjInfo, nil } return ObjectInfo{}, BackendDown{} } // when backend is up, do a sanity check on cached object - cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object) + cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object, opts) if err != nil { return objInfo, nil } @@ -560,24 +560,24 @@ func (c cacheObjects) isCacheExclude(bucket, object string) bool { } // PutObject - caches the uploaded object for single Put operations -func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { +func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) { putObjectFn := c.PutObjectFn dcache, err := c.cache.getCacheFS(ctx, bucket, object) if err != nil { // disk cache could not be located,execute backend call. - return putObjectFn(ctx, bucket, object, r, metadata) + return putObjectFn(ctx, bucket, object, r, metadata, opts) } size := r.Size() // fetch from backend if there is no space on cache drive if !dcache.diskAvailable(size * cacheSizeMultiplier) { - return putObjectFn(ctx, bucket, object, r, metadata) + return putObjectFn(ctx, bucket, object, r, metadata, opts) } // fetch from backend if cache exclude pattern or cache-control // directive set to exclude if c.isCacheExclude(bucket, object) || filterFromCache(metadata) { dcache.Delete(ctx, bucket, object) - return putObjectFn(ctx, bucket, object, r, metadata) + return putObjectFn(ctx, bucket, object, r, metadata, opts) } objInfo = ObjectInfo{} // Initialize pipe to stream data to backend @@ -595,7 +595,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h oinfoCh := make(chan ObjectInfo) errCh := make(chan error) go func() { - oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata) + oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata, opts) if perr != nil { pipeWriter.CloseWithError(perr) wPipe.CloseWithError(perr) @@ -608,7 +608,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h }() go func() { - if err = dcache.Put(ctx, bucket, object, cHashReader, metadata); err != nil { + if err = dcache.Put(ctx, bucket, object, cHashReader, metadata, opts); err != nil { wPipe.CloseWithError(err) return } @@ -627,39 +627,39 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h } // NewMultipartUpload - Starts a new multipart upload operation to backend and cache. -func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) { +func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) { newMultipartUploadFn := c.NewMultipartUploadFn if c.isCacheExclude(bucket, object) || filterFromCache(metadata) { - return newMultipartUploadFn(ctx, bucket, object, metadata) + return newMultipartUploadFn(ctx, bucket, object, metadata, opts) } dcache, err := c.cache.getCacheFS(ctx, bucket, object) if err != nil { // disk cache could not be located,execute backend call. - return newMultipartUploadFn(ctx, bucket, object, metadata) + return newMultipartUploadFn(ctx, bucket, object, metadata, opts) } - uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata) + uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata, opts) if err != nil { return } // create new multipart upload in cache with same uploadID - dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID) + dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID, opts) return uploadID, err } // PutObjectPart - uploads part to backend and cache simultaneously. -func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) { +func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) { putObjectPartFn := c.PutObjectPartFn dcache, err := c.cache.getCacheFS(ctx, bucket, object) if err != nil { // disk cache could not be located,execute backend call. - return putObjectPartFn(ctx, bucket, object, uploadID, partID, data) + return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts) } if c.isCacheExclude(bucket, object) { - return putObjectPartFn(ctx, bucket, object, uploadID, partID, data) + return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts) } // make sure cache has at least cacheSizeMultiplier * size available @@ -669,7 +669,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI case dcache.purgeChan <- struct{}{}: default: } - return putObjectPartFn(ctx, bucket, object, uploadID, partID, data) + return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts) } info = PartInfo{} @@ -688,7 +688,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI pinfoCh := make(chan PartInfo) errorCh := make(chan error) go func() { - info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader) + info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader, opts) if err != nil { close(pinfoCh) pipeWriter.CloseWithError(err) @@ -700,7 +700,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI pinfoCh <- info }() go func() { - if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader); perr != nil { + if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader, opts); perr != nil { wPipe.CloseWithError(perr) return } @@ -876,14 +876,14 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) { cache: dcache, exclude: config.Exclude, listPool: newTreeWalkPool(globalLookupTimeout), - GetObjectFn: func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { - return newObjectLayerFn().GetObject(ctx, bucket, object, startOffset, length, writer, etag) + GetObjectFn: func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error { + return newObjectLayerFn().GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts) }, - GetObjectInfoFn: func(ctx context.Context, bucket, object string) (ObjectInfo, error) { - return newObjectLayerFn().GetObjectInfo(ctx, bucket, object) + GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts) }, - PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { - return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata) + PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts) }, DeleteObjectFn: func(ctx context.Context, bucket, object string) error { return newObjectLayerFn().DeleteObject(ctx, bucket, object) @@ -900,11 +900,11 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) { GetBucketInfoFn: func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) { return newObjectLayerFn().GetBucketInfo(ctx, bucket) }, - NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) { - return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata) + NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) { + return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata, opts) }, - PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) { - return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data) + PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) { + return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts) }, AbortMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string) error { return newObjectLayerFn().AbortMultipartUpload(ctx, bucket, object, uploadID) diff --git a/cmd/disk-cache_test.go b/cmd/disk-cache_test.go index c03c3d2e1..839c3d51a 100644 --- a/cmd/disk-cache_test.go +++ b/cmd/disk-cache_test.go @@ -192,17 +192,18 @@ func TestDiskCache(t *testing.T) { objInfo.ContentType = contentType objInfo.ETag = etag objInfo.UserDefined = httpMeta + opts := ObjectOptions{} byteReader := bytes.NewReader([]byte(content)) hashReader, err := hash.NewReader(byteReader, int64(size), "", "") if err != nil { t.Fatal(err) } - err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta) + err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts) if err != nil { t.Fatal(err) } - cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName) + cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -219,7 +220,7 @@ func TestDiskCache(t *testing.T) { t.Fatal("Cached content-type does not match") } writer := bytes.NewBuffer(nil) - err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "") + err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts) if err != nil { t.Fatal(err) } @@ -266,6 +267,7 @@ func TestDiskCacheMaxUse(t *testing.T) { objInfo.ContentType = contentType objInfo.ETag = etag objInfo.UserDefined = httpMeta + opts := ObjectOptions{} byteReader := bytes.NewReader([]byte(content)) hashReader, err := hash.NewReader(byteReader, int64(size), "", "") @@ -273,16 +275,16 @@ func TestDiskCacheMaxUse(t *testing.T) { t.Fatal(err) } if !cache.diskAvailable(int64(size)) { - err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta) + err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts) if err != errDiskFull { t.Fatal("Cache max-use limit violated.") } } else { - err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta) + err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts) if err != nil { t.Fatal(err) } - cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName) + cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -299,7 +301,7 @@ func TestDiskCacheMaxUse(t *testing.T) { t.Fatal("Cached content-type does not match") } writer := bytes.NewBuffer(nil) - err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "") + err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts) if err != nil { t.Fatal(err) } diff --git a/cmd/dummy-object-layer_test.go b/cmd/dummy-object-layer_test.go index 0c0a512da..81eaa73f8 100644 --- a/cmd/dummy-object-layer_test.go +++ b/cmd/dummy-object-layer_test.go @@ -59,19 +59,19 @@ func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix, return } -func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) { +func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) { return } -func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) { +func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { return } -func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { +func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) { return } -func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) { +func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) { return } @@ -83,15 +83,15 @@ func (api *DummyObjectLayer) ListMultipartUploads(ctx context.Context, bucket, p return } -func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) { +func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) { return } -func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo) (info PartInfo, err error) { +func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error) { return } -func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) { +func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) { return } diff --git a/cmd/fs-v1-metadata_test.go b/cmd/fs-v1-metadata_test.go index 28c3eddba..336b5acb8 100644 --- a/cmd/fs-v1-metadata_test.go +++ b/cmd/fs-v1-metadata_test.go @@ -54,7 +54,7 @@ func TestReadFSMetadata(t *testing.T) { if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil { t.Fatal("Unexpected err: ", err) } - if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil { + if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil { t.Fatal("Unexpected err: ", err) } @@ -89,7 +89,7 @@ func TestWriteFSMetadata(t *testing.T) { if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil { t.Fatal("Unexpected err: ", err) } - if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil { + if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil { t.Fatal("Unexpected err: ", err) } diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index bb6526332..9c794ab7b 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -209,7 +209,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k // subsequent request each UUID is unique. // // Implements S3 compatible initiate multipart API. -func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (string, error) { +func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, opts ObjectOptions) (string, error) { if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil { return "", toObjectErr(err, bucket) } @@ -249,7 +249,7 @@ func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object stri // object. Internally incoming data is written to '.minio.sys/tmp' location // and safely renamed to '.minio.sys/multipart' for reach parts. func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, - startOffset int64, length int64, srcInfo ObjectInfo) (pi PartInfo, e error) { + startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) { if err := checkNewMultipartArgs(ctx, srcBucket, srcObject, fs); err != nil { return pi, toObjectErr(err) @@ -257,7 +257,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d // Initialize pipe. go func() { - if gerr := fs.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil { + if gerr := fs.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { logger.LogIf(ctx, gerr) return @@ -271,7 +271,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d } }() - partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader) + partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader, dstOpts) if err != nil { return pi, toObjectErr(err, dstBucket, dstObject) } @@ -283,7 +283,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d // an ongoing multipart transaction. Internally incoming data is // written to '.minio.sys/tmp' location and safely renamed to // '.minio.sys/multipart' for reach parts. -func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) { +func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, e error) { if err := checkPutObjectPartArgs(ctx, bucket, object, fs); err != nil { return pi, toObjectErr(err, bucket) } diff --git a/cmd/fs-v1-multipart_test.go b/cmd/fs-v1-multipart_test.go index 15724d9b5..7b528dbf2 100644 --- a/cmd/fs-v1-multipart_test.go +++ b/cmd/fs-v1-multipart_test.go @@ -42,7 +42,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) { objectName := "object" obj.MakeBucketWithLocation(context.Background(), bucketName, "") - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, ObjectOptions{}) if err != nil { t.Fatal("Unexpected err: ", err) } @@ -81,7 +81,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) { // Test with disk removed. os.RemoveAll(disk) - if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}); err != nil { + if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}); err != nil { if !isSameType(err, BucketNotFound{}) { t.Fatal("Unexpected error ", err) } @@ -105,7 +105,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) { t.Fatal("Cannot create bucket, err: ", err) } - uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}) + uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}) if err != nil { t.Fatal("Unexpected error ", err) } @@ -114,7 +114,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) { sha256sum := "" fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) - _, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum)) + _, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{}) if !isSameType(err, BucketNotFound{}) { t.Fatal("Unexpected error ", err) } @@ -136,7 +136,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) { t.Fatal("Cannot create bucket, err: ", err) } - uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}) + uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}) if err != nil { t.Fatal("Unexpected error ", err) } @@ -168,14 +168,14 @@ func TestCompleteMultipartUpload(t *testing.T) { t.Fatal("Cannot create bucket, err: ", err) } - uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}) + uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}) if err != nil { t.Fatal("Unexpected error ", err) } md5Hex := getMD5Hash(data) - if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil { + if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil { t.Fatal("Unexpected error ", err) } @@ -202,14 +202,14 @@ func TestAbortMultipartUpload(t *testing.T) { t.Fatal("Cannot create bucket, err: ", err) } - uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}) + uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}) if err != nil { t.Fatal("Unexpected error ", err) } md5Hex := getMD5Hash(data) - if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil { + if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil { t.Fatal("Unexpected error ", err) } time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process." @@ -234,7 +234,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) { t.Fatal("Cannot create bucket, err: ", err) } - _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}) + _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}) if err != nil { t.Fatal("Unexpected error ", err) } diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index d7ea10b59..ca7cdb679 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -414,7 +414,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error { // CopyObject - copy object source object to destination object. // if source object and destination object are same we only // update metadata. -func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo) (oi ObjectInfo, e error) { +func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) { cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) // Hold write lock on destination since in both cases // - if source and destination are same @@ -504,7 +504,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) { +func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) { if err = checkGetObjArgs(ctx, bucket, object); err != nil { return err } @@ -710,7 +710,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s } // GetObjectInfo - reads object metadata and replies back ObjectInfo. -func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) { +func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) { oi, err := fs.getObjectInfoWithLock(ctx, bucket, object) if err == errCorruptedFormat || err == io.EOF { objectLock := fs.nsMutex.NewNSLock(bucket, object) @@ -754,7 +754,7 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin // until EOF, writes data directly to configured filesystem path. // Additionally writes `fs.json` which carries the necessary metadata // for future object operations. -func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) { +func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) { if err := checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil { return ObjectInfo{}, err } diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 43b822799..6d84fd841 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -40,7 +40,7 @@ func TestFSParentDirIsObject(t *testing.T) { } objectContent := "12345" objInfo, err := obj.PutObject(context.Background(), bucketName, objectName, - mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil) + mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{}) if err != nil { t.Fatal(err) } @@ -124,7 +124,7 @@ func TestFSShutdown(t *testing.T) { objectContent := "12345" obj.MakeBucketWithLocation(context.Background(), bucketName, "") - obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil) + obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{}) return fs, disk } @@ -203,7 +203,7 @@ func TestFSPutObject(t *testing.T) { } // With a regular object. - _, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}) if err == nil { t.Fatal("Unexpected should fail here, bucket doesn't exist") } @@ -212,7 +212,7 @@ func TestFSPutObject(t *testing.T) { } // With a directory object. - _, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil) + _, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{}) if err == nil { t.Fatal("Unexpected should fail here, bucket doesn't exist") } @@ -220,11 +220,11 @@ func TestFSPutObject(t *testing.T) { t.Fatalf("Expected error type BucketNotFound, got %#v", err) } - _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}) if err != nil { t.Fatal(err) } - _, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}) if err == nil { t.Fatal("Unexpected should fail here, backend corruption occurred") } @@ -239,7 +239,7 @@ func TestFSPutObject(t *testing.T) { } } - _, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil) + _, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{}) if err == nil { t.Fatal("Unexpected should fail here, backned corruption occurred") } @@ -267,7 +267,7 @@ func TestFSDeleteObject(t *testing.T) { objectName := "object" obj.MakeBucketWithLocation(context.Background(), bucketName, "") - obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}) // Test with invalid bucket name if err := fs.DeleteObject(context.Background(), "fo", objectName); !isSameType(err, BucketNameInvalid{}) { diff --git a/cmd/gateway-unsupported.go b/cmd/gateway-unsupported.go index 1a0c5e1d3..0fbbcfb20 100644 --- a/cmd/gateway-unsupported.go +++ b/cmd/gateway-unsupported.go @@ -35,19 +35,19 @@ func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket str } // NewMultipartUpload upload object in multiple parts -func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) { +func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) { logger.LogIf(ctx, NotImplemented{}) return "", NotImplemented{} } // CopyObjectPart copy part of object to uploadID for another object -func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo) (pi PartInfo, err error) { +func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, err error) { logger.LogIf(ctx, NotImplemented{}) return pi, NotImplemented{} } // PutObjectPart puts a part of object in bucket -func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) { +func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, err error) { logger.LogIf(ctx, NotImplemented{}) return pi, NotImplemented{} } @@ -131,7 +131,7 @@ func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, // CopyObject copies a blob from source container to destination container. func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string, - srcInfo ObjectInfo) (objInfo ObjectInfo, err error) { + srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) { logger.LogIf(ctx, NotImplemented{}) return objInfo, NotImplemented{} } diff --git a/cmd/gateway/azure/gateway-azure.go b/cmd/gateway/azure/gateway-azure.go index c52c11699..3cc2b4885 100644 --- a/cmd/gateway/azure/gateway-azure.go +++ b/cmd/gateway/azure/gateway-azure.go @@ -621,7 +621,7 @@ func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { // startOffset cannot be negative. if startOffset < 0 { return azureToObjectError(minio.InvalidRange{}, bucket, object) @@ -653,7 +653,7 @@ func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, sta // GetObjectInfo - reads blob metadata properties and replies back minio.ObjectInfo, // uses zure equivalent GetBlobProperties. -func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) { +func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { blob := a.client.GetContainerReference(bucket).GetBlobReference(object) err = blob.GetProperties(nil) if err != nil { @@ -674,7 +674,7 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string) // PutObject - Create a new blob with the incoming data, // uses Azure equivalent CreateBlockBlobFromReader. -func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { +func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { blob := a.client.GetContainerReference(bucket).GetBlobReference(object) blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata) if err != nil { @@ -684,12 +684,12 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, dat if err != nil { return objInfo, azureToObjectError(err, bucket, object) } - return a.GetObjectInfo(ctx, bucket, object) + return a.GetObjectInfo(ctx, bucket, object, opts) } // CopyObject - Copies a blob from source container to destination container. // Uses Azure equivalent CopyBlob API. -func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { +func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL() destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject) azureMeta, props, err := s3MetaToAzureProperties(ctx, srcInfo.UserDefined) @@ -716,7 +716,7 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des if err != nil { return objInfo, azureToObjectError(err, srcBucket, srcObject) } - return a.GetObjectInfo(ctx, destBucket, destObject) + return a.GetObjectInfo(ctx, destBucket, destObject, dstOpts) } // DeleteObject - Deletes a blob on azure container, uses Azure @@ -763,7 +763,7 @@ func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, obje } // NewMultipartUpload - Use Azure equivalent CreateBlockBlob. -func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) { +func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts minio.ObjectOptions) (uploadID string, err error) { uploadID, err = getAzureUploadID() if err != nil { logger.LogIf(ctx, err) @@ -787,7 +787,7 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st } // PutObjectPart - Use Azure equivalent PutBlockWithLength. -func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info minio.PartInfo, err error) { +func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (info minio.PartInfo, err error) { if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil { return info, err } @@ -1035,7 +1035,7 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje return objInfo, azureToObjectError(err, bucket, object) } } - return a.GetObjectInfo(ctx, bucket, object) + return a.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{}) } // SetBucketPolicy - Azure supports three types of container policies: diff --git a/cmd/gateway/b2/gateway-b2.go b/cmd/gateway/b2/gateway-b2.go index 4c3366f99..dfe5d93e1 100644 --- a/cmd/gateway/b2/gateway-b2.go +++ b/cmd/gateway/b2/gateway-b2.go @@ -400,7 +400,7 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { bkt, err := l.Bucket(ctx, bucket) if err != nil { return err @@ -417,7 +417,7 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, } // GetObjectInfo reads object info and replies back ObjectInfo -func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) { +func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { bkt, err := l.Bucket(ctx, bucket) if err != nil { return objInfo, err @@ -508,7 +508,7 @@ func (nb *Reader) Read(p []byte) (int, error) { } // PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB. -func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { +func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { bkt, err := l.Bucket(ctx, bucket) if err != nil { return objInfo, err @@ -608,7 +608,7 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre // Each large file must consist of at least 2 parts, and all of the parts except the // last one must be at least 5MB in size. The last part must contain at least one byte. // For more information - https://www.backblaze.com/b2/docs/large_files.html -func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (string, error) { +func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (string, error) { var uploadID string bkt, err := l.Bucket(ctx, bucket) if err != nil { @@ -627,7 +627,7 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec } // PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API. -func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi minio.PartInfo, err error) { +func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) { bkt, err := l.Bucket(ctx, bucket) if err != nil { return pi, err @@ -723,7 +723,7 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, return oi, b2ToObjectError(err, bucket, object, uploadID) } - return l.GetObjectInfo(ctx, bucket, object) + return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{}) } // SetBucketPolicy - B2 supports 2 types of bucket policies: diff --git a/cmd/gateway/gcs/gateway-gcs.go b/cmd/gateway/gcs/gateway-gcs.go index 4b838ee96..451499fbb 100644 --- a/cmd/gateway/gcs/gateway-gcs.go +++ b/cmd/gateway/gcs/gateway-gcs.go @@ -742,7 +742,7 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { @@ -831,7 +831,7 @@ func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAt } // GetObjectInfo - reads object info and replies back ObjectInfo -func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) { +func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) { // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { @@ -849,7 +849,7 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st } // PutObject - Create a new object with the incoming data, -func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string) (minio.ObjectInfo, error) { +func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) { // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // otherwise gcs will just return object not exist in case of non-existing bucket if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { @@ -888,7 +888,7 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d // CopyObject - Copies a blob from source container to destination container. func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string, - srcInfo minio.ObjectInfo) (minio.ObjectInfo, error) { + srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (minio.ObjectInfo, error) { src := l.client.Bucket(srcBucket).Object(srcObject) dst := l.client.Bucket(destBucket).Object(destObject) @@ -917,7 +917,7 @@ func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object str } // NewMultipartUpload - upload object in multiple parts -func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string) (uploadID string, err error) { +func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) { // generate new uploadid uploadID = minio.MustGetUUID() @@ -1030,7 +1030,7 @@ func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key } // PutObjectPart puts a part of object in bucket -func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader) (minio.PartInfo, error) { +func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader, opts minio.ObjectOptions) (minio.PartInfo, error) { if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil { return minio.PartInfo{}, err } diff --git a/cmd/gateway/manta/gateway-manta.go b/cmd/gateway/manta/gateway-manta.go index 8cc3b7b38..32341854c 100644 --- a/cmd/gateway/manta/gateway-manta.go +++ b/cmd/gateway/manta/gateway-manta.go @@ -513,7 +513,7 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti // indicates the total length of the object. // // https://apidocs.joyent.com/manta/api.html#GetObject -func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { // Start offset cannot be negative. if startOffset < 0 { logger.LogIf(ctx, fmt.Errorf("Unexpected error")) @@ -545,7 +545,7 @@ func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, st // uses Triton equivalent GetBlobProperties. // // https://apidocs.joyent.com/manta/api.html#GetObject -func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) { +func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { info, err := t.client.Objects().GetInfo(ctx, &storage.GetInfoInput{ ObjectPath: path.Join(mantaRoot, bucket, object), }) @@ -583,7 +583,7 @@ func (d dummySeeker) Seek(offset int64, whence int) (int64, error) { // CreateBlockBlobFromReader. // // https://apidocs.joyent.com/manta/api.html#PutObject -func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { +func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{ ContentLength: uint64(data.Size()), ObjectPath: path.Join(mantaRoot, bucket, object), @@ -602,14 +602,14 @@ func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, da return objInfo, err } - return t.GetObjectInfo(ctx, bucket, object) + return t.GetObjectInfo(ctx, bucket, object, opts) } // CopyObject - Copies a blob from source container to destination container. // Uses Manta Snaplinks API. // // https://apidocs.joyent.com/manta/api.html#PutSnapLink -func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { +func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { if err = t.client.SnapLinks().Put(ctx, &storage.PutSnapLinkInput{ SourcePath: path.Join(mantaRoot, srcBucket, srcObject), LinkPath: path.Join(mantaRoot, destBucket, destObject), @@ -618,7 +618,7 @@ func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, de return objInfo, err } - return t.GetObjectInfo(ctx, destBucket, destObject) + return t.GetObjectInfo(ctx, destBucket, destObject, dstOpts) } // DeleteObject - Delete a blob in Manta, uses Triton equivalent DeleteBlob API. diff --git a/cmd/gateway/oss/gateway-oss.go b/cmd/gateway/oss/gateway-oss.go index 4a7c4c5a9..e01a8a8e0 100644 --- a/cmd/gateway/oss/gateway-oss.go +++ b/cmd/gateway/oss/gateway-oss.go @@ -552,7 +552,7 @@ func ossGetObject(ctx context.Context, client *oss.Client, bucket, key string, s // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error { +func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { return ossGetObject(ctx, l.Client, bucket, key, startOffset, length, writer, etag) } @@ -602,7 +602,7 @@ func ossGetObjectInfo(ctx context.Context, client *oss.Client, bucket, object st } // GetObjectInfo reads object info and replies back ObjectInfo. -func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) { +func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { return ossGetObjectInfo(ctx, l.Client, bucket, object) } @@ -630,12 +630,12 @@ func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string } // PutObject creates a new object with the incoming data. -func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { +func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { return ossPutObject(ctx, l.Client, bucket, object, data, metadata) } // CopyObject copies an object from source bucket to a destination bucket. -func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { +func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { bkt, err := l.Client.Bucket(srcBucket) if err != nil { logger.LogIf(ctx, err) @@ -659,7 +659,7 @@ func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu logger.LogIf(ctx, err) return objInfo, ossToObjectError(err, srcBucket, srcObject) } - return l.GetObjectInfo(ctx, dstBucket, dstObject) + return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts) } // DeleteObject deletes a blob in bucket. @@ -725,7 +725,7 @@ func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, k } // NewMultipartUpload upload object in multiple parts. -func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) { +func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) { bkt, err := l.Client.Bucket(bucket) if err != nil { logger.LogIf(ctx, err) @@ -748,7 +748,7 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri } // PutObjectPart puts a part of object in bucket. -func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, err error) { +func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) { bkt, err := l.Client.Bucket(bucket) if err != nil { logger.LogIf(ctx, err) @@ -828,7 +828,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par // CopyObjectPart creates a part in a multipart upload by copying // existing object or a part of it. func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, - partID int, startOffset, length int64, srcInfo minio.ObjectInfo) (p minio.PartInfo, err error) { + partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) { bkt, err := l.Client.Bucket(destBucket) if err != nil { @@ -954,7 +954,7 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object return oi, ossToObjectError(err, bucket, object) } - return l.GetObjectInfo(ctx, bucket, object) + return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{}) } // SetBucketPolicy sets policy on bucket. diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index 851e3a07d..6e51ef3eb 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -307,12 +307,14 @@ func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, o minio.ObjectOptions) error { if length < 0 && length != -1 { return minio.ErrorRespToObjectError(minio.InvalidRange{}, bucket, key) } opts := miniogo.GetObjectOptions{} + opts.ServerSideEncryption = o.ServerSideEncryption + if startOffset >= 0 && length >= 0 { if err := opts.SetRange(startOffset, startOffset+length-1); err != nil { logger.LogIf(ctx, err) @@ -333,8 +335,8 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st } // GetObjectInfo reads object info and replies back ObjectInfo -func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) { - oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{}) +func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { + oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{miniogo.GetObjectOptions{ServerSideEncryption: opts.ServerSideEncryption}}) if err != nil { return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object) } @@ -343,8 +345,8 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str } // PutObject creates a new object with the incoming data, -func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { - oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata)) +func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { + oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata), opts.ServerSideEncryption) if err != nil { return objInfo, minio.ErrorRespToObjectError(err, bucket, object) } @@ -353,7 +355,7 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, } // CopyObject copies an object from source bucket to a destination bucket. -func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { +func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { // Set this header such that following CopyObject() always sets the right metadata on the destination. // metadata input is already a trickled down value from interpreting x-amz-metadata-directive at // handler layer. So what we have right now is supposed to be applied on the destination object anyways. @@ -363,7 +365,7 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil { return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject) } - return l.GetObjectInfo(ctx, dstBucket, dstObject) + return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts) } // DeleteObject deletes a blob in bucket @@ -387,9 +389,9 @@ func (l *s3Objects) ListMultipartUploads(ctx context.Context, bucket string, pre } // NewMultipartUpload upload object in multiple parts -func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) { +func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) { // Create PutObject options - opts := miniogo.PutObjectOptions{UserMetadata: metadata} + opts := miniogo.PutObjectOptions{UserMetadata: metadata, ServerSideEncryption: o.ServerSideEncryption} uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts) if err != nil { return uploadID, minio.ErrorRespToObjectError(err, bucket, object) @@ -398,8 +400,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec } // PutObjectPart puts a part of object in bucket -func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, e error) { - info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString()) +func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) { + info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), opts.ServerSideEncryption) if err != nil { return pi, minio.ErrorRespToObjectError(err, bucket, object) } @@ -410,7 +412,7 @@ func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object str // CopyObjectPart creates a part in a multipart upload by copying // existing object or a part of it. func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, - partID int, startOffset, length int64, srcInfo minio.ObjectInfo) (p minio.PartInfo, err error) { + partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) { srcInfo.UserDefined = map[string]string{ "x-amz-copy-source-if-match": srcInfo.ETag, @@ -448,7 +450,7 @@ func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, return oi, minio.ErrorRespToObjectError(err, bucket, object) } - return l.GetObjectInfo(ctx, bucket, object) + return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{}) } // SetBucketPolicy sets policy on bucket diff --git a/cmd/gateway/sia/gateway-sia.go b/cmd/gateway/sia/gateway-sia.go index 8695b9b85..3fddbe8ac 100644 --- a/cmd/gateway/sia/gateway-sia.go +++ b/cmd/gateway/sia/gateway-sia.go @@ -431,7 +431,7 @@ func (s *siaObjects) ListObjects(ctx context.Context, bucket string, prefix stri return loi, nil } -func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { dstFile := path.Join(s.TempDir, minio.MustGetUUID()) defer os.Remove(dstFile) @@ -512,7 +512,7 @@ func (s *siaObjects) findSiaObject(ctx context.Context, bucket, object string) ( } // GetObjectInfo reads object info and replies back ObjectInfo -func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) { +func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) { so, err := s.findSiaObject(ctx, bucket, object) if err != nil { return minio.ObjectInfo{}, err @@ -529,7 +529,7 @@ func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object st } // PutObject creates a new object with the incoming data, -func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { +func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { srcFile := path.Join(s.TempDir, minio.MustGetUUID()) writer, err := os.Create(srcFile) if err != nil { diff --git a/cmd/object-api-deleteobject_test.go b/cmd/object-api-deleteobject_test.go index 7992e2ed3..4047292b8 100644 --- a/cmd/object-api-deleteobject_test.go +++ b/cmd/object-api-deleteobject_test.go @@ -93,7 +93,7 @@ func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) { for _, object := range testCase.objectToUploads { md5Bytes := md5.Sum([]byte(object.content)) _, err = obj.PutObject(context.Background(), testCase.bucketName, object.name, mustGetHashReader(t, bytes.NewBufferString(object.content), - int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), nil) + int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), nil, ObjectOptions{}) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } diff --git a/cmd/object-api-getobject_test.go b/cmd/object-api-getobject_test.go index 6434f1392..29105388e 100644 --- a/cmd/object-api-getobject_test.go +++ b/cmd/object-api-getobject_test.go @@ -75,7 +75,7 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) { // iterate through the above set of inputs and upkoad the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData) + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{}) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -155,7 +155,7 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) { } for i, testCase := range testCases { - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "") + err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{}) if err != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) } @@ -220,7 +220,7 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [ // iterate through the above set of inputs and upkoad the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData) + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{}) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -265,7 +265,7 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [ } } - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "") + err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{}) if err != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) } @@ -333,7 +333,7 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str // iterate through the above set of inputs and upkoad the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData) + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{}) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -418,7 +418,7 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str } for i, testCase := range testCases { - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "") + err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{}) if err != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) } diff --git a/cmd/object-api-getobjectinfo_test.go b/cmd/object-api-getobjectinfo_test.go index 69b0695c8..1ecfdebce 100644 --- a/cmd/object-api-getobjectinfo_test.go +++ b/cmd/object-api-getobjectinfo_test.go @@ -34,15 +34,14 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) { if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } - - // Put a regular object - _, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetHashReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), nil) + opts := ObjectOptions{} + _, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetHashReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), nil, opts) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } // Put an empty directory - _, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/empty-dir/", mustGetHashReader(t, bytes.NewBufferString(""), int64(len("")), "", ""), nil) + _, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/empty-dir/", mustGetHashReader(t, bytes.NewBufferString(""), int64(len("")), "", ""), nil, opts) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -83,7 +82,7 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) { {"test-getobjectinfo", "Asia/empty-dir/", resultCases[1], nil, true}, } for i, testCase := range testCases { - result, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName) + result, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName, opts) if err != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) } diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 066da89e1..56294f9ac 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -20,11 +20,17 @@ import ( "context" "io" + "github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio/pkg/hash" "github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/policy" ) +// ObjectOptions represents object options for ObjectLayer operations +type ObjectOptions struct { + ServerSideEncryption encrypt.ServerSide +} + // ObjectLayer implements primitives for object API layer. type ObjectLayer interface { // Storage operations. @@ -40,18 +46,19 @@ type ObjectLayer interface { ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) // Object operations. - GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) - GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) - PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) - CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) + + GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) + GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) + PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) + CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) DeleteObject(ctx context.Context, bucket, object string) error // Multipart operations. ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) - NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) + NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, - startOffset int64, length int64, srcInfo ObjectInfo) (info PartInfo, err error) - PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) + startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error) + PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 78c8ffb85..468dd34e5 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -71,7 +71,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { for _, object := range testObjects { md5Bytes := md5.Sum([]byte(object.content)) _, err = obj.PutObject(context.Background(), testBuckets[0], object.name, mustGetHashReader(t, bytes.NewBufferString(object.content), - int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), object.meta) + int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), object.meta, ObjectOptions{}) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -617,7 +617,7 @@ func BenchmarkListObjects(b *testing.B) { // Insert objects to be listed and benchmarked later. for i := 0; i < 20000; i++ { key := "obj" + strconv.Itoa(i) - _, err = obj.PutObject(context.Background(), bucket, key, mustGetHashReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, key, mustGetHashReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), nil, ObjectOptions{}) if err != nil { b.Fatal(err) } diff --git a/cmd/object-api-multipart_test.go b/cmd/object-api-multipart_test.go index bcc233d3a..6cd2171cc 100644 --- a/cmd/object-api-multipart_test.go +++ b/cmd/object-api-multipart_test.go @@ -40,15 +40,15 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr bucket := "minio-bucket" object := "minio-object" - - _, err := obj.NewMultipartUpload(context.Background(), "--", object, nil) + opts := ObjectOptions{} + _, err := obj.NewMultipartUpload(context.Background(), "--", object, nil, opts) if err == nil { t.Fatalf("%s: Expected to fail since bucket name is invalid.", instanceType) } errMsg := "Bucket not found: minio-bucket" // opearation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist. - _, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil) + _, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts) if err == nil { t.Fatalf("%s: Expected to fail since the NewMultipartUpload is intialized on a non-existent bucket.", instanceType) } @@ -63,12 +63,12 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr t.Fatalf("%s : %s", instanceType, err.Error()) } - _, err = obj.NewMultipartUpload(context.Background(), bucket, "\\", nil) + _, err = obj.NewMultipartUpload(context.Background(), bucket, "\\", nil, opts) if err == nil { t.Fatalf("%s: Expected to fail since object name is invalid.", instanceType) } - uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -94,7 +94,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test bucket := "minio-bucket" object := "minio-object" - + opts := ObjectOptions{} // Create bucket before intiating NewMultipartUpload. err := obj.MakeBucketWithLocation(context.Background(), bucket, "") if err != nil { @@ -102,7 +102,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test t.Fatalf("%s : %s", instanceType, err.Error()) } - uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -148,7 +148,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE t.Fatalf("%s : %s", instanceType, err.Error()) } - _, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil) + _, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil, ObjectOptions{}) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -184,7 +184,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [ } // Initiate Multipart Upload on the above created bucket. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, ObjectOptions{}) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -221,7 +221,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [ sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum)) + _, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{}) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -235,7 +235,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [ // Object part upload should fail with quorum not available. testCase := createPartCases[len(createPartCases)-1] - _, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum)) + _, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{}) if err == nil { t.Fatalf("Test %s: expected to fail but passed instead", instanceType) } @@ -256,7 +256,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH // Generating cases for which the PutObjectPart fails. bucket := "minio-bucket" object := "minio-object" - + opts := ObjectOptions{} // Create bucket before intiating NewMultipartUpload. err := obj.MakeBucketWithLocation(context.Background(), bucket, "") if err != nil { @@ -264,7 +264,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH t.Fatalf("%s : %s", instanceType, err.Error()) } // Initiate Multipart Upload on the above created bucket. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -354,7 +354,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH // Validate all the test cases. for i, testCase := range testCases { - actualInfo, actualErr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, testCase.inputSHA256)) + actualInfo, actualErr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, testCase.inputSHA256), opts) // All are test cases above are expected to fail. if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) @@ -389,7 +389,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan bucketNames := []string{"minio-bucket", "minio-2-bucket", "minio-3-bucket"} objectNames := []string{"minio-object-1.txt", "minio-object.txt", "neymar-1.jpeg", "neymar.jpeg", "parrot-1.png", "parrot.png"} uploadIDs := []string{} - + opts := ObjectOptions{} // bucketnames[0]. // objectNames[0]. // uploadIds [0]. @@ -400,7 +400,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan t.Fatalf("%s : %s", instanceType, err.Error()) } // Initiate Multipart Upload on the above created bucket. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -420,7 +420,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan for i := 0; i < 3; i++ { // Initiate Multipart Upload on bucketNames[1] for the same object 3 times. // Used to test the listing for the case of multiple uploadID's for a given object. - uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[1], objectNames[0], nil) + uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[1], objectNames[0], nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -442,7 +442,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan // Used to test the listing for the case of multiple objects for a given bucket. for i := 0; i < 6; i++ { var uploadID string - uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[2], objectNames[i], nil) + uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[2], objectNames[i], nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -488,7 +488,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum)) + _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -1273,8 +1273,9 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks // Failed to create newbucket, abort. t.Fatalf("%s : %s", instanceType, err.Error()) } + opts := ObjectOptions{} // Initiate Multipart Upload on the above created bucket. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -1308,7 +1309,7 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum)) + _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -1507,7 +1508,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler) bucketNames := []string{"minio-bucket", "minio-2-bucket"} objectNames := []string{"minio-object-1.txt"} uploadIDs := []string{} - + opts := ObjectOptions{} // bucketnames[0]. // objectNames[0]. // uploadIds [0]. @@ -1518,7 +1519,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler) t.Fatalf("%s : %s", instanceType, err.Error()) } // Initiate Multipart Upload on the above created bucket. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -1549,7 +1550,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler) sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum)) + _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts) if err != nil { t.Fatalf("%s : %s", instanceType, err.Error()) } @@ -1763,8 +1764,9 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T // Failed to create newbucket, abort. t.Fatalf("%s : %s", instanceType, err) } + opts := ObjectOptions{} // Initiate Multipart Upload on the above created bucket. - uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], map[string]string{"X-Amz-Meta-Id": "id"}) + uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], map[string]string{"X-Amz-Meta-Id": "id"}, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err) @@ -1799,7 +1801,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, part := range parts { - _, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, sha256sum)) + _, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, sha256sum), opts) if err != nil { t.Fatalf("%s : %s", instanceType, err) } diff --git a/cmd/object-api-putobject_test.go b/cmd/object-api-putobject_test.go index 353556ac5..2eb0072f0 100644 --- a/cmd/object-api-putobject_test.go +++ b/cmd/object-api-putobject_test.go @@ -162,7 +162,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl } for i, testCase := range testCases { - objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta) + objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta, ObjectOptions{}) if actualErr != nil && testCase.expectedError == nil { t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error()) } @@ -235,7 +235,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di sha256sum := "" for i, testCase := range testCases { - objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta) + objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{}) if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) } @@ -284,7 +284,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di InsufficientWriteQuorum{}, } - _, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta) + _, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{}) if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", len(testCases)+1, instanceType, actualErr.Error()) } @@ -316,7 +316,7 @@ func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disk data := []byte("hello, world") // Create object. - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{}) if err != nil { // Failed to create object, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -347,9 +347,9 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str // Failed to create newbucket, abort. t.Fatalf("%s : %s", instanceType, err.Error()) } - + opts := ObjectOptions{} // Initiate Multipart Upload on the above created bucket. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -361,7 +361,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str md5Writer.Write(fiveMBBytes) etag1 := hex.EncodeToString(md5Writer.Sum(nil)) sha256sum := "" - _, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum)) + _, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum), opts) if err != nil { // Failed to upload object part, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -372,7 +372,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str md5Writer = md5.New() md5Writer.Write(data) etag2 := hex.EncodeToString(md5Writer.Sum(nil)) - _, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), etag2, sha256sum)) + _, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), etag2, sha256sum), opts) if err != nil { // Failed to upload object part, abort. t.Fatalf("%s : %s", instanceType, err.Error()) diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index f9ff7afc0..d53b84c43 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -89,7 +89,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r if api.CacheAPI() != nil { getObjectInfo = api.CacheAPI().GetObjectInfo } - + opts := ObjectOptions{} if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone { if getRequestAuthType(r) == authTypeAnonymous { // As per "Permission" section in @@ -106,7 +106,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r ConditionValues: getConditionValues(r, ""), IsOwner: false, }) { - _, err := getObjectInfo(ctx, bucket, object) + _, err := getObjectInfo(ctx, bucket, object, opts) if toAPIErrorCode(err) == ErrNoSuchKey { s3Error = ErrNoSuchKey } @@ -125,7 +125,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r return } - objInfo, err := getObjectInfo(ctx, bucket, object) + objInfo, err := getObjectInfo(ctx, bucket, object, opts) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return @@ -187,7 +187,6 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r if api.CacheAPI() != nil && !crypto.SSEC.IsRequested(r.Header) { getObject = api.CacheAPI().GetObject } - reader, pipewriter := io.Pipe() // Get the object. @@ -212,7 +211,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r } go func() { defer reader.Close() - if gerr := getObject(ctx, bucket, object, 0, objInfo.Size, writer, objInfo.ETag); gerr != nil { + if gerr := getObject(ctx, bucket, object, 0, objInfo.Size, writer, objInfo.ETag, opts); gerr != nil { pipewriter.CloseWithError(gerr) return } @@ -279,6 +278,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] + opts := ObjectOptions{} getObjectInfo := objectAPI.GetObjectInfo if api.CacheAPI() != nil { @@ -297,7 +297,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req ConditionValues: getConditionValues(r, ""), IsOwner: false, }) { - _, err := getObjectInfo(ctx, bucket, object) + _, err := getObjectInfo(ctx, bucket, object, opts) if toAPIErrorCode(err) == ErrNoSuchKey { s3Error = ErrNoSuchKey } @@ -307,7 +307,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req return } - objInfo, err := getObjectInfo(ctx, bucket, object) + objInfo, err := getObjectInfo(ctx, bucket, object, opts) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return @@ -390,7 +390,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req } // Reads the object at startOffset and writes to mw. - if err = getObject(ctx, bucket, object, startOffset, length, httpWriter, objInfo.ETag); err != nil { + if err = getObject(ctx, bucket, object, startOffset, length, httpWriter, objInfo.ETag, opts); err != nil { if !httpWriter.HasWritten() && !statusCodeWritten { // write error response only if no data or headers has been written to client yet writeErrorResponse(w, toAPIErrorCode(err), r.URL) } @@ -449,6 +449,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re getObjectInfo = api.CacheAPI().GetObjectInfo } + opts := ObjectOptions{} if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone { if getRequestAuthType(r) == authTypeAnonymous { // As per "Permission" section in https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html @@ -461,7 +462,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re ConditionValues: getConditionValues(r, ""), IsOwner: false, }) { - _, err := getObjectInfo(ctx, bucket, object) + _, err := getObjectInfo(ctx, bucket, object, opts) if toAPIErrorCode(err) == ErrNoSuchKey { s3Error = ErrNoSuchKey } @@ -471,7 +472,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re return } - objInfo, err := getObjectInfo(ctx, bucket, object) + objInfo, err := getObjectInfo(ctx, bucket, object, opts) if err != nil { writeErrorResponseHeadersOnly(w, toAPIErrorCode(err)) return @@ -603,8 +604,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return } + var srcOpts, dstOpts ObjectOptions cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) - srcInfo, err := objectAPI.GetObjectInfo(ctx, srcBucket, srcObject) + srcInfo, err := objectAPI.GetObjectInfo(ctx, srcBucket, srcObject, srcOpts) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return @@ -612,7 +614,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // Deny if WORM is enabled if globalWORMEnabled { - if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject); err == nil { + if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts); err == nil { writeErrorResponse(w, ErrMethodNotAllowed, r.URL) return } @@ -799,7 +801,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re var dstRecords []dns.SrvRecord if dstRecords, err = globalDNSConfig.Get(dstBucket); err == nil { go func() { - if gerr := objectAPI.GetObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag); gerr != nil { + if gerr := objectAPI.GetObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil { pipeWriter.CloseWithError(gerr) writeErrorResponse(w, ErrInternalError, r.URL) return @@ -816,7 +818,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re writeErrorResponse(w, ErrInternalError, r.URL) return } - remoteObjInfo, rerr := client.PutObject(dstBucket, dstObject, srcInfo.Reader, srcInfo.Size, "", "", srcInfo.UserDefined) + remoteObjInfo, rerr := client.PutObject(dstBucket, dstObject, srcInfo.Reader, srcInfo.Size, "", "", srcInfo.UserDefined, dstOpts.ServerSideEncryption) if rerr != nil { pipeWriter.CloseWithError(rerr) writeErrorResponse(w, ErrInternalError, r.URL) @@ -828,7 +830,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } else { // Copy source object to destination, if source and destination // object is same then only metadata is updated. - objInfo, err = objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo) + objInfo, err = objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) if err != nil { pipeWriter.CloseWithError(err) writeErrorResponse(w, toAPIErrorCode(err), r.URL) @@ -1009,10 +1011,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req writeErrorResponse(w, toAPIErrorCode(err), r.URL) return } - + opts := ObjectOptions{} // Deny if WORM is enabled if globalWORMEnabled { - if _, err = objectAPI.GetObjectInfo(ctx, bucket, object); err == nil { + if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil { writeErrorResponse(w, ErrMethodNotAllowed, r.URL) return } @@ -1039,7 +1041,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req } // Create the object.. - objInfo, err := putObject(ctx, bucket, object, hashReader, metadata) + objInfo, err := putObject(ctx, bucket, object, hashReader, metadata, opts) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return @@ -1095,6 +1097,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r vars := mux.Vars(r) bucket := vars["bucket"] object := vars["object"] + opts := ObjectOptions{} if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone { writeErrorResponse(w, s3Error, r.URL) @@ -1103,7 +1106,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r // Deny if WORM is enabled if globalWORMEnabled { - if _, err := objectAPI.GetObjectInfo(ctx, bucket, object); err == nil { + if _, err := objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil { writeErrorResponse(w, ErrMethodNotAllowed, r.URL) return } @@ -1148,7 +1151,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r if api.CacheAPI() != nil && !hasServerSideEncryptionHeader(r.Header) { newMultipartUpload = api.CacheAPI().NewMultipartUpload } - uploadID, err := newMultipartUpload(ctx, bucket, object, metadata) + uploadID, err := newMultipartUpload(ctx, bucket, object, metadata, opts) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return @@ -1212,8 +1215,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt writeErrorResponse(w, ErrInvalidMaxParts, r.URL) return } - - srcInfo, err := objectAPI.GetObjectInfo(ctx, srcBucket, srcObject) + var srcOpts, dstOpts ObjectOptions + srcInfo, err := objectAPI.GetObjectInfo(ctx, srcBucket, srcObject, srcOpts) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return @@ -1221,7 +1224,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt // Deny if WORM is enabled if globalWORMEnabled { - if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject); err == nil { + if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts); err == nil { writeErrorResponse(w, ErrMethodNotAllowed, r.URL) return } @@ -1346,11 +1349,10 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt } } srcInfo.Writer = writer - // Copy source object to destination, if source and destination // object is same then only metadata is updated. partInfo, err := objectAPI.CopyObjectPart(ctx, srcBucket, srcObject, dstBucket, - dstObject, uploadID, partID, startOffset, getLength, srcInfo) + dstObject, uploadID, partID, startOffset, getLength, srcInfo, srcOpts, dstOpts) if err != nil { writeErrorResponse(w, toAPIErrorCode(err), r.URL) return @@ -1496,9 +1498,10 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http return } + opts := ObjectOptions{} // Deny if WORM is enabled if globalWORMEnabled { - if _, err = objectAPI.GetObjectInfo(ctx, bucket, object); err == nil { + if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil { writeErrorResponse(w, ErrMethodNotAllowed, r.URL) return } @@ -1559,7 +1562,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http if api.CacheAPI() != nil && !hasServerSideEncryptionHeader(r.Header) { putObjectPart = api.CacheAPI().PutObjectPart } - partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, hashReader) + partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, hashReader, opts) if err != nil { // Verify if the underlying error is signature mismatch. writeErrorResponse(w, toAPIErrorCode(err), r.URL) @@ -1597,7 +1600,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, // Deny if WORM is enabled if globalWORMEnabled { - if _, err := objectAPI.GetObjectInfo(ctx, bucket, object); err == nil { + if _, err := objectAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err == nil { writeErrorResponse(w, ErrMethodNotAllowed, r.URL) return } @@ -1672,7 +1675,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite // Deny if WORM is enabled if globalWORMEnabled { - if _, err := objectAPI.GetObjectInfo(ctx, bucket, object); err == nil { + if _, err := objectAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err == nil { writeErrorResponse(w, ErrMethodNotAllowed, r.URL) return } diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go index 91ae35cfd..47cfd73f1 100644 --- a/cmd/object-handlers_test.go +++ b/cmd/object-handlers_test.go @@ -77,7 +77,7 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string, // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData) + _, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{}) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -224,7 +224,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData) + _, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{}) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -746,13 +746,14 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam if err != nil { t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) } + opts := ObjectOptions{} if testCase.shouldPass { // Verify whether the bucket obtained object is same as the one created. if !bytes.Equal(testCase.expectedContent, actualContent) { t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent)) continue } - objInfo, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName) + objInfo, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName, opts) if err != nil { t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) } @@ -764,7 +765,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam t.Fatalf("Test %d: %s: ContentEncoding is set to \"%s\" which is unexpected, expected \"%s\"", i+1, instanceType, objInfo.ContentEncoding, expectedContentEncoding) } buffer := new(bytes.Buffer) - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(testCase.dataLen), buffer, objInfo.ETag) + err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(testCase.dataLen), buffer, objInfo.ETag, opts) if err != nil { t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) } @@ -787,6 +788,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a var err error objectName := "test-object" + opts := ObjectOptions{} // byte data for PutObject. bytesData := generateBytesData(6 * humanize.KiByte) @@ -936,7 +938,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a buffer := new(bytes.Buffer) // Fetch the object to check whether the content is same as the one uploaded via PutObject. - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "") + err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "", opts) if err != nil { t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) } @@ -979,7 +981,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a if testCase.expectedRespStatus == http.StatusOK { buffer := new(bytes.Buffer) // Fetch the object to check whether the content is same as the one uploaded via PutObject. - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "") + err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "", opts) if err != nil { t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) } @@ -1035,7 +1037,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam objectName := "test-object" var err error - + opts := ObjectOptions{} // set of byte data for PutObject. // object has to be created before running tests for Copy Object. // this is required even to assert the copied object, @@ -1060,7 +1062,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam for i, input := range putObjectInputs { // uploading the object. _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, - mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData) + mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -1073,7 +1075,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam // PutObjectPart API HTTP Handler has to be tested in isolation, // that is without any other handler being registered, // That's why NewMultipartUpload is initiated using ObjectLayer. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("Minio %s : %s", instanceType, err) @@ -1127,7 +1129,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam } var buf bytes.Buffer - if err = obj.GetObject(context.Background(), bucketName, testObject, 0, int64(len(bytesData[0].byteData)), &buf, ""); err != nil { + if err = obj.GetObject(context.Background(), bucketName, testObject, 0, int64(len(bytesData[0].byteData)), &buf, "", opts); err != nil { t.Fatalf("Test: %s reading completed file failed: %v", instanceType, err) } if !bytes.Equal(buf.Bytes(), bytesData[0].byteData) { @@ -1146,7 +1148,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri objectName := "test-object" var err error - + opts := ObjectOptions{} // set of byte data for PutObject. // object has to be created before running tests for Copy Object. // this is required even to assert the copied object, @@ -1170,7 +1172,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData) + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -1183,7 +1185,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri // PutObjectPart API HTTP Handler has to be tested in isolation, // that is without any other handler being registered, // That's why NewMultipartUpload is initiated using ObjectLayer. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("Minio %s : %s", instanceType, err) @@ -1474,7 +1476,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // object used for anonymous HTTP request test. anonObject := "anon-object" var err error - + opts := ObjectOptions{} // set of byte data for PutObject. // object has to be created before running tests for Copy Object. // this is required even to assert the copied object, @@ -1507,7 +1509,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData) + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -1776,7 +1778,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, if rec.Code == http.StatusOK { // See if the new object is formed. // testing whether the copy was successful. - err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "") + err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts) if err != nil { t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) } @@ -2092,6 +2094,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s var err error + var opts ObjectOptions // object used for the test. objectName := "test-object-new-multipart" @@ -2102,7 +2105,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s for i := 0; i < 2; i++ { // initiate new multipart uploadID. - uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil) + uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("Minio %s : %s", instanceType, err) @@ -2144,7 +2147,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s // Iterating over creatPartCases to generate multipart chunks. for _, part := range parts { _, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, - mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, "")) + mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""), opts) if err != nil { t.Fatalf("%s : %s", instanceType, err) } @@ -2440,7 +2443,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri credentials auth.Credentials, t *testing.T) { var err error - + opts := ObjectOptions{} // object used for the test. objectName := "test-object-new-multipart" @@ -2451,7 +2454,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri for i := 0; i < 2; i++ { // initiate new multipart uploadID. - uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil) + uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("Minio %s : %s", instanceType, err) @@ -2493,7 +2496,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri // Iterating over createPartCases to generate multipart chunks. for _, part := range parts { _, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, - mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, "")) + mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""), opts) if err != nil { t.Fatalf("%s : %s", instanceType, err) } @@ -2603,7 +2606,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string credentials auth.Credentials, t *testing.T) { var err error - + var opts ObjectOptions objectName := "test-object" // Object used for anonymous API request test. anonObjectName := "test-anon-obj" @@ -2631,7 +2634,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string // iterate through the above set of inputs and upload the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData) + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -2921,11 +2924,11 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin // Initiate Multipart upload for testing PutObjectPartHandler. testObject := "testobject" - + var opts ObjectOptions // PutObjectPart API HTTP Handler has to be tested in isolation, // that is without any other handler being registered, // That's why NewMultipartUpload is initiated using ObjectLayer. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("Minio %s : %s", instanceType, err) @@ -3324,11 +3327,11 @@ func TestAPIListObjectPartsHandler(t *testing.T) { func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, credentials auth.Credentials, t *testing.T) { testObject := "testobject" - + var opts ObjectOptions // PutObjectPart API HTTP Handler has to be tested in isolation, // that is without any other handler being registered, // That's why NewMultipartUpload is initiated using ObjectLayer. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("Minio %s : %s", instanceType, err) @@ -3337,7 +3340,7 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str uploadIDCopy := uploadID // create an object Part, will be used to test list object parts. - _, err = obj.PutObjectPart(context.Background(), bucketName, testObject, uploadID, 1, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "5d41402abc4b2a76b9719d911017c592", "")) + _, err = obj.PutObjectPart(context.Background(), bucketName, testObject, uploadID, 1, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "5d41402abc4b2a76b9719d911017c592", ""), opts) if err != nil { t.Fatalf("Minio %s : %s.", instanceType, err) } diff --git a/cmd/object_api_suite_test.go b/cmd/object_api_suite_test.go index 49b1544ae..a086d6563 100644 --- a/cmd/object_api_suite_test.go +++ b/cmd/object_api_suite_test.go @@ -90,11 +90,12 @@ func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(t *testing.T) { // Tests validate creation of part files during Multipart operation. func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) { + var opts ObjectOptions err := obj.MakeBucketWithLocation(context.Background(), "bucket", "") if err != nil { t.Fatalf("%s: %s", instanceType, err) } - uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -105,7 +106,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr expectedETaghex := getMD5Hash(data) var calcPartInfo PartInfo - calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBuffer(data), int64(len(data)), expectedETaghex, "")) + calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBuffer(data), int64(len(data)), expectedETaghex, ""), opts) if err != nil { t.Errorf("%s: %s", instanceType, err) } @@ -133,11 +134,12 @@ func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(t *testing.T) { // Tests validate abortion of Multipart operation. func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHandler) { + var opts ObjectOptions err := obj.MakeBucketWithLocation(context.Background(), "bucket", "") if err != nil { t.Fatalf("%s: %s", instanceType, err) } - uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -155,7 +157,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan metadata["md5"] = expectedETaghex var calcPartInfo PartInfo - calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), expectedETaghex, "")) + calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), expectedETaghex, ""), opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -179,6 +181,7 @@ func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) { func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) { objects := make(map[string][]byte) err := obj.MakeBucketWithLocation(context.Background(), "bucket", "") + var opts ObjectOptions if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -196,7 +199,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH metadata := make(map[string]string) metadata["etag"] = expectedETaghex var objInfo ObjectInfo - objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), metadata) + objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), metadata, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -207,7 +210,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH for key, value := range objects { var byteBuffer bytes.Buffer - err = obj.GetObject(context.Background(), "bucket", key, 0, int64(len(value)), &byteBuffer, "") + err = obj.GetObject(context.Background(), "bucket", key, 0, int64(len(value)), &byteBuffer, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -215,7 +218,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH t.Errorf("%s: Mismatch of GetObject data with the expected one.", instanceType) } - objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", key) + objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", key, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -246,10 +249,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) { } uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." + var opts ObjectOptions // check before paging occurs. for i := 0; i < 5; i++ { key := "obj" + strconv.Itoa(i) - _, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -269,7 +273,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) { // check after paging occurs pages work. for i := 6; i <= 10; i++ { key := "obj" + strconv.Itoa(i) - _, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -286,11 +290,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) { } // check paging with prefix at end returns less objects. { - _, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } - _, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -328,11 +332,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) { // check delimited results with delimiter and prefix. { - _, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } - _, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -441,22 +445,23 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan t.Fatalf("%s: %s", instanceType, err) } + var opts ObjectOptions uploadContent := "The list of parts was not in ascending order. The parts list must be specified in order by part number." length := int64(len(uploadContent)) - _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } uploadContent = "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." length = int64(len(uploadContent)) - _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } var bytesBuffer bytes.Buffer - err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer, "") + err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -472,7 +477,8 @@ func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) { // Tests validate that bucket operation on non-existent bucket fails. func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t TestErrHandler) { - _, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetHashReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), nil) + var opts ObjectOptions + _, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetHashReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), nil, opts) if err == nil { t.Fatal("Expected error but found nil") } @@ -519,11 +525,12 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) { } var bytesBuffer1 bytes.Buffer - _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerEOF, length, "", ""), nil) + var opts ObjectOptions + _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerEOF, length, "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } - err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer1, "") + err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer1, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -532,11 +539,11 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) { } var bytesBuffer2 bytes.Buffer - _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerNoEOF, length, "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerNoEOF, length, "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } - err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer2, "") + err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer2, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -557,16 +564,17 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle t.Fatalf("%s: %s", instanceType, err) } + var opts ObjectOptions uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.` length := int64(len(uploadContent)) - _, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } var bytesBuffer bytes.Buffer - err = obj.GetObject(context.Background(), "bucket", "dir1/dir2/object", 0, length, &bytesBuffer, "") + err = obj.GetObject(context.Background(), "bucket", "dir1/dir2/object", 0, length, &bytesBuffer, "", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } @@ -703,7 +711,7 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE t.Fatalf("%s: %s", instanceType, err) } - _, err = obj.GetObjectInfo(context.Background(), "bucket", "dir1") + _, err = obj.GetObjectInfo(context.Background(), "bucket", "dir1", ObjectOptions{}) if err == nil { t.Fatalf("%s: Expected error but found nil", instanceType) } @@ -732,7 +740,8 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, } content := "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag." length := int64(len(content)) - _, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetHashReader(t, bytes.NewBufferString(content), length, "", ""), nil) + var opts ObjectOptions + _, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetHashReader(t, bytes.NewBufferString(content), length, "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) @@ -753,7 +762,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, } for i, testCase := range testCases { - _, expectedErr := obj.GetObjectInfo(context.Background(), bucketName, testCase.dir) + _, expectedErr := obj.GetObjectInfo(context.Background(), bucketName, testCase.dir, opts) if expectedErr != nil && expectedErr.Error() != testCase.err.Error() { t.Errorf("Test %d, %s: Expected error %s, got %s", i+1, instanceType, testCase.err, expectedErr) } @@ -771,13 +780,14 @@ func testContentType(obj ObjectLayer, instanceType string, t TestErrHandler) { if err != nil { t.Fatalf("%s: %s", instanceType, err) } + var opts ObjectOptions uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." // Test empty. - _, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil) + _, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } - objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", "minio.png") + objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", "minio.png", opts) if err != nil { t.Fatalf("%s: %s", instanceType, err) } diff --git a/cmd/post-policy_test.go b/cmd/post-policy_test.go index 8aa0a14d7..c4550d8f8 100644 --- a/cmd/post-policy_test.go +++ b/cmd/post-policy_test.go @@ -124,6 +124,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr // get random bucket name. bucketName := getRandomBucketName() + var opts ObjectOptions // Register the API end points with XL/FS object layer. apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"}) @@ -227,7 +228,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr } // When the operation is successful, check if sending metadata is successful too if rec.Code == http.StatusNoContent { - objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, testCase.objectName+"/upload.txt") + objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, testCase.objectName+"/upload.txt", opts) if err != nil { t.Error("Unexpected error: ", err) } @@ -427,6 +428,8 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t // Key specified in Form data keyName := "test/object" + var opts ObjectOptions + // The final name of the upload object targetObj := keyName + "/upload.txt" @@ -478,7 +481,7 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t } // Get the uploaded object info - info, err := obj.GetObjectInfo(context.Background(), bucketName, targetObj) + info, err := obj.GetObjectInfo(context.Background(), bucketName, targetObj, opts) if err != nil { t.Error("Unexpected error: ", err) } diff --git a/cmd/storage-class_test.go b/cmd/storage-class_test.go index dfbde5e5c..1a113bba2 100644 --- a/cmd/storage-class_test.go +++ b/cmd/storage-class_test.go @@ -175,6 +175,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin resetGlobalStorageEnvs() bucket := getRandomBucketName() + var opts ObjectOptions // make data with more than one part partCount := 3 data := bytes.Repeat([]byte("a"), int(globalPutPartSize)*partCount) @@ -188,7 +189,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin // Object for test case 1 - No StorageClass defined, no MetaData in PutObject object1 := "object1" - _, err = obj.PutObject(context.Background(), bucket, object1, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object1, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, opts) if err != nil { t.Fatalf("Failed to putObject %v", err) } @@ -199,7 +200,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin object2 := "object2" metadata2 := make(map[string]string) metadata2["x-amz-storage-class"] = reducedRedundancyStorageClass - _, err = obj.PutObject(context.Background(), bucket, object2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata2) + _, err = obj.PutObject(context.Background(), bucket, object2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata2, opts) if err != nil { t.Fatalf("Failed to putObject %v", err) } @@ -210,7 +211,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin object3 := "object3" metadata3 := make(map[string]string) metadata3["x-amz-storage-class"] = standardStorageClass - _, err = obj.PutObject(context.Background(), bucket, object3, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata3) + _, err = obj.PutObject(context.Background(), bucket, object3, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata3, opts) if err != nil { t.Fatalf("Failed to putObject %v", err) } @@ -226,7 +227,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin Scheme: "EC", } - _, err = obj.PutObject(context.Background(), bucket, object4, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata4) + _, err = obj.PutObject(context.Background(), bucket, object4, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata4, opts) if err != nil { t.Fatalf("Failed to putObject %v", err) } @@ -244,7 +245,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin Scheme: "EC", } - _, err = obj.PutObject(context.Background(), bucket, object5, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata5) + _, err = obj.PutObject(context.Background(), bucket, object5, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata5, opts) if err != nil { t.Fatalf("Failed to putObject %v", err) } @@ -262,7 +263,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin Scheme: "EC", } - _, err = obj.PutObject(context.Background(), bucket, object6, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata6) + _, err = obj.PutObject(context.Background(), bucket, object6, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata6, opts) if err != nil { t.Fatalf("Failed to putObject %v", err) } @@ -280,7 +281,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin Scheme: "EC", } - _, err = obj.PutObject(context.Background(), bucket, object7, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata7) + _, err = obj.PutObject(context.Background(), bucket, object7, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata7, opts) if err != nil { t.Fatalf("Failed to putObject %v", err) } diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 296d441e5..aa619188c 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -411,7 +411,7 @@ next: if !hasSuffix(objectName, slashSeparator) && objectName != "" { // Deny if WORM is enabled if globalWORMEnabled { - if _, err = objectAPI.GetObjectInfo(context.Background(), args.BucketName, objectName); err == nil { + if _, err = objectAPI.GetObjectInfo(context.Background(), args.BucketName, objectName, ObjectOptions{}); err == nil { return toJSONError(errMethodNotAllowed) } } @@ -653,16 +653,16 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) { writeWebErrorResponse(w, err) return } - + opts := ObjectOptions{} // Deny if WORM is enabled if globalWORMEnabled { - if _, err = objectAPI.GetObjectInfo(context.Background(), bucket, object); err == nil { + if _, err = objectAPI.GetObjectInfo(context.Background(), bucket, object, opts); err == nil { writeWebErrorResponse(w, errMethodNotAllowed) return } } - objInfo, err := putObject(context.Background(), bucket, object, hashReader, metadata) + objInfo, err := putObject(context.Background(), bucket, object, hashReader, metadata, opts) if err != nil { writeWebErrorResponse(w, err) return @@ -703,7 +703,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { return } } - + opts := ObjectOptions{} getObject := objectAPI.GetObject if web.CacheAPI() != nil { getObject = web.CacheAPI().GetObject @@ -712,7 +712,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { if web.CacheAPI() != nil { getObjectInfo = web.CacheAPI().GetObjectInfo } - objInfo, err := getObjectInfo(context.Background(), bucket, object) + objInfo, err := getObjectInfo(context.Background(), bucket, object, opts) if err != nil { writeWebErrorResponse(w, err) return @@ -745,7 +745,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { // Add content disposition. w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(object))) - if err = getObject(context.Background(), bucket, object, 0, -1, httpWriter, ""); err != nil { + if err = getObject(context.Background(), bucket, object, 0, -1, httpWriter, "", opts); err != nil { /// No need to print error, response writer already written to. return } @@ -814,10 +814,11 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) { if web.CacheAPI() != nil { getObjectInfo = web.CacheAPI().GetObjectInfo } + opts := ObjectOptions{} for _, object := range args.Objects { // Writes compressed object file to the response. zipit := func(objectName string) error { - info, err := getObjectInfo(context.Background(), args.BucketName, objectName) + info, err := getObjectInfo(context.Background(), args.BucketName, objectName, opts) if err != nil { return err } @@ -853,7 +854,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) { } } httpWriter := ioutil.WriteOnClose(writer) - if err = getObject(context.Background(), args.BucketName, objectName, 0, length, httpWriter, ""); err != nil { + if err = getObject(context.Background(), args.BucketName, objectName, 0, length, httpWriter, "", opts); err != nil { return err } if err = httpWriter.Close(); err != nil { diff --git a/cmd/web-handlers_test.go b/cmd/web-handlers_test.go index 27770f0f7..395f0b2d3 100644 --- a/cmd/web-handlers_test.go +++ b/cmd/web-handlers_test.go @@ -318,6 +318,8 @@ func testDeleteBucketWebHandler(obj ObjectLayer, instanceType string, t TestErrH } bucketName := getRandomBucketName() + var opts ObjectOptions + err = obj.MakeBucketWithLocation(context.Background(), bucketName, "") if err != nil { t.Fatalf("failed to create bucket: %s (%s)", err.Error(), instanceType) @@ -347,7 +349,7 @@ func testDeleteBucketWebHandler(obj ObjectLayer, instanceType string, t TestErrH for _, test := range testCases { if test.initWithObject { data := bytes.NewBufferString("hello") - _, err = obj.PutObject(context.Background(), test.bucketName, "object", mustGetHashReader(t, data, int64(data.Len()), "", ""), nil) + _, err = obj.PutObject(context.Background(), test.bucketName, "object", mustGetHashReader(t, data, int64(data.Len()), "", ""), nil, opts) // _, err = obj.PutObject(test.bucketName, "object", int64(data.Len()), data, nil, "") if err != nil { t.Fatalf("could not put object to %s, %s", test.bucketName, err.Error()) @@ -483,7 +485,7 @@ func testListObjectsWebHandler(obj ObjectLayer, instanceType string, t TestErrHa data := bytes.Repeat([]byte("a"), objectSize) metadata := map[string]string{"etag": "c9a34cfc85d982698c6ac89f76071abd"} - _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata) + _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{}) if err != nil { t.Fatalf("Was not able to upload an object, %v", err) @@ -587,14 +589,14 @@ func testRemoveObjectWebHandler(obj ObjectLayer, instanceType string, t TestErrH data := bytes.Repeat([]byte("a"), objectSize) metadata := map[string]string{"etag": "c9a34cfc85d982698c6ac89f76071abd"} - _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata) + _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{}) if err != nil { t.Fatalf("Was not able to upload an object, %v", err) } objectName = "a/object" metadata = map[string]string{"etag": "c9a34cfc85d982698c6ac89f76071abd"} - _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata) + _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{}) if err != nil { t.Fatalf("Was not able to upload an object, %v", err) } @@ -895,7 +897,7 @@ func testUploadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandler } var byteBuffer bytes.Buffer - err = obj.GetObject(context.Background(), bucketName, objectName, 0, int64(len(content)), &byteBuffer, "") + err = obj.GetObject(context.Background(), bucketName, objectName, 0, int64(len(content)), &byteBuffer, "", ObjectOptions{}) if err != nil { t.Fatalf("Failed, %v", err) } @@ -985,7 +987,7 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl content := []byte("temporary file's content") metadata := map[string]string{"etag": "01ce59706106fe5e02e7f55fffda7f34"} - _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(content), int64(len(content)), metadata["etag"], ""), metadata) + _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(content), int64(len(content)), metadata["etag"], ""), metadata, ObjectOptions{}) if err != nil { t.Fatalf("Was not able to upload an object, %v", err) } @@ -1069,6 +1071,7 @@ func TestWebHandlerDownloadZip(t *testing.T) { func testWebHandlerDownloadZip(obj ObjectLayer, instanceType string, t TestErrHandler) { apiRouter := initTestWebRPCEndPoint(obj) credentials := globalServerConfig.GetCredential() + var opts ObjectOptions authorization, err := authenticateURL(credentials.AccessKey, credentials.SecretKey) if err != nil { @@ -1087,9 +1090,9 @@ func testWebHandlerDownloadZip(obj ObjectLayer, instanceType string, t TestErrHa t.Fatalf("%s : %s", instanceType, err) } - obj.PutObject(context.Background(), bucket, "a/one", mustGetHashReader(t, strings.NewReader(fileOne), int64(len(fileOne)), "", ""), nil) - obj.PutObject(context.Background(), bucket, "a/b/two", mustGetHashReader(t, strings.NewReader(fileTwo), int64(len(fileTwo)), "", ""), nil) - obj.PutObject(context.Background(), bucket, "a/c/three", mustGetHashReader(t, strings.NewReader(fileThree), int64(len(fileThree)), "", ""), nil) + obj.PutObject(context.Background(), bucket, "a/one", mustGetHashReader(t, strings.NewReader(fileOne), int64(len(fileOne)), "", ""), nil, opts) + obj.PutObject(context.Background(), bucket, "a/b/two", mustGetHashReader(t, strings.NewReader(fileTwo), int64(len(fileTwo)), "", ""), nil, opts) + obj.PutObject(context.Background(), bucket, "a/c/three", mustGetHashReader(t, strings.NewReader(fileThree), int64(len(fileThree)), "", ""), nil, opts) test := func(token string) (int, []byte) { rec := httptest.NewRecorder() @@ -1174,7 +1177,7 @@ func testWebPresignedGetHandler(obj ObjectLayer, instanceType string, t TestErrH data := bytes.Repeat([]byte("a"), objectSize) metadata := map[string]string{"etag": "c9a34cfc85d982698c6ac89f76071abd"} - _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata) + _, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{}) if err != nil { t.Fatalf("Was not able to upload an object, %v", err) } diff --git a/cmd/xl-sets.go b/cmd/xl-sets.go index 2f0808c34..0022b8655 100644 --- a/cmd/xl-sets.go +++ b/cmd/xl-sets.go @@ -579,18 +579,18 @@ func (s *xlSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, err err // --- Object Operations --- // GetObject - reads an object from the hashedSet based on the object name. -func (s *xlSets) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { - return s.getHashedSet(object).GetObject(ctx, bucket, object, startOffset, length, writer, etag) +func (s *xlSets) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error { + return s.getHashedSet(object).GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts) } // PutObject - writes an object to hashedSet based on the object name. -func (s *xlSets) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { - return s.getHashedSet(object).PutObject(ctx, bucket, object, data, metadata) +func (s *xlSets) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + return s.getHashedSet(object).PutObject(ctx, bucket, object, data, metadata, opts) } // GetObjectInfo - reads object metadata from the hashedSet based on the object name. -func (s *xlSets) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) { - return s.getHashedSet(object).GetObjectInfo(ctx, bucket, object) +func (s *xlSets) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + return s.getHashedSet(object).GetObjectInfo(ctx, bucket, object, opts) } // DeleteObject - deletes an object from the hashedSet based on the object name. @@ -599,14 +599,14 @@ func (s *xlSets) DeleteObject(ctx context.Context, bucket string, object string) } // CopyObject - copies objects from one hashedSet to another hashedSet, on server side. -func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) { +func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) { srcSet := s.getHashedSet(srcObject) destSet := s.getHashedSet(destObject) // Check if this request is only metadata update. cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(destBucket, destObject)) if cpSrcDstSame && srcInfo.metadataOnly { - return srcSet.CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo) + return srcSet.CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts) } // Hold write lock on destination since in both cases @@ -632,7 +632,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke } go func() { - if gerr := srcSet.getObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag); gerr != nil { + if gerr := srcSet.getObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { logger.LogIf(ctx, gerr) } @@ -645,7 +645,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke } }() - return destSet.putObject(ctx, destBucket, destObject, srcInfo.Reader, srcInfo.UserDefined) + return destSet.putObject(ctx, destBucket, destObject, srcInfo.Reader, srcInfo.UserDefined, dstOpts) } // Returns function "listDir" of the type listDirFunc. @@ -833,19 +833,19 @@ func (s *xlSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMa } // Initiate a new multipart upload on a hashedSet based on object name. -func (s *xlSets) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) { - return s.getHashedSet(object).NewMultipartUpload(ctx, bucket, object, metadata) +func (s *xlSets) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) { + return s.getHashedSet(object).NewMultipartUpload(ctx, bucket, object, metadata, opts) } // Copies a part of an object from source hashedSet to destination hashedSet. func (s *xlSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, - startOffset int64, length int64, srcInfo ObjectInfo) (partInfo PartInfo, err error) { + startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (partInfo PartInfo, err error) { srcSet := s.getHashedSet(srcObject) destSet := s.getHashedSet(destObject) go func() { - if gerr := srcSet.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil { + if gerr := srcSet.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { logger.LogIf(ctx, gerr) return @@ -857,12 +857,12 @@ func (s *xlSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destB } }() - return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, srcInfo.Reader) + return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, srcInfo.Reader, dstOpts) } // PutObjectPart - writes part of an object to hashedSet based on the object name. -func (s *xlSets) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) { - return s.getHashedSet(object).PutObjectPart(ctx, bucket, object, uploadID, partID, data) +func (s *xlSets) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) { + return s.getHashedSet(object).PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts) } // ListObjectParts - lists all uploaded parts to an object in hashedSet. diff --git a/cmd/xl-v1-common_test.go b/cmd/xl-v1-common_test.go index 698c2cb1d..a75c1b8ba 100644 --- a/cmd/xl-v1-common_test.go +++ b/cmd/xl-v1-common_test.go @@ -43,7 +43,7 @@ func TestXLParentDirIsObject(t *testing.T) { } objectContent := "12345" objInfo, err := obj.PutObject(context.Background(), bucketName, objectName, - mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil) + mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{}) if err != nil { t.Fatal(err) } diff --git a/cmd/xl-v1-healing-common_test.go b/cmd/xl-v1-healing-common_test.go index 8ae943746..096a1f07d 100644 --- a/cmd/xl-v1-healing-common_test.go +++ b/cmd/xl-v1-healing-common_test.go @@ -200,7 +200,7 @@ func TestListOnlineDisks(t *testing.T) { t.Fatalf("Failed to make a bucket %v", err) } - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{}) if err != nil { t.Fatalf("Failed to putObject %v", err) } @@ -292,7 +292,7 @@ func TestDisksWithAllParts(t *testing.T) { t.Fatalf("Failed to make a bucket %v", err) } - _, err = obj.PutObject(ctx, bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil) + _, err = obj.PutObject(ctx, bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{}) if err != nil { t.Fatalf("Failed to putObject %v", err) } diff --git a/cmd/xl-v1-healing_test.go b/cmd/xl-v1-healing_test.go index a7a54f76e..c38afae0e 100644 --- a/cmd/xl-v1-healing_test.go +++ b/cmd/xl-v1-healing_test.go @@ -75,6 +75,7 @@ func TestHealObjectXL(t *testing.T) { bucket := "bucket" object := "object" data := bytes.Repeat([]byte("a"), 5*1024*1024) + var opts ObjectOptions err = obj.MakeBucketWithLocation(context.Background(), bucket, "") if err != nil { @@ -83,14 +84,14 @@ func TestHealObjectXL(t *testing.T) { // Create an object with multiple parts uploaded in decreasing // part number. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts) if err != nil { t.Fatalf("Failed to create a multipart upload - %v", err) } var uploadedParts []CompletePart for _, partID := range []int{2, 1} { - pInfo, err1 := obj.PutObjectPart(context.Background(), bucket, object, uploadID, partID, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", "")) + pInfo, err1 := obj.PutObjectPart(context.Background(), bucket, object, uploadID, partID, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) if err1 != nil { t.Fatalf("Failed to upload a part - %v", err1) } diff --git a/cmd/xl-v1-metadata_test.go b/cmd/xl-v1-metadata_test.go index 3e59a66e1..7411de84a 100644 --- a/cmd/xl-v1-metadata_test.go +++ b/cmd/xl-v1-metadata_test.go @@ -67,7 +67,7 @@ func testXLReadStat(obj ObjectLayer, instanceType string, disks []string, t *tes // iterate through the above set of inputs and upkoad the object. for i, input := range putObjectInputs { // uploading the object. - _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData) + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{}) // if object upload fails stop the test. if err != nil { t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) @@ -108,6 +108,7 @@ func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t bucketNames := []string{"minio-bucket", "minio-2-bucket"} objectNames := []string{"minio-object-1.txt"} uploadIDs := []string{} + var opts ObjectOptions // bucketnames[0]. // objectNames[0]. @@ -119,7 +120,7 @@ func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t t.Fatalf("%s : %s", instanceType, err.Error()) } // Initiate Multipart Upload on the above created bucket. - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts) if err != nil { // Failed to create NewMultipartUpload, abort. t.Fatalf("%s : %s", instanceType, err.Error()) @@ -150,7 +151,7 @@ func testXLReadMetaParts(obj ObjectLayer, instanceType string, disks []string, t sha256sum := "" // Iterating over creatPartCases to generate multipart chunks. for _, testCase := range createPartCases { - _, perr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum)) + _, perr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts) if perr != nil { t.Fatalf("%s : %s", instanceType, perr) } diff --git a/cmd/xl-v1-multipart.go b/cmd/xl-v1-multipart.go index 2acfff4e5..b4d37af19 100644 --- a/cmd/xl-v1-multipart.go +++ b/cmd/xl-v1-multipart.go @@ -240,7 +240,7 @@ func (xl xlObjects) newMultipartUpload(ctx context.Context, bucket string, objec // subsequent request each UUID is unique. // // Implements S3 compatible initiate multipart API. -func (xl xlObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (string, error) { +func (xl xlObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, opts ObjectOptions) (string, error) { if err := checkNewMultipartArgs(ctx, bucket, object, xl); err != nil { return "", err } @@ -256,7 +256,7 @@ func (xl xlObjects) NewMultipartUpload(ctx context.Context, bucket, object strin // data is read from an existing object. // // Implements S3 compatible Upload Part Copy API. -func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo) (pi PartInfo, e error) { +func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) { // Hold read locks on source object only if we are // going to read data from source object. objectSRLock := xl.nsMutex.NewNSLock(srcBucket, srcObject) @@ -270,7 +270,7 @@ func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, ds } go func() { - if gerr := xl.getObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil { + if gerr := xl.getObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil { if gerr = srcInfo.Writer.Close(); gerr != nil { logger.LogIf(ctx, gerr) } @@ -283,7 +283,7 @@ func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, ds } }() - partInfo, err := xl.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader) + partInfo, err := xl.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader, dstOpts) if err != nil { return pi, toObjectErr(err, dstBucket, dstObject) } @@ -297,7 +297,7 @@ func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, ds // of the multipart transaction. // // Implements S3 compatible Upload Part API. -func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) { +func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, e error) { if err := checkPutObjectPartArgs(ctx, bucket, object, xl); err != nil { return pi, err } diff --git a/cmd/xl-v1-multipart_test.go b/cmd/xl-v1-multipart_test.go index 0e1d7cddc..1df1e5d67 100644 --- a/cmd/xl-v1-multipart_test.go +++ b/cmd/xl-v1-multipart_test.go @@ -40,9 +40,10 @@ func TestXLCleanupStaleMultipartUploads(t *testing.T) { bucketName := "bucket" objectName := "object" + var opts ObjectOptions obj.MakeBucketWithLocation(context.Background(), bucketName, "") - uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil) + uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, opts) if err != nil { t.Fatal("Unexpected err: ", err) } diff --git a/cmd/xl-v1-object.go b/cmd/xl-v1-object.go index df0e9fbd2..eaabeba8d 100644 --- a/cmd/xl-v1-object.go +++ b/cmd/xl-v1-object.go @@ -80,7 +80,7 @@ func (xl xlObjects) prepareFile(ctx context.Context, bucket, object string, size // CopyObject - copy object source object to destination object. // if source object and destination object are same we only // update metadata. -func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo) (oi ObjectInfo, e error) { +func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) { cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) // Read metadata associated with the object from all disks. @@ -138,7 +138,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc go func() { var startOffset int64 // Read the whole file. - if gerr := xl.getObject(ctx, srcBucket, srcObject, startOffset, length, pipeWriter, srcInfo.ETag); gerr != nil { + if gerr := xl.getObject(ctx, srcBucket, srcObject, startOffset, length, pipeWriter, srcInfo.ETag, srcOpts); gerr != nil { pipeWriter.CloseWithError(toObjectErr(gerr, srcBucket, srcObject)) return } @@ -151,7 +151,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc return oi, toObjectErr(err, dstBucket, dstObject) } - objInfo, err := xl.putObject(ctx, dstBucket, dstObject, hashReader, srcInfo.UserDefined) + objInfo, err := xl.putObject(ctx, dstBucket, dstObject, hashReader, srcInfo.UserDefined, dstOpts) if err != nil { return oi, toObjectErr(err, dstBucket, dstObject) } @@ -168,18 +168,18 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (xl xlObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (xl xlObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error { // Lock the object before reading. objectLock := xl.nsMutex.NewNSLock(bucket, object) if err := objectLock.GetRLock(globalObjectTimeout); err != nil { return err } defer objectLock.RUnlock() - return xl.getObject(ctx, bucket, object, startOffset, length, writer, etag) + return xl.getObject(ctx, bucket, object, startOffset, length, writer, etag, opts) } // getObject wrapper for xl GetObject -func (xl xlObjects) getObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (xl xlObjects) getObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error { if err := checkGetObjArgs(ctx, bucket, object); err != nil { return err @@ -350,7 +350,7 @@ func (xl xlObjects) getObjectInfoDir(ctx context.Context, bucket, object string) } // GetObjectInfo - reads object metadata and replies back ObjectInfo. -func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) { +func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) { // Lock the object before reading. objectLock := xl.nsMutex.NewNSLock(bucket, object) if err := objectLock.GetRLock(globalObjectTimeout); err != nil { @@ -512,7 +512,7 @@ func renameObject(ctx context.Context, disks []StorageAPI, srcBucket, srcObject, // until EOF, erasure codes the data across all disk and additionally // writes `xl.json` which carries the necessary metadata for future // object operations. -func (xl xlObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { +func (xl xlObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) { // Validate put object input args. if err = checkPutObjectArgs(ctx, bucket, object, xl, data.Size()); err != nil { return ObjectInfo{}, err @@ -523,11 +523,11 @@ func (xl xlObjects) PutObject(ctx context.Context, bucket string, object string, return objInfo, err } defer objectLock.Unlock() - return xl.putObject(ctx, bucket, object, data, metadata) + return xl.putObject(ctx, bucket, object, data, metadata, opts) } // putObject wrapper for xl PutObject -func (xl xlObjects) putObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) { +func (xl xlObjects) putObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) { uniqueID := mustGetUUID() tempObj := uniqueID diff --git a/cmd/xl-v1-object_test.go b/cmd/xl-v1-object_test.go index f9e292325..fc215a5bc 100644 --- a/cmd/xl-v1-object_test.go +++ b/cmd/xl-v1-object_test.go @@ -34,6 +34,7 @@ func TestRepeatPutObjectPart(t *testing.T) { var objLayer ObjectLayer var disks []string var err error + var opts ObjectOptions objLayer, disks, err = prepareXL16() if err != nil { @@ -48,18 +49,18 @@ func TestRepeatPutObjectPart(t *testing.T) { t.Fatal(err) } - uploadID, err := objLayer.NewMultipartUpload(context.Background(), "bucket1", "mpartObj1", nil) + uploadID, err := objLayer.NewMultipartUpload(context.Background(), "bucket1", "mpartObj1", nil, opts) if err != nil { t.Fatal(err) } fiveMBBytes := bytes.Repeat([]byte("a"), 5*humanize.MiByte) md5Hex := getMD5Hash(fiveMBBytes) - _, err = objLayer.PutObjectPart(context.Background(), "bucket1", "mpartObj1", uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, "")) + _, err = objLayer.PutObjectPart(context.Background(), "bucket1", "mpartObj1", uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts) if err != nil { t.Fatal(err) } // PutObjectPart should succeed even if part already exists. ref: https://github.com/minio/minio/issues/1930 - _, err = objLayer.PutObjectPart(context.Background(), "bucket1", "mpartObj1", uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, "")) + _, err = objLayer.PutObjectPart(context.Background(), "bucket1", "mpartObj1", uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts) if err != nil { t.Fatal(err) } @@ -93,7 +94,7 @@ func TestXLDeleteObjectBasic(t *testing.T) { } // Create object "dir/obj" under bucket "bucket" for Test 7 to pass - _, err = xl.PutObject(context.Background(), "bucket", "dir/obj", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = xl.PutObject(context.Background(), "bucket", "dir/obj", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}) if err != nil { t.Fatalf("XL Object upload failed: %s", err) } @@ -109,7 +110,6 @@ func TestXLDeleteObjectBasic(t *testing.T) { // Cleanup backend directories removeRoots(fsDirs) } - func TestXLDeleteObjectDiskNotFound(t *testing.T) { // Reset global storage class flags resetGlobalStorageEnvs() @@ -129,8 +129,9 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) { bucket := "bucket" object := "object" + opts := ObjectOptions{} // Create object "obj" under bucket "bucket". - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, opts) if err != nil { t.Fatal(err) } @@ -145,7 +146,7 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) { } // Create "obj" under "bucket". - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, opts) if err != nil { t.Fatal(err) } @@ -179,8 +180,9 @@ func TestGetObjectNoQuorum(t *testing.T) { bucket := "bucket" object := "object" + opts := ObjectOptions{} // Create "object" under "bucket". - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, opts) if err != nil { t.Fatal(err) } @@ -203,7 +205,7 @@ func TestGetObjectNoQuorum(t *testing.T) { } } // Fetch object from store. - err = xl.GetObject(context.Background(), bucket, object, 0, int64(len("abcd")), ioutil.Discard, "") + err = xl.GetObject(context.Background(), bucket, object, 0, int64(len("abcd")), ioutil.Discard, "", opts) if err != toObjectErr(errXLReadQuorum, bucket, object) { t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err) } @@ -229,8 +231,9 @@ func TestPutObjectNoQuorum(t *testing.T) { bucket := "bucket" object := "object" + opts := ObjectOptions{} // Create "object" under "bucket". - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, opts) if err != nil { t.Fatal(err) } @@ -253,7 +256,7 @@ func TestPutObjectNoQuorum(t *testing.T) { } } // Upload new content to same object "object" - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, opts) if err != toObjectErr(errXLWriteQuorum, bucket, object) { t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err) } @@ -287,7 +290,7 @@ func TestHealing(t *testing.T) { t.Fatal(err) } - _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), length, "", ""), nil) + _, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), length, "", ""), nil, ObjectOptions{}) if err != nil { t.Fatal(err) } diff --git a/vendor/github.com/minio/minio-go/README.md b/vendor/github.com/minio/minio-go/README.md index 91b42049f..85ff2cb3d 100644 --- a/vendor/github.com/minio/minio-go/README.md +++ b/vendor/github.com/minio/minio-go/README.md @@ -139,7 +139,7 @@ The full API Reference is available here. ### API Reference : File Object Operations * [`FPutObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) -* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) +* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FGetObject) * [`FPutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FPutObjectWithContext) * [`FGetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FGetObjectWithContext) @@ -183,6 +183,10 @@ The full API Reference is available here. * [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) * [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) +### Full Examples : Bucket lifecycle Operations +* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go) +* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go) + ### Full Examples : Bucket notification Operations * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) diff --git a/vendor/github.com/minio/minio-go/api-compose-object.go b/vendor/github.com/minio/minio-go/api-compose-object.go index 5d3ede70d..a69122e40 100644 --- a/vendor/github.com/minio/minio-go/api-compose-object.go +++ b/vendor/github.com/minio/minio-go/api-compose-object.go @@ -101,7 +101,11 @@ func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) ma r["x-amz-metadata-directive"] = "REPLACE" } for k, v := range d.userMetadata { - r["x-amz-meta-"+k] = v + if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { + r[k] = v + } else { + r["x-amz-meta-"+k] = v + } } return r } @@ -373,15 +377,6 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object)) } - // Since we did a HEAD to get size, we use the ETag - // value to make sure the object has not changed by - // the time we perform the copy. This is done, only if - // the user has not set their own ETag match - // condition. - if src.Headers.Get("x-amz-copy-source-if-match") == "" { - src.SetMatchETagCond(etag) - } - // Check if a segment is specified, and if so, is the // segment within object bounds? if src.start != -1 { @@ -429,7 +424,15 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo // Now, handle multipart-copy cases. - // 1. Initiate a new multipart upload. + // 1. Ensure that the object has not been changed while + // we are copying data. + for _, src := range srcs { + if src.Headers.Get("x-amz-copy-source-if-match") == "" { + src.SetMatchETagCond(etag) + } + } + + // 2. Initiate a new multipart upload. // Set user-metadata on the destination object. If no // user-metadata is specified, and there is only one source, @@ -449,13 +452,13 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo return err } - // 2. Perform copy part uploads + // 3. Perform copy part uploads objParts := []CompletePart{} partIndex := 1 for i, src := range srcs { h := src.Headers if src.encryption != nil { - src.encryption.Marshal(h) + encrypt.SSECopy(src.encryption).Marshal(h) } // Add destination encryption headers if dst.encryption != nil { @@ -480,14 +483,14 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo return err } if progress != nil { - io.CopyN(ioutil.Discard, progress, start+end-1) + io.CopyN(ioutil.Discard, progress, end-start+1) } objParts = append(objParts, complPart) partIndex++ } } - // 3. Make final complete-multipart request. + // 4. Make final complete-multipart request. _, err = c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID, completeMultipartUpload{Parts: objParts}) if err != nil { diff --git a/vendor/github.com/minio/minio-go/api-get-lifecycle.go b/vendor/github.com/minio/minio-go/api-get-lifecycle.go new file mode 100644 index 000000000..8097bfc02 --- /dev/null +++ b/vendor/github.com/minio/minio-go/api-get-lifecycle.go @@ -0,0 +1,77 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// GetBucketLifecycle - get bucket lifecycle. +func (c Client) GetBucketLifecycle(bucketName string) (string, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return "", err + } + bucketLifecycle, err := c.getBucketLifecycle(bucketName) + if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "NoSuchLifecycleConfiguration" { + return "", nil + } + return "", err + } + return bucketLifecycle, nil +} + +// Request server for current bucket lifecycle. +func (c Client) getBucketLifecycle(bucketName string) (string, error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute GET on bucket to get lifecycle. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return "", err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return "", httpRespToErrorResponse(resp, bucketName, "") + } + } + + bucketLifecycleBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + lifecycle := string(bucketLifecycleBuf) + return lifecycle, err +} diff --git a/vendor/github.com/minio/minio-go/api-get-object-acl.go b/vendor/github.com/minio/minio-go/api-get-object-acl.go new file mode 100644 index 000000000..af5544da3 --- /dev/null +++ b/vendor/github.com/minio/minio-go/api-get-object-acl.go @@ -0,0 +1,136 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "net/http" + "net/url" +) + +type accessControlPolicy struct { + Owner struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + } `xml:"Owner"` + AccessControlList struct { + Grant []struct { + Grantee struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + URI string `xml:"URI"` + } `xml:"Grantee"` + Permission string `xml:"Permission"` + } `xml:"Grant"` + } `xml:"AccessControlList"` +} + +//GetObjectACL get object ACLs +func (c Client) GetObjectACL(bucketName, objectName string) (*ObjectInfo, error) { + + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: url.Values{ + "acl": []string{""}, + }, + }) + if err != nil { + return nil, err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + + res := &accessControlPolicy{} + + if err := xmlDecoder(resp.Body, res); err != nil { + return nil, err + } + + objInfo, err := c.statObject(context.Background(), bucketName, objectName, StatObjectOptions{}) + if err != nil { + return nil, err + } + + cannedACL := getCannedACL(res) + if cannedACL != "" { + objInfo.Metadata.Add("X-Amz-Acl", cannedACL) + return &objInfo, nil + } + + grantACL := getAmzGrantACL(res) + for k, v := range grantACL { + objInfo.Metadata[k] = v + } + + return &objInfo, nil +} + +func getCannedACL(aCPolicy *accessControlPolicy) string { + grants := aCPolicy.AccessControlList.Grant + + switch { + case len(grants) == 1: + if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" { + return "private" + } + case len(grants) == 2: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { + return "authenticated-read" + } + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { + return "public-read" + } + if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID { + return "bucket-owner-read" + } + } + case len(grants) == 3: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { + return "public-read-write" + } + } + } + return "" +} + +func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string { + grants := aCPolicy.AccessControlList.Grant + res := map[string][]string{} + + for _, g := range grants { + switch { + case g.Permission == "READ": + res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID) + case g.Permission == "WRITE": + res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID) + case g.Permission == "READ_ACP": + res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID) + case g.Permission == "WRITE_ACP": + res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID) + case g.Permission == "FULL_CONTROL": + res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID) + } + } + return res +} diff --git a/vendor/github.com/minio/minio-go/api-get-options.go b/vendor/github.com/minio/minio-go/api-get-options.go index a5a87526f..dbf062d61 100644 --- a/vendor/github.com/minio/minio-go/api-get-options.go +++ b/vendor/github.com/minio/minio-go/api-get-options.go @@ -44,7 +44,7 @@ func (o GetObjectOptions) Header() http.Header { for k, v := range o.headers { headers.Set(k, v) } - if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() != encrypt.S3 { + if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC { o.ServerSideEncryption.Marshal(headers) } return headers diff --git a/vendor/github.com/minio/minio-go/api-list.go b/vendor/github.com/minio/minio-go/api-list.go index bea286b32..04f757339 100644 --- a/vendor/github.com/minio/minio-go/api-list.go +++ b/vendor/github.com/minio/minio-go/api-list.go @@ -633,30 +633,27 @@ func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsI return partsInfo, nil } -// findUploadID lists all incomplete uploads and finds the uploadID of the matching object name. -func (c Client) findUploadID(bucketName, objectName string) (uploadID string, err error) { +// findUploadIDs lists all incomplete uploads and find the uploadIDs of the matching object name. +func (c Client) findUploadIDs(bucketName, objectName string) ([]string, error) { + var uploadIDs []string // Make list incomplete uploads recursive. isRecursive := true // Turn off size aggregation of individual parts, in this request. isAggregateSize := false - // latestUpload to track the latest multipart info for objectName. - var latestUpload ObjectMultipartInfo // Create done channel to cleanup the routine. doneCh := make(chan struct{}) defer close(doneCh) // List all incomplete uploads. for mpUpload := range c.listIncompleteUploads(bucketName, objectName, isRecursive, isAggregateSize, doneCh) { if mpUpload.Err != nil { - return "", mpUpload.Err + return nil, mpUpload.Err } if objectName == mpUpload.Key { - if mpUpload.Initiated.Sub(latestUpload.Initiated) > 0 { - latestUpload = mpUpload - } + uploadIDs = append(uploadIDs, mpUpload.UploadID) } } // Return the latest upload id. - return latestUpload.UploadID, nil + return uploadIDs, nil } // getTotalMultipartSize - calculate total uploaded size for the a given multipart object. diff --git a/vendor/github.com/minio/minio-go/api-put-bucket.go b/vendor/github.com/minio/minio-go/api-put-bucket.go index cb9d8f27a..33dc0cf3d 100644 --- a/vendor/github.com/minio/minio-go/api-put-bucket.go +++ b/vendor/github.com/minio/minio-go/api-put-bucket.go @@ -178,6 +178,87 @@ func (c Client) removeBucketPolicy(bucketName string) error { return nil } +// SetBucketLifecycle set the lifecycle on an existing bucket. +func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If lifecycle is empty then delete it. + if lifecycle == "" { + return c.removeBucketLifecycle(bucketName) + } + + // Save the updated lifecycle. + return c.putBucketLifecycle(bucketName, lifecycle) +} + +// Saves a new bucket lifecycle. +func (c Client) putBucketLifecycle(bucketName, lifecycle string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Content-length is mandatory for put lifecycle request + lifecycleReader := strings.NewReader(lifecycle) + b, err := ioutil.ReadAll(lifecycleReader) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: lifecycleReader, + contentLength: int64(len(b)), + contentMD5Base64: sumMD5Base64(b), + } + + // Execute PUT to upload a new bucket lifecycle. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// Remove lifecycle from a bucket. +func (c Client) removeBucketLifecycle(bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + // SetBucketNotification saves a new bucket notification. func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error { // Input validation. diff --git a/vendor/github.com/minio/minio-go/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/api-put-object-multipart.go index 52dc069d0..8805ecf96 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/api-put-object-multipart.go @@ -259,7 +259,7 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID // Set encryption headers, if any. customHeader := make(http.Header) - if sse != nil && sse.Type() != encrypt.S3 && sse.Type() != encrypt.KMS { + if sse != nil { sse.Marshal(customHeader) } diff --git a/vendor/github.com/minio/minio-go/api-remove.go b/vendor/github.com/minio/minio-go/api-remove.go index c2ffcdd34..f33df4dfc 100644 --- a/vendor/github.com/minio/minio-go/api-remove.go +++ b/vendor/github.com/minio/minio-go/api-remove.go @@ -233,18 +233,20 @@ func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } - // Find multipart upload id of the object to be aborted. - uploadID, err := c.findUploadID(bucketName, objectName) + // Find multipart upload ids of the object to be aborted. + uploadIDs, err := c.findUploadIDs(bucketName, objectName) if err != nil { return err } - if uploadID != "" { - // Upload id found, abort the incomplete multipart upload. + + for _, uploadID := range uploadIDs { + // abort incomplete multipart upload, based on the upload id passed. err := c.abortMultipartUpload(context.Background(), bucketName, objectName, uploadID) if err != nil { return err } } + return nil } diff --git a/vendor/github.com/minio/minio-go/api.go b/vendor/github.com/minio/minio-go/api.go index 237ddbcae..7c93ca1bb 100644 --- a/vendor/github.com/minio/minio-go/api.go +++ b/vendor/github.com/minio/minio-go/api.go @@ -99,7 +99,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.3" + libraryVersion = "v6.0.6" ) // User Agent should always following the below style. @@ -455,22 +455,9 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { return err } } else { - // WORKAROUND for https://github.com/golang/go/issues/13942. - // httputil.DumpResponse does not print response headers for - // all successful calls which have response ContentLength set - // to zero. Keep this workaround until the above bug is fixed. - if resp.ContentLength == 0 { - var buffer bytes.Buffer - if err = resp.Header.Write(&buffer); err != nil { - return err - } - respTrace = buffer.Bytes() - respTrace = append(respTrace, []byte("\r\n")...) - } else { - respTrace, err = httputil.DumpResponse(resp, false) - if err != nil { - return err - } + respTrace, err = httputil.DumpResponse(resp, false) + if err != nil { + return err } } @@ -599,8 +586,8 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // Initiate the request. res, err = c.do(req) if err != nil { - // For supported network errors verify. - if isNetErrorRetryable(err) { + // For supported http requests errors verify. + if isHTTPReqErrorRetryable(err) { continue // Retry. } // For other errors, return here no need to retry. diff --git a/vendor/github.com/minio/minio-go/core.go b/vendor/github.com/minio/minio-go/core.go index a5017d868..0c651d600 100644 --- a/vendor/github.com/minio/minio-go/core.go +++ b/vendor/github.com/minio/minio-go/core.go @@ -21,6 +21,8 @@ import ( "context" "io" "strings" + + "github.com/minio/minio-go/pkg/encrypt" ) // Core - Inherits Client and adds new methods to expose the low level S3 APIs. @@ -68,7 +70,7 @@ func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string } // PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string) (ObjectInfo, error) { +func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { opts := PutObjectOptions{} m := make(map[string]string) for k, v := range metadata { @@ -89,6 +91,7 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba } } opts.UserMetadata = m + opts.ServerSideEncryption = sse return c.putObjectDo(context.Background(), bucket, object, data, md5Base64, sha256Hex, size, opts) } @@ -104,8 +107,8 @@ func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, de } // PutObjectPart - Upload an object part. -func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string) (ObjectPart, error) { - return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, nil) +func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { + return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) } // ListObjectParts - List uploaded parts of an incomplete upload.x diff --git a/vendor/github.com/minio/minio-go/functional_tests.go b/vendor/github.com/minio/minio-go/functional_tests.go deleted file mode 100644 index a28e922a1..000000000 --- a/vendor/github.com/minio/minio-go/functional_tests.go +++ /dev/null @@ -1,7499 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "math/rand" - "mime/multipart" - "net/http" - "net/url" - "os" - "path/filepath" - "reflect" - "runtime" - "strconv" - "strings" - "time" - - humanize "github.com/dustin/go-humanize" - minio "github.com/minio/minio-go" - log "github.com/sirupsen/logrus" - - "github.com/minio/minio-go/pkg/encrypt" -) - -const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return prefix + string(b[0:30-len(prefix)]) -} - -var dataFileMap = map[string]int{ - "datafile-1-b": 1, - "datafile-10-kB": 10 * humanize.KiByte, - "datafile-33-kB": 33 * humanize.KiByte, - "datafile-100-kB": 100 * humanize.KiByte, - "datafile-1.03-MB": 1056 * humanize.KiByte, - "datafile-1-MB": 1 * humanize.MiByte, - "datafile-5-MB": 5 * humanize.MiByte, - "datafile-6-MB": 6 * humanize.MiByte, - "datafile-11-MB": 11 * humanize.MiByte, - "datafile-65-MB": 65 * humanize.MiByte, -} - -func isFullMode() bool { - return os.Getenv("MINT_MODE") == "full" -} - -func getFuncName() string { - pc, _, _, _ := runtime.Caller(1) - return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.") -} - -// Tests bucket re-create errors. -func testMakeBucketError() { - region := "eu-central-1" - - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "MakeBucket(bucketName, region)" - // initialize logging params - args := map[string]interface{}{ - "bucketName": "", - "region": region, - } - - // skipping region functional tests for non s3 runs - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, region); err != nil { - logError(testName, function, args, startTime, "", "MakeBucket Failed", err) - return - } - if err = c.MakeBucket(bucketName, region); err == nil { - logError(testName, function, args, startTime, "", "Bucket already exists", err) - return - } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - logError(testName, function, args, startTime, "", "Invalid error returned by server", err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -func testMetadataSizeLimit() { - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader, objectSize, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts.UserMetadata": "", - } - rand.Seed(startTime.Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) - return - } - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "Make bucket failed", err) - return - } - - const HeaderSizeLimit = 8 * 1024 - const UserMetadataLimit = 2 * 1024 - - // Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail - metadata := make(map[string]string) - metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test"))) - args["metadata"] = fmt.Sprint(metadata) - - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) - if err == nil { - logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil) - return - } - - // Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail - metadata = make(map[string]string) - metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test"))) - args["metadata"] = fmt.Sprint(metadata) - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) - if err == nil { - logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests various bucket supported formats. -func testMakeBucketRegions() { - region := "eu-central-1" - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "MakeBucket(bucketName, region)" - // initialize logging params - args := map[string]interface{}{ - "bucketName": "", - "region": region, - } - - // skipping region functional tests for non s3 runs - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, region); err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - region = "us-west-2" - args["region"] = region - if err = c.MakeBucket(bucketName+".withperiod", region); err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName+".withperiod", c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -// Test PutObject using a large data to trigger multipart readat -func testPutObjectReadAt() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "objectContentType", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "Make bucket failed", err) - return - } - - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - // Object content type - objectContentType := "binary/octet-stream" - args["objectContentType"] = objectContentType - - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "Get Object failed", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat Object failed", err) - return - } - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) - return - } - if st.ContentType != objectContentType { - logError(testName, function, args, startTime, "", "Content types don't match", err) - return - } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", "Object Close failed", err) - return - } - if err := r.Close(); err == nil { - logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test PutObject using a large data to trigger multipart readat -func testPutObjectWithMetadata() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader,size, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", - } - - if !isFullMode() { - ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "Make bucket failed", err) - return - } - - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - // Object custom metadata - customContentType := "custom/contenttype" - - args["metadata"] = map[string][]string{ - "Content-Type": {customContentType}, - } - - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ - ContentType: customContentType}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) - return - } - if st.ContentType != customContentType { - logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err) - return - } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", "Object Close failed", err) - return - } - if err := r.Close(); err == nil { - logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -func testPutObjectWithContentLanguage() { - // initialize logging params - objectName := "test-object" - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader, size, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": objectName, - "size": -1, - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - data := bytes.Repeat([]byte("a"), int(0)) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ - ContentLanguage: "en-US", - }) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != 0 { - logError(testName, function, args, startTime, "", "Expected upload object '0' doesn't match with PutObject return value", err) - return - } - - objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - - if objInfo.Metadata.Get("Content-Language") != "en-US" { - logError(testName, function, args, startTime, "", "Expected content-language 'en-US' doesn't match with StatObject return value", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test put object with streaming signature. -func testPutObjectStreaming() { - // initialize logging params - objectName := "test-object" - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader,size,opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": objectName, - "size": -1, - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Upload an object. - sizes := []int64{0, 64*1024 - 1, 64 * 1024} - - for _, size := range sizes { - data := bytes.Repeat([]byte("a"), int(size)) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) - return - } - - if n != size { - logError(testName, function, args, startTime, "", "Expected upload object size doesn't match with PutObjectStreaming return value", err) - return - } - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test get object seeker from the end, using whence set to '2'. -func testGetObjectSeekEnd() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) - return - } - - pos, err := r.Seek(-100, 2) - if err != nil { - logError(testName, function, args, startTime, "", "Object Seek failed", err) - return - } - if pos != st.Size-100 { - logError(testName, function, args, startTime, "", "Incorrect position", err) - return - } - buf2 := make([]byte, 100) - m, err := io.ReadFull(r, buf2) - if err != nil { - logError(testName, function, args, startTime, "", "Error reading through io.ReadFull", err) - return - } - if m != len(buf2) { - logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err) - return - } - hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) - hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) - if hexBuf1 != hexBuf2 { - logError(testName, function, args, startTime, "", "Values at same index dont match", err) - return - } - pos, err = r.Seek(-100, 2) - if err != nil { - logError(testName, function, args, startTime, "", "Object Seek failed", err) - return - } - if pos != st.Size-100 { - logError(testName, function, args, startTime, "", "Incorrect position", err) - return - } - if err = r.Close(); err != nil { - logError(testName, function, args, startTime, "", "ObjectClose failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test get object reader to not throw error on being closed twice. -func testGetObjectClosedTwice() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "PutObject response doesn't match sent bytes, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) - return - } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", "Object Close failed", err) - return - } - if err := r.Close(); err == nil { - logError(testName, function, args, startTime, "", "Already closed object. No error returned", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test RemoveObjectsWithContext request context cancels after timeout -func testRemoveObjectsWithContext() { - // Initialize logging params. - startTime := time.Now() - testName := getFuncName() - function := "RemoveObjectsWithContext(ctx, bucketName, objectsCh)" - args := map[string]interface{}{ - "bucketName": "", - } - - // Seed random based on current tie. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - } - - // Generate put data. - r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) - - // Multi remove of 20 objects. - nrObjects := 20 - objectsCh := make(chan string) - go func() { - defer close(objectsCh) - for i := 0; i < nrObjects; i++ { - objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - continue - } - objectsCh <- objectName - } - }() - // Set context to cancel in 1 nanosecond. - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - args["ctx"] = ctx - defer cancel() - - // Call RemoveObjectsWithContext API with short timeout. - errorCh := c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) - // Check for error. - select { - case r := <-errorCh: - if r.Err == nil { - logError(testName, function, args, startTime, "", "RemoveObjectsWithContext should fail on short timeout", err) - return - } - } - // Set context with longer timeout. - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - args["ctx"] = ctx - defer cancel() - // Perform RemoveObjectsWithContext with the longer timeout. Expect the removals to succeed. - errorCh = c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) - select { - case r, more := <-errorCh: - if more || r.Err != nil { - logError(testName, function, args, startTime, "", "Unexpected error", r.Err) - return - } - } - - // Delete all objects and buckets. - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -// Test removing multiple objects with Remove API -func testRemoveMultipleObjects() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "RemoveObjects(bucketName, objectsCh)" - args := map[string]interface{}{ - "bucketName": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) - - // Multi remove of 1100 objects - nrObjects := 200 - - objectsCh := make(chan string) - - go func() { - defer close(objectsCh) - // Upload objects and send them to objectsCh - for i := 0; i < nrObjects; i++ { - objectName := "sample" + strconv.Itoa(i) + ".txt" - _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - continue - } - objectsCh <- objectName - } - }() - - // Call RemoveObjects API - errorCh := c.RemoveObjects(bucketName, objectsCh) - - // Check if errorCh doesn't receive any error - select { - case r, more := <-errorCh: - if more { - logError(testName, function, args, startTime, "", "Unexpected error", r.Err) - return - } - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests FPutObject of a big file to trigger multipart -func testFPutObjectMultipart() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "FPutObject(bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileName": "", - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var fileName = getMintDataDirFilePath("datafile-65-MB") - if fileName == "" { - // Make a temp file with minPartSize bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - logError(testName, function, args, startTime, "", "TempFile creation failed", err) - return - } - // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload. - if _, err = io.Copy(file, getDataReader("datafile-65-MB")); err != nil { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - if err = file.Close(); err != nil { - logError(testName, function, args, startTime, "", "File Close failed", err) - return - } - fileName = file.Name() - args["fileName"] = fileName - } - totalSize := dataFileMap["datafile-65-MB"] - // Set base object name - objectName := bucketName + "FPutObject" + "-standard" - args["objectName"] = objectName - - objectContentType := "testapplication/octet-stream" - args["objectContentType"] = objectContentType - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) - if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - objInfo, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Unexpected error", err) - return - } - if objInfo.Size != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err) - return - } - if objInfo.ContentType != objectContentType { - logError(testName, function, args, startTime, "", "ContentType doesn't match", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests FPutObject with null contentType (default = application/octet-stream) -func testFPutObject() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "FPutObject(bucketName, objectName, fileName, opts)" - - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileName": "", - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - location := "us-east-1" - - // Make a new bucket. - args["bucketName"] = bucketName - args["location"] = location - function = "MakeBucket()bucketName, location" - err = c.MakeBucket(bucketName, location) - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. - // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getMintDataDirFilePath("datafile-65-MB") - if fName == "" { - // Make a temp file with minPartSize bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - logError(testName, function, args, startTime, "", "TempFile creation failed", err) - return - } - - // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload. - if _, err = io.Copy(file, getDataReader("datafile-65-MB")); err != nil { - logError(testName, function, args, startTime, "", "File copy failed", err) - return - } - // Close the file pro-actively for windows. - if err = file.Close(); err != nil { - logError(testName, function, args, startTime, "", "File close failed", err) - return - } - defer os.Remove(file.Name()) - fName = file.Name() - } - totalSize := dataFileMap["datafile-65-MB"] - - // Set base object name - function = "FPutObject(bucketName, objectName, fileName, opts)" - objectName := bucketName + "FPutObject" - args["objectName"] = objectName + "-standard" - args["fileName"] = fName - args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"} - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err := c.FPutObject(bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - - if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) - return - } - - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - args["objectName"] = objectName + "-Octet" - n, err = c.FPutObject(bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "File close failed", err) - return - } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) - return - } - srcFile, err := os.Open(fName) - if err != nil { - logError(testName, function, args, startTime, "", "File open failed", err) - return - } - defer srcFile.Close() - // Add extension to temp file name - tmpFile, err := os.Create(fName + ".gtar") - if err != nil { - logError(testName, function, args, startTime, "", "File create failed", err) - return - } - defer tmpFile.Close() - _, err = io.Copy(tmpFile, srcFile) - if err != nil { - logError(testName, function, args, startTime, "", "File copy failed", err) - return - } - - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - args["objectName"] = objectName + "-GTar" - n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) - return - } - - // Check headers - function = "StatObject(bucketName, objectName, opts)" - args["objectName"] = objectName + "-standard" - rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - if rStandard.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err) - return - } - - function = "StatObject(bucketName, objectName, opts)" - args["objectName"] = objectName + "-Octet" - rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - if rOctet.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err) - return - } - - function = "StatObject(bucketName, objectName, opts)" - args["objectName"] = objectName + "-GTar" - rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - if rGTar.ContentType != "application/x-gtar" { - logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-gtar, got "+rGTar.ContentType, err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - if err = os.Remove(fName + ".gtar"); err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests FPutObjectWithContext request context cancels after timeout -func testFPutObjectWithContext() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "FPutObject(bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileName": "", - "opts": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Upload 1 parts worth of data to use multipart upload. - // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getMintDataDirFilePath("datafile-1-MB") - if fName == "" { - // Make a temp file with 1 MiB bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") - if err != nil { - logError(testName, function, args, startTime, "", "TempFile creation failed", err) - return - } - - // Upload 1 parts to trigger multipart upload - if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { - logError(testName, function, args, startTime, "", "File copy failed", err) - return - } - // Close the file pro-actively for windows. - if err = file.Close(); err != nil { - logError(testName, function, args, startTime, "", "File close failed", err) - return - } - defer os.Remove(file.Name()) - fName = file.Name() - } - totalSize := dataFileMap["datafile-1-MB"] - - // Set base object name - objectName := bucketName + "FPutObjectWithContext" - args["objectName"] = objectName - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - args["ctx"] = ctx - defer cancel() - - // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) - _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) - return - } - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - defer cancel() - // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed - n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on long timeout", err) - return - } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) - return - } - - _, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Tests FPutObjectWithContext request context cancels after timeout -func testFPutObjectWithContextV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "FPutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "minio.PutObjectOptions{ContentType:objectContentType}", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Upload 1 parts worth of data to use multipart upload. - // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getMintDataDirFilePath("datafile-1-MB") - if fName == "" { - // Make a temp file with 1 MiB bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") - if err != nil { - logError(testName, function, args, startTime, "", "Temp file creation failed", err) - return - } - - // Upload 1 parts to trigger multipart upload - if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { - logError(testName, function, args, startTime, "", "File copy failed", err) - return - } - - // Close the file pro-actively for windows. - if err = file.Close(); err != nil { - logError(testName, function, args, startTime, "", "File close failed", err) - return - } - defer os.Remove(file.Name()) - fName = file.Name() - } - totalSize := dataFileMap["datafile-1-MB"] - - // Set base object name - objectName := bucketName + "FPutObjectWithContext" - args["objectName"] = objectName - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - args["ctx"] = ctx - defer cancel() - - // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) - _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) - return - } - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - defer cancel() - // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed - n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on longer timeout", err) - return - } - if n != int64(totalSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match:wanted"+string(totalSize)+" got "+string(n), err) - return - } - - _, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Test validates putObject with context to see if request cancellation is honored. -func testPutObjectWithContext() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "ctx": "", - "bucketName": "", - "objectName": "", - "opts": "", - } - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket call failed", err) - return - } - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - args["objectName"] = objectName - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - args["ctx"] = ctx - args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"} - defer cancel() - - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err == nil { - logError(testName, function, args, startTime, "", "PutObjectWithContext should fail on short timeout", err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - args["ctx"] = ctx - - defer cancel() - reader = getDataReader("datafile-33-kB") - defer reader.Close() - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Tests get object ReaderSeeker interface methods. -func testGetObjectReadSeekFunctional() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - defer func() { - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - }() - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat object failed", err) - return - } - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) - return - } - - // This following function helps us to compare data from the reader after seek - // with the data from the original buffer - cmpData := func(r io.Reader, start, end int) { - if end-start == 0 { - return - } - buffer := bytes.NewBuffer([]byte{}) - if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "CopyN failed", err) - return - } - } - if !bytes.Equal(buf[start:end], buffer.Bytes()) { - logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) - return - } - } - - // Generic seek error for errors other than io.EOF - seekErr := errors.New("seek error") - - testCases := []struct { - offset int64 - whence int - pos int64 - err error - shouldCmp bool - start int - end int - }{ - // Start from offset 0, fetch data and compare - {0, 0, 0, nil, true, 0, 0}, - // Start from offset 2048, fetch data and compare - {2048, 0, 2048, nil, true, 2048, bufSize}, - // Start from offset larger than possible - {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, - // Move to offset 0 without comparing - {0, 0, 0, nil, false, 0, 0}, - // Move one step forward and compare - {1, 1, 1, nil, true, 1, bufSize}, - // Move larger than possible - {int64(bufSize), 1, 0, seekErr, false, 0, 0}, - // Provide negative offset with CUR_SEEK - {int64(-1), 1, 0, seekErr, false, 0, 0}, - // Test with whence SEEK_END and with positive offset - {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, - // Test with whence SEEK_END and with negative offset - {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, - // Test with whence SEEK_END and with large negative offset - {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, - } - - for i, testCase := range testCases { - // Perform seek operation - n, err := r.Seek(testCase.offset, testCase.whence) - // We expect an error - if testCase.err == seekErr && err == nil { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) - return - } - // We expect a specific error - if testCase.err != seekErr && testCase.err != err { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) - return - } - // If we expect an error go to the next loop - if testCase.err != nil { - continue - } - // Check the returned seek pos - if n != testCase.pos { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err) - return - } - // Compare only if shouldCmp is activated - if testCase.shouldCmp { - cmpData(r, testCase.start, testCase.end) - } - } - successLogger(testName, function, args, startTime).Info() -} - -// Tests get object ReaderAt interface methods. -func testGetObjectReadAtFunctional() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - - // read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - offset := int64(2048) - - // read directly - buf1 := make([]byte, 512) - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - // Test readAt before stat is called such that objectInfo doesn't change. - m, err := r.ReadAt(buf1, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf1) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) - return - } - if !bytes.Equal(buf1, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - offset += 512 - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) - return - } - - m, err = r.ReadAt(buf2, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf2) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) - return - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf3) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) - return - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf4) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) - return - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - } - if m != len(buf5) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) - return - } - if !bytes.Equal(buf, buf5) { - logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) - return - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -// Test Presigned Post Policy -func testPresignedPostPolicy() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PresignedPostPolicy(policy)" - args := map[string]interface{}{ - "policy": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "") - metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - - policy := minio.NewPostPolicy() - - if err := policy.SetBucket(""); err == nil { - logError(testName, function, args, startTime, "", "SetBucket did not fail for invalid conditions", err) - return - } - if err := policy.SetKey(""); err == nil { - logError(testName, function, args, startTime, "", "SetKey did not fail for invalid conditions", err) - return - } - if err := policy.SetKeyStartsWith(""); err == nil { - logError(testName, function, args, startTime, "", "SetKeyStartsWith did not fail for invalid conditions", err) - return - } - if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { - logError(testName, function, args, startTime, "", "SetExpires did not fail for invalid conditions", err) - return - } - if err := policy.SetContentType(""); err == nil { - logError(testName, function, args, startTime, "", "SetContentType did not fail for invalid conditions", err) - return - } - if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { - logError(testName, function, args, startTime, "", "SetContentLengthRange did not fail for invalid conditions", err) - return - } - if err := policy.SetUserMetadata("", ""); err == nil { - logError(testName, function, args, startTime, "", "SetUserMetadata did not fail for invalid conditions", err) - return - } - - policy.SetBucket(bucketName) - policy.SetKey(objectName) - policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days - policy.SetContentType("binary/octet-stream") - policy.SetContentLengthRange(10, 1024*1024) - policy.SetUserMetadata(metadataKey, metadataValue) - args["policy"] = policy.String() - - presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(policy) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) - return - } - - var formBuf bytes.Buffer - writer := multipart.NewWriter(&formBuf) - for k, v := range formData { - writer.WriteField(k, v) - } - - // Get a 33KB file to upload and test if set post policy works - var filePath = getMintDataDirFilePath("datafile-33-kB") - if filePath == "" { - // Make a temp file with 33 KB data. - file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest") - if err != nil { - logError(testName, function, args, startTime, "", "TempFile creation failed", err) - return - } - if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - if err = file.Close(); err != nil { - logError(testName, function, args, startTime, "", "File Close failed", err) - return - } - filePath = file.Name() - } - - // add file to post request - f, err := os.Open(filePath) - defer f.Close() - if err != nil { - logError(testName, function, args, startTime, "", "File open failed", err) - return - } - w, err := writer.CreateFormFile("file", filePath) - if err != nil { - logError(testName, function, args, startTime, "", "CreateFormFile failed", err) - return - } - - _, err = io.Copy(w, f) - if err != nil { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - writer.Close() - - // make post request with correct form data - res, err := http.Post(presignedPostPolicyURL.String(), writer.FormDataContentType(), bytes.NewReader(formBuf.Bytes())) - if err != nil { - logError(testName, function, args, startTime, "", "Http request failed", err) - return - } - defer res.Body.Close() - if res.StatusCode != http.StatusNoContent { - logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status)) - return - } - - // expected path should be absolute path of the object - var scheme string - if mustParseBool(os.Getenv(enableHTTPS)) { - scheme = "https://" - } else { - scheme = "http://" - } - - expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName - expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName - - if val, ok := res.Header["Location"]; ok { - if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS { - logError(testName, function, args, startTime, "", "Location in header response is incorrect", err) - return - } - } else { - logError(testName, function, args, startTime, "", "Location not found in header response", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests copy object -func testCopyObject() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(dst, src)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - // Check the various fields of source object against destination object. - objInfo, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - - // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - args["src"] = src - - // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) - return - } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) - return - } - err = src.SetMatchETagCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) - return - } - err = src.SetMatchETagExceptCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) - return - } - - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) - return - } - err = src.SetMatchETagCond(objInfo.ETag) - if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) - return - } - - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) - args["dst"] = dst - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - - // Perform the Copy - err = c.CopyObject(dst, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return - } - - // Source object - r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - // Check the various fields of source object against destination object. - objInfo, err = r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - objInfoCopy, err := readerCopy.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - if objInfo.Size != objInfoCopy.Size { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err) - return - } - - // Close all the get readers before proceeding with CopyObject operations. - r.Close() - readerCopy.Close() - - // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) - return - } - err = src.SetMatchETagExceptCond(objInfo.ETag) - if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) - return - } - - // Perform the Copy which should fail - err = c.CopyObject(dst, src) - if err == nil { - logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) - return - } - - // Perform the Copy which should update only metadata. - src = minio.NewSourceInfo(bucketName, objectName, nil) - dst, err = minio.NewDestinationInfo(bucketName, objectName, nil, map[string]string{ - "Copy": "should be same", - }) - args["dst"] = dst - args["src"] = src - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - - err = c.CopyObject(dst, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) - return - } - - stOpts := minio.StatObjectOptions{} - stOpts.SetMatchETag(objInfo.ETag) - objInfo, err = c.StatObject(bucketName, objectName, stOpts) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err) - return - } - - if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" { - logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - if err = cleanupBucket(bucketName+"-copy", c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -// Tests SSE-C get object ReaderSeeker interface methods. -func testEncryptedGetObjectReadSeekFunctional() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - defer func() { - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - }() - - // Generate 65MiB of data. - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ - ContentType: "binary/octet-stream", - ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), - }) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ - ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), - }) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - defer r.Close() - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat object failed", err) - return - } - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) - return - } - - // This following function helps us to compare data from the reader after seek - // with the data from the original buffer - cmpData := func(r io.Reader, start, end int) { - if end-start == 0 { - return - } - buffer := bytes.NewBuffer([]byte{}) - if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "CopyN failed", err) - return - } - } - if !bytes.Equal(buf[start:end], buffer.Bytes()) { - logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) - return - } - } - - testCases := []struct { - offset int64 - whence int - pos int64 - err error - shouldCmp bool - start int - end int - }{ - // Start from offset 0, fetch data and compare - {0, 0, 0, nil, true, 0, 0}, - // Start from offset 2048, fetch data and compare - {2048, 0, 2048, nil, true, 2048, bufSize}, - // Start from offset larger than possible - {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, - // Move to offset 0 without comparing - {0, 0, 0, nil, false, 0, 0}, - // Move one step forward and compare - {1, 1, 1, nil, true, 1, bufSize}, - // Move larger than possible - {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, - // Provide negative offset with CUR_SEEK - {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, - // Test with whence SEEK_END and with positive offset - {1024, 2, 0, io.EOF, false, 0, 0}, - // Test with whence SEEK_END and with negative offset - {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, - // Test with whence SEEK_END and with large negative offset - {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, - // Test with invalid whence - {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, - } - - for i, testCase := range testCases { - // Perform seek operation - n, err := r.Seek(testCase.offset, testCase.whence) - if err != nil && testCase.err == nil { - // We expected success. - logError(testName, function, args, startTime, "", - fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) - return - } - if err == nil && testCase.err != nil { - // We expected failure, but got success. - logError(testName, function, args, startTime, "", - fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) - return - } - if err != nil && testCase.err != nil { - if err.Error() != testCase.err.Error() { - // We expect a specific error - logError(testName, function, args, startTime, "", - fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) - return - } - } - // Check the returned seek pos - if n != testCase.pos { - logError(testName, function, args, startTime, "", - fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) - return - } - // Compare only if shouldCmp is activated - if testCase.shouldCmp { - cmpData(r, testCase.start, testCase.end) - } - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests SSE-C get object ReaderAt interface methods. -func testEncryptedGetObjectReadAtFunctional() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 65MiB of data. - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ - ContentType: "binary/octet-stream", - ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), - }) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) - return - } - - // read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ - ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), - }) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - defer r.Close() - - offset := int64(2048) - - // read directly - buf1 := make([]byte, 512) - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - // Test readAt before stat is called such that objectInfo doesn't change. - m, err := r.ReadAt(buf1, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf1) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) - return - } - if !bytes.Equal(buf1, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - offset += 512 - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) - return - } - - m, err = r.ReadAt(buf2, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf2) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) - return - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf3) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) - return - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf4) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) - return - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - } - if m != len(buf5) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) - return - } - if !bytes.Equal(buf, buf5) { - logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) - return - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -// TestEncryptionPutGet tests client side encryption -func testEncryptionPutGet() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutEncryptedObject(bucketName, objectName, reader, sse)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "sse": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - testCases := []struct { - buf []byte - }{ - {buf: bytes.Repeat([]byte("F"), 1)}, - {buf: bytes.Repeat([]byte("F"), 15)}, - {buf: bytes.Repeat([]byte("F"), 16)}, - {buf: bytes.Repeat([]byte("F"), 17)}, - {buf: bytes.Repeat([]byte("F"), 31)}, - {buf: bytes.Repeat([]byte("F"), 32)}, - {buf: bytes.Repeat([]byte("F"), 33)}, - {buf: bytes.Repeat([]byte("F"), 1024)}, - {buf: bytes.Repeat([]byte("F"), 1024*2)}, - {buf: bytes.Repeat([]byte("F"), 1024*1024)}, - } - - const password = "correct horse battery staple" // https://xkcd.com/936/ - - for i, testCase := range testCases { - // Generate a random object name - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - // Secured object - sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) - args["sse"] = sse - - // Put encrypted data - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) - if err != nil { - logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) - if err != nil { - logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) - return - } - defer r.Close() - - // Compare the sent object with the received one - recvBuffer := bytes.NewBuffer([]byte{}) - if _, err = io.Copy(recvBuffer, r); err != nil { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) - return - } - if recvBuffer.Len() != len(testCase.buf) { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) - return - } - if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) - return - } - - successLogger(testName, function, args, startTime).Info() - - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// TestEncryptionFPut tests client side encryption -func testEncryptionFPut() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "filePath": "", - "contentType": "", - "sse": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Object custom metadata - customContentType := "custom/contenttype" - args["metadata"] = customContentType - - testCases := []struct { - buf []byte - }{ - {buf: bytes.Repeat([]byte("F"), 0)}, - {buf: bytes.Repeat([]byte("F"), 1)}, - {buf: bytes.Repeat([]byte("F"), 15)}, - {buf: bytes.Repeat([]byte("F"), 16)}, - {buf: bytes.Repeat([]byte("F"), 17)}, - {buf: bytes.Repeat([]byte("F"), 31)}, - {buf: bytes.Repeat([]byte("F"), 32)}, - {buf: bytes.Repeat([]byte("F"), 33)}, - {buf: bytes.Repeat([]byte("F"), 1024)}, - {buf: bytes.Repeat([]byte("F"), 1024*2)}, - {buf: bytes.Repeat([]byte("F"), 1024*1024)}, - } - - const password = "correct horse battery staple" // https://xkcd.com/936/ - for i, testCase := range testCases { - // Generate a random object name - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - // Secured object - sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) - args["sse"] = sse - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - logError(testName, function, args, startTime, "", "file create failed", err) - return - } - _, err = file.Write(testCase.buf) - if err != nil { - logError(testName, function, args, startTime, "", "file write failed", err) - return - } - file.Close() - // Put encrypted data - if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { - logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) - if err != nil { - logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) - return - } - defer r.Close() - - // Compare the sent object with the received one - recvBuffer := bytes.NewBuffer([]byte{}) - if _, err = io.Copy(recvBuffer, r); err != nil { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) - return - } - if recvBuffer.Len() != len(testCase.buf) { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) - return - } - if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) - return - } - - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -func testBucketNotification() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "SetBucketNotification(bucketName)" - args := map[string]interface{}{ - "bucketName": "", - } - - if os.Getenv("NOTIFY_BUCKET") == "" || - os.Getenv("NOTIFY_SERVICE") == "" || - os.Getenv("NOTIFY_REGION") == "" || - os.Getenv("NOTIFY_ACCOUNTID") == "" || - os.Getenv("NOTIFY_RESOURCE") == "" { - ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - bucketName := os.Getenv("NOTIFY_BUCKET") - args["bucketName"] = bucketName - - topicArn := minio.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) - queueArn := minio.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") - - topicConfig := minio.NewNotificationConfig(topicArn) - - topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) - topicConfig.AddFilterSuffix("jpg") - - queueConfig := minio.NewNotificationConfig(queueArn) - queueConfig.AddEvents(minio.ObjectCreatedAll) - queueConfig.AddFilterPrefix("photos/") - - bNotification := minio.BucketNotification{} - bNotification.AddTopic(topicConfig) - - // Add the same topicConfig again, should have no effect - // because it is duplicated - bNotification.AddTopic(topicConfig) - if len(bNotification.TopicConfigs) != 1 { - logError(testName, function, args, startTime, "", "Duplicate entry added", err) - return - } - - // Add and remove a queue config - bNotification.AddQueue(queueConfig) - bNotification.RemoveQueueByArn(queueArn) - - err = c.SetBucketNotification(bucketName, bNotification) - if err != nil { - logError(testName, function, args, startTime, "", "SetBucketNotification failed", err) - return - } - - bNotification, err = c.GetBucketNotification(bucketName) - if err != nil { - logError(testName, function, args, startTime, "", "GetBucketNotification failed", err) - return - } - - if len(bNotification.TopicConfigs) != 1 { - logError(testName, function, args, startTime, "", "Topic config is empty", err) - return - } - - if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { - logError(testName, function, args, startTime, "", "Couldn't get the suffix", err) - return - } - - err = c.RemoveAllBucketNotification(bucketName) - if err != nil { - logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests comprehensive list of all methods. -func testFunctional() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "testFunctional()" - functionAll := "" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, nil, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - - // Make a new bucket. - function = "MakeBucket(bucketName, region)" - functionAll = "MakeBucket(bucketName, region)" - args["bucketName"] = bucketName - err = c.MakeBucket(bucketName, "us-east-1") - - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - logError(testName, function, args, startTime, "", "File creation failed", err) - return - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - logError(testName, function, args, startTime, "", "File write failed", err) - return - } - } - file.Close() - - // Verify if bucket exits and you have access. - var exists bool - function = "BucketExists(bucketName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - exists, err = c.BucketExists(bucketName) - - if err != nil { - logError(testName, function, args, startTime, "", "BucketExists failed", err) - return - } - if !exists { - logError(testName, function, args, startTime, "", "Could not find the bucket", err) - return - } - - // Asserting the default bucket policy. - function = "GetBucketPolicy(bucketName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - nilPolicy, err := c.GetBucketPolicy(bucketName) - if err != nil { - logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) - return - } - if nilPolicy != "" { - logError(testName, function, args, startTime, "", "policy should be set to nil", err) - return - } - - // Set the bucket policy to 'public readonly'. - function = "SetBucketPolicy(bucketName, readOnlyPolicy)" - functionAll += ", " + function - - readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` - args = map[string]interface{}{ - "bucketName": bucketName, - "bucketPolicy": readOnlyPolicy, - } - - err = c.SetBucketPolicy(bucketName, readOnlyPolicy) - if err != nil { - logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) - return - } - // should return policy `readonly`. - function = "GetBucketPolicy(bucketName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - _, err = c.GetBucketPolicy(bucketName) - if err != nil { - logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) - return - } - - // Make the bucket 'public writeonly'. - function = "SetBucketPolicy(bucketName, writeOnlyPolicy)" - functionAll += ", " + function - - writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` - args = map[string]interface{}{ - "bucketName": bucketName, - "bucketPolicy": writeOnlyPolicy, - } - err = c.SetBucketPolicy(bucketName, writeOnlyPolicy) - - if err != nil { - logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) - return - } - // should return policy `writeonly`. - function = "GetBucketPolicy(bucketName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - - _, err = c.GetBucketPolicy(bucketName) - if err != nil { - logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) - return - } - - // Make the bucket 'public read/write'. - function = "SetBucketPolicy(bucketName, readWritePolicy)" - functionAll += ", " + function - - readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` - - args = map[string]interface{}{ - "bucketName": bucketName, - "bucketPolicy": readWritePolicy, - } - err = c.SetBucketPolicy(bucketName, readWritePolicy) - - if err != nil { - logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) - return - } - // should return policy `readwrite`. - function = "GetBucketPolicy(bucketName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - _, err = c.GetBucketPolicy(bucketName) - if err != nil { - logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) - return - } - - // List all buckets. - function = "ListBuckets()" - functionAll += ", " + function - args = nil - buckets, err := c.ListBuckets() - - if len(buckets) == 0 { - logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) - return - } - if err != nil { - logError(testName, function, args, startTime, "", "ListBuckets failed", err) - return - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err) - return - } - - objectName := bucketName + "unique" - - // Generate data - buf := bytes.Repeat([]byte("f"), 1<<19) - - function = "PutObject(bucketName, objectName, reader, contentType)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "contentType": "", - } - - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(len(buf)) { - logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) - return - } - - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName + "-nolength", - "contentType": "binary/octet-stream", - } - - n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(len(buf)) { - logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) - return - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - - function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "isRecursive": isRecursive, - } - - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) - return - } - - objFound = false - isRecursive = true // Recursive is true. - function = "ListObjectsV2(bucketName, objectName, isRecursive, doneCh)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "isRecursive": isRecursive, - } - - for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) - return - } - - incompObjNotFound := true - - function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "isRecursive": isRecursive, - } - - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) - return - } - - function = "GetObject(bucketName, objectName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - } - newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - if !bytes.Equal(newReadBytes, buf) { - logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err) - return - } - newReader.Close() - - function = "FGetObject(bucketName, objectName, fileName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "fileName": fileName + "-f", - } - err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) - - if err != nil { - logError(testName, function, args, startTime, "", "FGetObject failed", err) - return - } - - function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": "", - "expires": 3600 * time.Second, - } - if _, err = c.PresignedHeadObject(bucketName, "", 3600*time.Second, nil); err == nil { - logError(testName, function, args, startTime, "", "PresignedHeadObject success", err) - return - } - - // Generate presigned HEAD object url. - function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - } - presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) - - if err != nil { - logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) - return - } - // Verify if presigned url works. - resp, err := http.Head(presignedHeadURL.String()) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) - return - } - if resp.StatusCode != http.StatusOK { - logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err) - return - } - if resp.Header.Get("ETag") == "" { - logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) - return - } - resp.Body.Close() - - function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": "", - "expires": 3600 * time.Second, - } - _, err = c.PresignedGetObject(bucketName, "", 3600*time.Second, nil) - if err == nil { - logError(testName, function, args, startTime, "", "PresignedGetObject success", err) - return - } - - // Generate presigned GET object url. - function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - } - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) - return - } - - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) - return - } - if resp.StatusCode != http.StatusOK { - logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) - return - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) - return - } - resp.Body.Close() - if !bytes.Equal(newPresignedBytes, buf) { - logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) - return - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - "reqParams": reqParams, - } - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) - return - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) - return - } - if resp.StatusCode != http.StatusOK { - logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) - return - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) - return - } - if !bytes.Equal(newPresignedBytes, buf) { - logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err) - return - } - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err) - return - } - - function = "PresignedPutObject(bucketName, objectName, expires)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": "", - "expires": 3600 * time.Second, - } - _, err = c.PresignedPutObject(bucketName, "", 3600*time.Second) - if err == nil { - logError(testName, function, args, startTime, "", "PresignedPutObject success", err) - return - } - - function = "PresignedPutObject(bucketName, objectName, expires)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName + "-presigned", - "expires": 3600 * time.Second, - } - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - - if err != nil { - logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) - return - } - - buf = bytes.Repeat([]byte("g"), 1<<19) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) - if err != nil { - logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) - return - } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) - return - } - - newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) - return - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) - return - } - - if !bytes.Equal(newReadBytes, buf) { - logError(testName, function, args, startTime, "", "Bytes mismatch", err) - return - } - - function = "RemoveObject(bucketName, objectName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - } - err = c.RemoveObject(bucketName, objectName) - - if err != nil { - logError(testName, function, args, startTime, "", "RemoveObject failed", err) - return - } - args["objectName"] = objectName + "-f" - err = c.RemoveObject(bucketName, objectName+"-f") - - if err != nil { - logError(testName, function, args, startTime, "", "RemoveObject failed", err) - return - } - - args["objectName"] = objectName + "-nolength" - err = c.RemoveObject(bucketName, objectName+"-nolength") - - if err != nil { - logError(testName, function, args, startTime, "", "RemoveObject failed", err) - return - } - - args["objectName"] = objectName + "-presigned" - err = c.RemoveObject(bucketName, objectName+"-presigned") - - if err != nil { - logError(testName, function, args, startTime, "", "RemoveObject failed", err) - return - } - - function = "RemoveBucket(bucketName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - err = c.RemoveBucket(bucketName) - - if err != nil { - logError(testName, function, args, startTime, "", "RemoveBucket failed", err) - return - } - err = c.RemoveBucket(bucketName) - if err == nil { - logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) - return - } - if err.Error() != "The specified bucket does not exist" { - logError(testName, function, args, startTime, "", "RemoveBucket failed", err) - return - } - - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File Remove failed", err) - return - } - if err = os.Remove(fileName + "-f"); err != nil { - logError(testName, function, args, startTime, "", "File Remove failed", err) - return - } - successLogger(testName, functionAll, args, startTime).Info() -} - -// Test for validating GetObject Reader* methods functioning when the -// object is modified in the object store. -func testGetObjectModified() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - defer c.RemoveBucket(bucketName) - - // Upload an object. - objectName := "myobject" - args["objectName"] = objectName - content := "helloworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) - if err != nil { - logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) - return - } - - defer c.RemoveObject(bucketName, objectName) - - reader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) - return - } - defer reader.Close() - - // Read a few bytes of the object. - b := make([]byte, 5) - n, err := reader.ReadAt(b, 0) - if err != nil { - logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err) - return - } - - // Upload different contents to the same object while object is being read. - newContent := "goodbyeworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) - if err != nil { - logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) - return - } - - // Confirm that a Stat() call in between doesn't change the Object's cached etag. - _, err = reader.Stat() - expectedError := "At least one of the pre-conditions you specified did not hold" - if err.Error() != expectedError { - logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err) - return - } - - // Read again only to find object contents have been modified since last read. - _, err = reader.ReadAt(b, int64(n)) - if err.Error() != expectedError { - logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test validates putObject to upload a file seeked at a given offset. -func testPutObjectUploadSeekedObject() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, fileToUpload, contentType)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileToUpload": "", - "contentType": "binary/octet-stream", - } - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - defer c.RemoveBucket(bucketName) - - var tempfile *os.File - - if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" { - tempfile, err = os.Open(fileName) - if err != nil { - logError(testName, function, args, startTime, "", "File open failed", err) - return - } - args["fileToUpload"] = fileName - } else { - tempfile, err = ioutil.TempFile("", "minio-go-upload-test-") - if err != nil { - logError(testName, function, args, startTime, "", "TempFile create failed", err) - return - } - args["fileToUpload"] = tempfile.Name() - - // Generate 100kB data - if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil { - logError(testName, function, args, startTime, "", "File copy failed", err) - return - } - - defer os.Remove(tempfile.Name()) - - // Seek back to the beginning of the file. - tempfile.Seek(0, 0) - } - var length = 100 * humanize.KiByte - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - args["objectName"] = objectName - - offset := length / 2 - if _, err = tempfile.Seek(int64(offset), 0); err != nil { - logError(testName, function, args, startTime, "", "TempFile seek failed", err) - return - } - - n, err := c.PutObject(bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - if n != int64(length-offset) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid length returned, expected %d got %d", int64(length-offset), n), err) - return - } - tempfile.Close() - - obj, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - defer obj.Close() - - n, err = obj.Seek(int64(offset), 0) - if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) - return - } - if n != int64(offset) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err) - return - } - - n, err = c.PutObject(bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - if n != int64(length-offset) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests bucket re-create errors. -func testMakeBucketErrorV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": "", - "region": "eu-west-1", - } - - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - region := "eu-west-1" - args["bucketName"] = bucketName - args["region"] = region - - // Make a new bucket in 'eu-west-1'. - if err = c.MakeBucket(bucketName, region); err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - if err = c.MakeBucket(bucketName, region); err == nil { - logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) - return - } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - logError(testName, function, args, startTime, "", "Invalid error returned by server", err) - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test get object reader to not throw error on being closed twice. -func testGetObjectClosedTwiceV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": "", - "region": "eu-west-1", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) - return - } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - if err := r.Close(); err == nil { - logError(testName, function, args, startTime, "", "Object is already closed, should return error", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests FPutObject hidden contentType setting -func testFPutObjectV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "FPutObject(bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileName": "", - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Make a temp file with 11*1024*1024 bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") - if err != nil { - logError(testName, function, args, startTime, "", "TempFile creation failed", err) - return - } - - r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) - n, err := io.CopyN(file, r, 11*1024*1024) - if err != nil { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) - return - } - - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - logError(testName, function, args, startTime, "", "File close failed", err) - return - } - - // Set base object name - objectName := bucketName + "FPutObject" - args["objectName"] = objectName - args["fileName"] = file.Name() - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) - return - } - - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - args["objectName"] = objectName + "-Octet" - args["contentType"] = "" - - n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) - return - } - - // Add extension to temp file name - fileName := file.Name() - err = os.Rename(file.Name(), fileName+".gtar") - if err != nil { - logError(testName, function, args, startTime, "", "Rename failed", err) - return - } - - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - args["objectName"] = objectName + "-Octet" - args["contentType"] = "" - args["fileName"] = fileName + ".gtar" - - n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) - return - } - - // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - if rStandard.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) - return - } - - rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - if rOctet.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err) - return - } - - rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - if rGTar.ContentType != "application/x-gtar" { - logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - err = os.Remove(fileName + ".gtar") - if err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -// Tests various bucket supported formats. -func testMakeBucketRegionsV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": "", - "region": "eu-west-1", - } - - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { - args["bucketName"] = bucketName + ".withperiod" - args["region"] = "us-west-2" - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName+".withperiod", c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests get object ReaderSeeker interface methods. -func testGetObjectReadSeekFunctionalV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - // Save the data. - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - defer r.Close() - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) - return - } - - offset := int64(2048) - n, err = r.Seek(offset, 0) - if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) - return - } - if n != offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) - return - } - n, err = r.Seek(0, 1) - if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) - return - } - if n != offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) - return - } - _, err = r.Seek(offset, 2) - if err == nil { - logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) - return - } - n, err = r.Seek(-offset, 2) - if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) - return - } - if n != st.Size-offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) - return - } - - var buffer1 bytes.Buffer - if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - } - if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { - logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) - return - } - - // Seek again and read again. - n, err = r.Seek(offset-1, 0) - if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) - return - } - if n != (offset - 1) { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) - return - } - - var buffer2 bytes.Buffer - if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - } - // Verify now lesser bytes. - if !bytes.Equal(buf[2047:], buffer2.Bytes()) { - logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests get object ReaderAt interface methods. -func testGetObjectReadAtFunctionalV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) - return - } - - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - defer r.Close() - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) - return - } - - offset := int64(2048) - - // Read directly - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - m, err := r.ReadAt(buf2, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf2) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err) - return - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - offset += 512 - m, err = r.ReadAt(buf3, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf3) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err) - return - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - offset += 512 - m, err = r.ReadAt(buf4, offset) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf4) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err) - return - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) - return - } - - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) - if err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - } - if m != len(buf5) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err) - return - } - if !bytes.Equal(buf, buf5) { - logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) - return - } - - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests copy object -func testCopyObjectV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) - return - } - - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - // Check the various fields of source object against destination object. - objInfo, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - r.Close() - - // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - args["source"] = src - - // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) - return - } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) - return - } - err = src.SetMatchETagCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) - return - } - err = src.SetMatchETagExceptCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) - return - } - - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) - return - } - err = src.SetMatchETagCond(objInfo.ETag) - if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) - return - } - - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) - args["destination"] = dst - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - - // Perform the Copy - err = c.CopyObject(dst, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return - } - - // Source object - r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - // Check the various fields of source object against destination object. - objInfo, err = r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - objInfoCopy, err := readerCopy.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - if objInfo.Size != objInfoCopy.Size { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err) - return - } - - // Close all the readers. - r.Close() - readerCopy.Close() - - // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) - return - } - err = src.SetMatchETagExceptCond(objInfo.ETag) - if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) - return - } - - // Perform the Copy which should fail - err = c.CopyObject(dst, src) - if err == nil { - logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - if err = cleanupBucket(bucketName+"-copy", c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -func testComposeObjectErrorCasesWrapper(c *minio.Client) { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "ComposeObject(destination, sourceList)" - args := map[string]interface{}{} - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Test that more than 10K source objects cannot be - // concatenated. - srcArr := [10001]minio.SourceInfo{} - srcSlice := srcArr[:] - dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - - args["destination"] = dst - // Just explain about srcArr in args["sourceList"] - // to stop having 10,001 null headers logged - args["sourceList"] = "source array of 10,001 elements" - if err := c.ComposeObject(dst, srcSlice); err == nil { - logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) - return - } else if err.Error() != "There must be as least one and up to 10000 source objects." { - logError(testName, function, args, startTime, "", "Got unexpected error", err) - return - } - - // Create a source with invalid offset spec and check that - // error is returned: - // 1. Create the source object. - const badSrcSize = 5 * 1024 * 1024 - buf := bytes.Repeat([]byte("1"), badSrcSize) - _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - // 2. Set invalid range spec on the object (going beyond - // object size) - badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) - err = badSrc.SetRange(1, badSrcSize) - if err != nil { - logError(testName, function, args, startTime, "", "Setting NewSourceInfo failed", err) - return - } - // 3. ComposeObject call should fail. - if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { - logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) - return - } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { - logError(testName, function, args, startTime, "", "Got invalid error", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test expected error cases -func testComposeObjectErrorCasesV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "ComposeObject(destination, sourceList)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - testComposeObjectErrorCasesWrapper(c) -} - -func testComposeMultipleSources(c *minio.Client) { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "ComposeObject(destination, sourceList)" - args := map[string]interface{}{ - "destination": "", - "sourceList": "", - } - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Upload a small source object - const srcSize = 1024 * 1024 * 5 - buf := bytes.Repeat([]byte("1"), srcSize) - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - // We will append 10 copies of the object. - srcs := []minio.SourceInfo{} - for i := 0; i < 10; i++ { - srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) - } - // make the last part very small - err = srcs[9].SetRange(0, 0) - if err != nil { - logError(testName, function, args, startTime, "", "SetRange failed", err) - return - } - args["sourceList"] = srcs - - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) - args["destination"] = dst - - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - err = c.ComposeObject(dst, srcs) - if err != nil { - logError(testName, function, args, startTime, "", "ComposeObject failed", err) - return - } - - objProps, err := c.StatObject(bucketName, "dstObject", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - - if objProps.Size != 9*srcSize+1 { - logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -// Test concatenating multiple objects objects -func testCompose10KSourcesV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "ComposeObject(destination, sourceList)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - testComposeMultipleSources(c) -} - -func testEncryptedEmptyObject() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader, objectSize, opts)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) - return - } - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object")) - - // 1. create an sse-c encrypted object to copy by uploading - const srcSize = 0 - var buf []byte // Empty buffer - args["objectName"] = "object" - _, err = c.PutObject(bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject call failed", err) - return - } - - // 2. Test CopyObject for an empty object - dstInfo, err := minio.NewDestinationInfo(bucketName, "new-object", sse, nil) - if err != nil { - args["objectName"] = "new-object" - function = "NewDestinationInfo(bucketName, objectName, sse, userMetadata)" - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - srcInfo := minio.NewSourceInfo(bucketName, "object", sse) - if err = c.CopyObject(dstInfo, srcInfo); err != nil { - function = "CopyObject(dstInfo, srcInfo)" - logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) - return - } - - // 3. Test Key rotation - newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) - dstInfo, err = minio.NewDestinationInfo(bucketName, "new-object", newSSE, nil) - if err != nil { - args["objectName"] = "new-object" - function = "NewDestinationInfo(bucketName, objectName, encryptSSEC, userMetadata)" - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - - srcInfo = minio.NewSourceInfo(bucketName, "new-object", sse) - if err = c.CopyObject(dstInfo, srcInfo); err != nil { - function = "CopyObject(dstInfo, srcInfo)" - logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) - return - } - - // 4. Download the object. - reader, err := c.GetObject(bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - defer reader.Close() - - decBytes, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err) - return - } - if !bytes.Equal(decBytes, buf) { - logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err) - return - } - // Delete all objects and buckets - delete(args, "objectName") - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -func testEncryptedCopyObjectWrapper(c *minio.Client) { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) - sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) - - // 1. create an sse-c encrypted object to copy by uploading - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ - ServerSideEncryption: sseSrc, - }) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject call failed", err) - return - } - - // 2. copy object and change encryption key - src := minio.NewSourceInfo(bucketName, "srcObject", sseSrc) - args["source"] = src - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", sseDst, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - args["destination"] = dst - - err = c.CopyObject(dst, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return - } - - // 3. get copied object and check if content is equal - coreClient := minio.Core{c} - reader, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: sseDst}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - decBytes, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - if !bytes.Equal(decBytes, buf) { - logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) - return - } - reader.Close() - - // Test key rotation for source object in-place. - newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key - dst, err = minio.NewDestinationInfo(bucketName, "srcObject", newSSE, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - args["destination"] = dst - - err = c.CopyObject(dst, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return - } - - // Get copied object and check if content is equal - reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - decBytes, err = ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - if !bytes.Equal(decBytes, buf) { - logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) - return - } - reader.Close() - - // Test in-place decryption. - dst, err = minio.NewDestinationInfo(bucketName, "srcObject", nil, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - args["destination"] = dst - - src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) - args["source"] = src - err = c.CopyObject(dst, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return - } - - // Get copied decrypted object and check if content is equal - reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - defer reader.Close() - - decBytes, err = ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - if !bytes.Equal(decBytes, buf) { - logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test encrypted copy object -func testEncryptedCopyObject() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // c.TraceOn(os.Stderr) - testEncryptedCopyObjectWrapper(c) -} - -// Test encrypted copy object -func testEncryptedCopyObjectV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // c.TraceOn(os.Stderr) - testEncryptedCopyObjectWrapper(c) -} - -func testDecryptedCopyObject() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" - if err = c.MakeBucket(bucketName, "us-east-1"); err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)) - _, err = c.PutObject(bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ - ServerSideEncryption: encryption, - }) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject call failed", err) - return - } - - src := minio.NewSourceInfo(bucketName, objectName, encrypt.SSECopy(encryption)) - args["source"] = src - dst, err := minio.NewDestinationInfo(bucketName, "decrypted-"+objectName, nil, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - args["destination"] = dst - - if err = c.CopyObject(dst, src); err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return - } - if _, err = c.GetObject(bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -func testUserMetadataCopying() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - // c.TraceOn(os.Stderr) - testUserMetadataCopyingWrapper(c) -} - -func testUserMetadataCopyingWrapper(c *minio.Client) { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - fetchMeta := func(object string) (h http.Header) { - objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - h = make(http.Header) - for k, vs := range objInfo.Metadata { - if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { - for _, v := range vs { - h.Add(k, v) - } - } - } - return h - } - - // 1. create a client encrypted object to copy by uploading - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB - metadata := make(http.Header) - metadata.Set("x-amz-meta-myheader", "myvalue") - m := make(map[string]string) - m["x-amz-meta-myheader"] = "myvalue" - _, err = c.PutObject(bucketName, "srcObject", - bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err) - return - } - if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { - logError(testName, function, args, startTime, "", "Metadata match failed", err) - return - } - - // 2. create source - src := minio.NewSourceInfo(bucketName, "srcObject", nil) - // 2.1 create destination with metadata set - dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - - // 3. Check that copying to an object with metadata set resets - // the headers on the copy. - args["source"] = src - args["destination"] = dst1 - err = c.CopyObject(dst1, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return - } - - expectedHeaders := make(http.Header) - expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { - logError(testName, function, args, startTime, "", "Metadata match failed", err) - return - } - - // 4. create destination with no metadata set and same source - dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", nil, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - src = minio.NewSourceInfo(bucketName, "srcObject", nil) - - // 5. Check that copying to an object with no metadata set, - // copies metadata. - args["source"] = src - args["destination"] = dst2 - err = c.CopyObject(dst2, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return - } - - expectedHeaders = metadata - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { - logError(testName, function, args, startTime, "", "Metadata match failed", err) - return - } - - // 6. Compose a pair of sources. - srcs := []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", nil, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - - function = "ComposeObject(destination, sources)" - args["source"] = srcs - args["destination"] = dst3 - err = c.ComposeObject(dst3, srcs) - if err != nil { - logError(testName, function, args, startTime, "", "ComposeObject failed", err) - return - } - - // Check that no headers are copied in this case - if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { - logError(testName, function, args, startTime, "", "Metadata match failed", err) - return - } - - // 7. Compose a pair of sources with dest user metadata set. - srcs = []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } - - function = "ComposeObject(destination, sources)" - args["source"] = srcs - args["destination"] = dst4 - err = c.ComposeObject(dst4, srcs) - if err != nil { - logError(testName, function, args, startTime, "", "ComposeObject failed", err) - return - } - - // Check that no headers are copied in this case - expectedHeaders = make(http.Header) - expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { - logError(testName, function, args, startTime, "", "Metadata match failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -func testUserMetadataCopyingV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // c.TraceOn(os.Stderr) - testUserMetadataCopyingWrapper(c) -} - -func testStorageClassMetadataPutObject() { - // initialize logging params - startTime := time.Now() - function := "testStorageClassMetadataPutObject()" - args := map[string]interface{}{} - testName := getFuncName() - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) - return - } - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - fetchMeta := func(object string) (h http.Header) { - objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - h = make(http.Header) - for k, vs := range objInfo.Metadata { - if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { - for _, v := range vs { - h.Add(k, v) - } - } - } - return h - } - - metadata := make(http.Header) - metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") - - emptyMetadata := make(http.Header) - - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB - - _, err = c.PutObject(bucketName, "srcObjectRRSClass", - bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - // Get the returned metadata - returnedMeta := fetchMeta("srcObjectRRSClass") - - // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) - if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { - logError(testName, function, args, startTime, "", "Metadata match failed", err) - return - } - - metadata = make(http.Header) - metadata.Set("x-amz-storage-class", "STANDARD") - - _, err = c.PutObject(bucketName, "srcObjectSSClass", - bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) { - logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -func testStorageClassInvalidMetadataPutObject() { - // initialize logging params - startTime := time.Now() - function := "testStorageClassInvalidMetadataPutObject()" - args := map[string]interface{}{} - testName := getFuncName() - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) - return - } - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB - - _, err = c.PutObject(bucketName, "srcObjectRRSClass", - bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"}) - if err == nil { - logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -func testStorageClassMetadataCopyObject() { - // initialize logging params - startTime := time.Now() - function := "testStorageClassMetadataCopyObject()" - args := map[string]interface{}{} - testName := getFuncName() - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) - return - } - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") - // Make a new bucket in 'us-east-1' (source bucket). - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - fetchMeta := func(object string) (h http.Header) { - objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - h = make(http.Header) - for k, vs := range objInfo.Metadata { - if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { - for _, v := range vs { - h.Add(k, v) - } - } - } - return h - } - - metadata := make(http.Header) - metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") - - emptyMetadata := make(http.Header) - - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) - - // Put an object with RRS Storage class - _, err = c.PutObject(bucketName, "srcObjectRRSClass", - bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - // Make server side copy of object uploaded in previous step - src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) - dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) - c.CopyObject(dst, src) - - // Get the returned metadata - returnedMeta := fetchMeta("srcObjectRRSClassCopy") - - // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) - if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { - logError(testName, function, args, startTime, "", "Metadata match failed", err) - return - } - - metadata = make(http.Header) - metadata.Set("x-amz-storage-class", "STANDARD") - - // Put an object with Standard Storage class - _, err = c.PutObject(bucketName, "srcObjectSSClass", - bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - // Make server side copy of object uploaded in previous step - src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) - dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) - c.CopyObject(dst, src) - - // Fetch the meta data of copied object - if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { - logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - successLogger(testName, function, args, startTime).Info() -} - -// Test put object with size -1 byte object. -func testPutObjectNoLengthV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader, size, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "size": -1, - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - objectName := bucketName + "unique" - args["objectName"] = objectName - - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") - defer reader.Close() - args["size"] = bufSize - - // Upload an object. - n, err := c.PutObject(bucketName, objectName, reader, -1, minio.PutObjectOptions{}) - - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) - return - } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(n), err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test put objects of unknown size. -func testPutObjectsUnknownV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader,size,opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "size": "", - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Issues are revealed by trying to upload multiple files of unknown size - // sequentially (on 4GB machines) - for i := 1; i <= 4; i++ { - // Simulate that we could be receiving byte slices of data that we want - // to upload as a file - rpipe, wpipe := io.Pipe() - defer rpipe.Close() - go func() { - b := []byte("test") - wpipe.Write(b) - wpipe.Close() - }() - - // Upload the object. - objectName := fmt.Sprintf("%sunique%d", bucketName, i) - args["objectName"] = objectName - - n, err := c.PutObject(bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) - return - } - args["size"] = n - if n != int64(4) { - logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(n), err) - return - } - - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test put object with 0 byte object. -func testPutObject0ByteV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader, size, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "size": 0, - "opts": "", - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - objectName := bucketName + "unique" - args["objectName"] = objectName - args["opts"] = minio.PutObjectOptions{} - - // Upload an object. - n, err := c.PutObject(bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) - - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) - return - } - if n != 0 { - logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(n), err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() -} - -// Test expected error cases -func testComposeObjectErrorCases() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "ComposeObject(destination, sourceList)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - testComposeObjectErrorCasesWrapper(c) -} - -// Test concatenating 10K objects -func testCompose10KSources() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "ComposeObject(destination, sourceList)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) - return - } - - testComposeMultipleSources(c) -} - -// Tests comprehensive list of all methods. -func testFunctionalV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "testFunctionalV2()" - functionAll := "" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // Enable to debug - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - location := "us-east-1" - // Make a new bucket. - function = "MakeBucket(bucketName, location)" - functionAll = "MakeBucket(bucketName, location)" - args = map[string]interface{}{ - "bucketName": bucketName, - "location": location, - } - err = c.MakeBucket(bucketName, location) - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - logError(testName, function, args, startTime, "", "file create failed", err) - return - } - for i := 0; i < 3; i++ { - buf := make([]byte, rand.Intn(1<<19)) - _, err = file.Write(buf) - if err != nil { - logError(testName, function, args, startTime, "", "file write failed", err) - return - } - } - file.Close() - - // Verify if bucket exits and you have access. - var exists bool - function = "BucketExists(bucketName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - exists, err = c.BucketExists(bucketName) - if err != nil { - logError(testName, function, args, startTime, "", "BucketExists failed", err) - return - } - if !exists { - logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err) - return - } - - // Make the bucket 'public read/write'. - function = "SetBucketPolicy(bucketName, bucketPolicy)" - functionAll += ", " + function - - readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}` - - args = map[string]interface{}{ - "bucketName": bucketName, - "bucketPolicy": readWritePolicy, - } - err = c.SetBucketPolicy(bucketName, readWritePolicy) - - if err != nil { - logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) - return - } - - // List all buckets. - function = "ListBuckets()" - functionAll += ", " + function - args = nil - buckets, err := c.ListBuckets() - if len(buckets) == 0 { - logError(testName, function, args, startTime, "", "List buckets cannot be empty", err) - return - } - if err != nil { - logError(testName, function, args, startTime, "", "ListBuckets failed", err) - return - } - - // Verify if previously created bucket is listed in list buckets. - bucketFound := false - for _, bucket := range buckets { - if bucket.Name == bucketName { - bucketFound = true - } - } - - // If bucket not found error out. - if !bucketFound { - logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err) - return - } - - objectName := bucketName + "unique" - - // Generate data - buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) - - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "contentType": "", - } - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - if n != int64(len(buf)) { - logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) - return - } - - objectNameNoLength := objectName + "-nolength" - args["objectName"] = objectNameNoLength - n, err = c.PutObject(bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - if n != int64(len(buf)) { - logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) - return - } - - // Instantiate a done channel to close all listing. - doneCh := make(chan struct{}) - defer close(doneCh) - - objFound := false - isRecursive := true // Recursive is true. - function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "isRecursive": isRecursive, - } - for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { - if obj.Key == objectName { - objFound = true - break - } - } - if !objFound { - logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err) - return - } - - incompObjNotFound := true - function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "isRecursive": isRecursive, - } - for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { - if objIncompl.Key != "" { - incompObjNotFound = false - break - } - } - if !incompObjNotFound { - logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) - return - } - - function = "GetObject(bucketName, objectName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - } - newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - newReadBytes, err := ioutil.ReadAll(newReader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - newReader.Close() - - if !bytes.Equal(newReadBytes, buf) { - logError(testName, function, args, startTime, "", "Bytes mismatch", err) - return - } - - function = "FGetObject(bucketName, objectName, fileName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "fileName": fileName + "-f", - } - err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FgetObject failed", err) - return - } - - // Generate presigned HEAD object url. - function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - } - presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) - return - } - // Verify if presigned url works. - resp, err := http.Head(presignedHeadURL.String()) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) - return - } - if resp.StatusCode != http.StatusOK { - logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err) - return - } - if resp.Header.Get("ETag") == "" { - logError(testName, function, args, startTime, "", "Got empty ETag", err) - return - } - resp.Body.Close() - - // Generate presigned GET object url. - function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - "expires": 3600 * time.Second, - } - presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) - return - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject URL GET request failed", err) - return - } - if resp.StatusCode != http.StatusOK { - logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) - return - } - newPresignedBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - resp.Body.Close() - if !bytes.Equal(newPresignedBytes, buf) { - logError(testName, function, args, startTime, "", "Bytes mismatch", err) - return - } - - // Set request parameters. - reqParams := make(url.Values) - reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") - // Generate presigned GET object url. - args["reqParams"] = reqParams - presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) - return - } - // Verify if presigned url works. - resp, err = http.Get(presignedGetURL.String()) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedGetObject URL GET request failed", err) - return - } - if resp.StatusCode != http.StatusOK { - logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) - return - } - newPresignedBytes, err = ioutil.ReadAll(resp.Body) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - if !bytes.Equal(newPresignedBytes, buf) { - logError(testName, function, args, startTime, "", "Bytes mismatch", err) - return - } - // Verify content disposition. - if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { - logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err) - return - } - - function = "PresignedPutObject(bucketName, objectName, expires)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName + "-presigned", - "expires": 3600 * time.Second, - } - presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) - return - } - - // Generate data more than 32K - buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) - if err != nil { - logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) - return - } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) - return - } - - function = "GetObject(bucketName, objectName)" - functionAll += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName + "-presigned", - } - newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - newReader.Close() - - if !bytes.Equal(newReadBytes, buf) { - logError(testName, function, args, startTime, "", "Bytes mismatch", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) - return - } - if err = os.Remove(fileName + "-f"); err != nil { - logError(testName, function, args, startTime, "", "File removes failed", err) - return - } - successLogger(testName, functionAll, args, startTime).Info() -} - -// Test get object with GetObjectWithContext -func testGetObjectWithContext() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObjectWithContext(ctx, bucketName, objectName)" - args := map[string]interface{}{ - "ctx": "", - "bucketName": "", - "objectName": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - args["ctx"] = ctx - defer cancel() - - r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) - return - } - - if _, err = r.Stat(); err == nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) - return - } - r.Close() - - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - args["ctx"] = ctx - defer cancel() - - // Read the data back - r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext failed", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "object Stat call failed", err) - return - } - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err) - return - } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", "object Close() call failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Test get object with FGetObjectWithContext -func testFGetObjectWithContext() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "FGetObjectWithContext(ctx, bucketName, objectName, fileName)" - args := map[string]interface{}{ - "ctx": "", - "bucketName": "", - "objectName": "", - "fileName": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - bufSize := dataFileMap["datafile-1-MB"] - var reader = getDataReader("datafile-1-MB") - defer reader.Close() - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - args["ctx"] = ctx - defer cancel() - - fileName := "tempfile-context" - args["fileName"] = fileName - // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) - if err == nil { - logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) - return - } - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - defer cancel() - - // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FGetObjectWithContext with long timeout failed", err) - return - } - if err = os.Remove(fileName + "-fcontext"); err != nil { - logError(testName, function, args, startTime, "", "Remove file failed", err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Test validates putObject with context to see if request cancellation is honored for V2. -func testPutObjectWithContextV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObjectWithContext(ctx, bucketName, objectName, reader, size, opts)" - args := map[string]interface{}{ - "ctx": "", - "bucketName": "", - "objectName": "", - "size": "", - "opts": "", - } - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - defer c.RemoveBucket(bucketName) - bufSize := dataFileMap["datatfile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - args["objectName"] = objectName - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - args["ctx"] = ctx - args["size"] = bufSize - defer cancel() - - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithContext with short timeout failed", err) - return - } - - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - args["ctx"] = ctx - - defer cancel() - reader = getDataReader("datafile-33-kB") - defer reader.Close() - _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Test get object with GetObjectWithContext -func testGetObjectWithContextV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "GetObjectWithContext(ctx, bucketName, objectName)" - args := map[string]interface{}{ - "ctx": "", - "bucketName": "", - "objectName": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject call failed", err) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - args["ctx"] = ctx - defer cancel() - - r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) - return - } - if _, err = r.Stat(); err == nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) - return - } - r.Close() - - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - defer cancel() - - // Read the data back - r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObjectWithContext shouldn't fail on longer timeout", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "object Stat call failed", err) - return - } - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err) - return - } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", " object Close() call failed", err) - return - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Test get object with FGetObjectWithContext -func testFGetObjectWithContextV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "FGetObjectWithContext(ctx, bucketName, objectName,fileName)" - args := map[string]interface{}{ - "ctx": "", - "bucketName": "", - "objectName": "", - "fileName": "", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket call failed", err) - return - } - - bufSize := dataFileMap["datatfile-1-MB"] - var reader = getDataReader("datafile-1-MB") - defer reader.Close() - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject call failed", err) - return - } - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) - args["ctx"] = ctx - defer cancel() - - fileName := "tempfile-context" - args["fileName"] = fileName - - // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) - if err == nil { - logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) - return - } - ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) - defer cancel() - - // Read the data back - err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FGetObjectWithContext call shouldn't fail on long timeout", err) - return - } - - if err = os.Remove(fileName + "-fcontext"); err != nil { - logError(testName, function, args, startTime, "", "Remove file failed", err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Test list object v1 and V2 storage class fields -func testListObjects() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)" - args := map[string]interface{}{ - "bucketName": "", - "objectPrefix": "", - "recursive": "true", - } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") - if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return - } - - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - // Save the data - objectName1 := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - _, err = c.PutObject(bucketName, objectName1, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "STANDARD"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject1 call failed", err) - return - } - - bufSize1 := dataFileMap["datafile-33-kB"] - var reader1 = getDataReader("datafile-33-kB") - defer reader1.Close() - objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - _, err = c.PutObject(bucketName, objectName2, reader1, int64(bufSize1), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "REDUCED_REDUNDANCY"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject2 call failed", err) - return - } - - // Create a done channel to control 'ListObjects' go routine. - doneCh := make(chan struct{}) - // Exit cleanly upon return. - defer close(doneCh) - - // check for storage-class from ListObjects result - for objInfo := range c.ListObjects(bucketName, "", true, doneCh) { - if objInfo.Err != nil { - logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) - return - } - if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { - logError(testName, function, args, startTime, "", "ListObjects doesn't return expected storage class", err) - return - } - if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { - logError(testName, function, args, startTime, "", "ListObjects doesn't return expected storage class", err) - return - } - } - - // check for storage-class from ListObjectsV2 result - for objInfo := range c.ListObjectsV2(bucketName, "", true, doneCh) { - if objInfo.Err != nil { - logError(testName, function, args, startTime, "", "ListObjectsV2 failed unexpectedly", err) - return - } - if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { - logError(testName, function, args, startTime, "", "ListObjectsV2 doesn't return expected storage class", err) - return - } - if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { - logError(testName, function, args, startTime, "", "ListObjectsV2 doesn't return expected storage class", err) - return - } - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - - successLogger(testName, function, args, startTime).Info() - -} - -// Convert string to bool and always return false if any error -func mustParseBool(str string) bool { - b, err := strconv.ParseBool(str) - if err != nil { - return false - } - return b -} - -func main() { - // Output to stdout instead of the default stderr - log.SetOutput(os.Stdout) - // create custom formatter - mintFormatter := mintJSONFormatter{} - // set custom formatter - log.SetFormatter(&mintFormatter) - // log Info or above -- success cases are Info level, failures are Fatal level - log.SetLevel(log.InfoLevel) - - tls := mustParseBool(os.Getenv(enableHTTPS)) - // execute tests - if isFullMode() { - testMakeBucketErrorV2() - testGetObjectClosedTwiceV2() - testFPutObjectV2() - testMakeBucketRegionsV2() - testGetObjectReadSeekFunctionalV2() - testGetObjectReadAtFunctionalV2() - testCopyObjectV2() - testFunctionalV2() - testComposeObjectErrorCasesV2() - testCompose10KSourcesV2() - testUserMetadataCopyingV2() - testPutObject0ByteV2() - testPutObjectNoLengthV2() - testPutObjectsUnknownV2() - testGetObjectWithContextV2() - testFPutObjectWithContextV2() - testFGetObjectWithContextV2() - testPutObjectWithContextV2() - testMakeBucketError() - testMakeBucketRegions() - testPutObjectWithMetadata() - testPutObjectReadAt() - testPutObjectStreaming() - testGetObjectSeekEnd() - testGetObjectClosedTwice() - testRemoveMultipleObjects() - testFPutObjectMultipart() - testFPutObject() - testGetObjectReadSeekFunctional() - testGetObjectReadAtFunctional() - testPresignedPostPolicy() - testCopyObject() - testComposeObjectErrorCases() - testCompose10KSources() - testUserMetadataCopying() - testBucketNotification() - testFunctional() - testGetObjectModified() - testPutObjectUploadSeekedObject() - testGetObjectWithContext() - testFPutObjectWithContext() - testFGetObjectWithContext() - testPutObjectWithContext() - testStorageClassMetadataPutObject() - testStorageClassInvalidMetadataPutObject() - testStorageClassMetadataCopyObject() - testPutObjectWithContentLanguage() - testListObjects() - - // SSE-C tests will only work over TLS connection. - if tls { - testEncryptionPutGet() - testEncryptionFPut() - testEncryptedGetObjectReadAtFunctional() - testEncryptedGetObjectReadSeekFunctional() - testEncryptedCopyObjectV2() - testEncryptedCopyObject() - testEncryptedEmptyObject() - testDecryptedCopyObject() - } - } else { - testFunctional() - testFunctionalV2() - } -} diff --git a/vendor/github.com/minio/minio-go/retry.go b/vendor/github.com/minio/minio-go/retry.go index 22c94347e..2a7670786 100644 --- a/vendor/github.com/minio/minio-go/retry.go +++ b/vendor/github.com/minio/minio-go/retry.go @@ -85,36 +85,32 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio return attemptCh } -// isNetErrorRetryable - is network error retryable. -func isNetErrorRetryable(err error) bool { +// isHTTPReqErrorRetryable - is http requests error retryable, such +// as i/o timeout, connection broken etc.. +func isHTTPReqErrorRetryable(err error) bool { if err == nil { return false } - switch err.(type) { - case net.Error: - switch err.(type) { + switch e := err.(type) { + case *url.Error: + switch e.Err.(type) { case *net.DNSError, *net.OpError, net.UnknownNetworkError: return true - case *url.Error: - // For a URL error, where it replies back "connection closed" - // retry again. - if strings.Contains(err.Error(), "Connection closed by foreign host") { - return true - } - default: - if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { - // If error is - tlsHandshakeTimeoutError, retry. - return true - } else if strings.Contains(err.Error(), "i/o timeout") { - // If error is - tcp timeoutError, retry. - return true - } else if strings.Contains(err.Error(), "connection timed out") { - // If err is a net.Dial timeout, retry. - return true - } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { - // If error is transport connection broken, retry. - return true - } + } + if strings.Contains(err.Error(), "Connection closed by foreign host") { + return true + } else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { + // If error is - tlsHandshakeTimeoutError, retry. + return true + } else if strings.Contains(err.Error(), "i/o timeout") { + // If error is - tcp timeoutError, retry. + return true + } else if strings.Contains(err.Error(), "connection timed out") { + // If err is a net.Dial timeout, retry. + return true + } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { + // If error is transport connection broken, retry. + return true } } return false diff --git a/vendor/github.com/minio/minio-go/utils.go b/vendor/github.com/minio/minio-go/utils.go index f80e25c41..8483f3834 100644 --- a/vendor/github.com/minio/minio-go/utils.go +++ b/vendor/github.com/minio/minio-go/utils.go @@ -223,6 +223,7 @@ var supportedHeaders = []string{ "content-disposition", "content-language", "x-amz-website-redirect-location", + "expires", // Add more supported headers here. } @@ -267,5 +268,5 @@ func isSSEHeader(headerKey string) bool { func isAmzHeader(headerKey string) bool { key := strings.ToLower(headerKey) - return strings.HasPrefix(key, "x-amz-meta-") || key == "x-amz-acl" + return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey) } diff --git a/vendor/vendor.json b/vendor/vendor.json index 747f18bf5..052bfbd66 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -645,10 +645,10 @@ "revisionTime": "2016-02-29T08:42:30-08:00" }, { - "checksumSHA1": "hdWmWbGpljSQMBOcpcwhAnP2aaQ=", + "checksumSHA1": "Wbe5TjRIOZiWVu4l4dwzCw/uP9w=", "path": "github.com/minio/minio-go", - "revision": "10531abd0af1579a12dc1977d67c0fec2b348679", - "revisionTime": "2018-06-13T23:01:28Z" + "revision": "519049881e73150d1bbeac1d443e7c96b76e1b8d", + "revisionTime": "2018-09-05T00:47:51Z" }, { "checksumSHA1": "Qsj+6JPmJ8R5rFNQSHqRb8xAwOw=",