From 33fe42df8aad3b4879e7e9108c897042eb9409fa Mon Sep 17 00:00:00 2001 From: Ersan Date: Wed, 14 Mar 2018 18:36:54 -0700 Subject: [PATCH] Adds context support for nas, oss and s3 --- cmd/gateway-unsupported.go | 41 ++++++++++++++------------- cmd/gateway/nas/gateway-nas.go | 8 ++++-- cmd/gateway/oss/gateway-oss.go | 51 +++++++++++++++++----------------- cmd/gateway/s3/gateway-s3.go | 51 +++++++++++++++++----------------- cmd/logger/log.go | 8 ++++-- cmd/utils.go | 4 ++- 6 files changed, 86 insertions(+), 77 deletions(-) diff --git a/cmd/gateway-unsupported.go b/cmd/gateway-unsupported.go index ede722aac..5af4e968f 100644 --- a/cmd/gateway-unsupported.go +++ b/cmd/gateway-unsupported.go @@ -17,6 +17,7 @@ package cmd import ( + "context" "time" "github.com/minio/minio-go/pkg/policy" @@ -29,87 +30,87 @@ import ( type GatewayUnsupported struct{} // ListMultipartUploads lists all multipart uploads. -func (a GatewayUnsupported) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) { +func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) { return lmi, errors.Trace(NotImplemented{}) } // NewMultipartUpload upload object in multiple parts -func (a GatewayUnsupported) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) { +func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) { return "", errors.Trace(NotImplemented{}) } // CopyObjectPart copy part of object to uploadID for another object -func (a GatewayUnsupported) CopyObjectPart(srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo) (pi PartInfo, err error) { +func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo) (pi PartInfo, err error) { return pi, errors.Trace(NotImplemented{}) } // PutObjectPart puts a part of object in bucket -func (a GatewayUnsupported) PutObjectPart(bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) { +func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) { return pi, errors.Trace(NotImplemented{}) } // ListObjectParts returns all object parts for specified object in specified bucket -func (a GatewayUnsupported) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) { +func (a GatewayUnsupported) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) { return lpi, errors.Trace(NotImplemented{}) } // AbortMultipartUpload aborts a ongoing multipart upload -func (a GatewayUnsupported) AbortMultipartUpload(bucket string, object string, uploadID string) error { +func (a GatewayUnsupported) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error { return errors.Trace(NotImplemented{}) } // CompleteMultipartUpload completes ongoing multipart upload and finalizes object -func (a GatewayUnsupported) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) { +func (a GatewayUnsupported) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) { return oi, errors.Trace(NotImplemented{}) } // SetBucketPolicy sets policy on bucket -func (a GatewayUnsupported) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error { +func (a GatewayUnsupported) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error { return errors.Trace(NotImplemented{}) } // GetBucketPolicy will get policy on bucket -func (a GatewayUnsupported) GetBucketPolicy(bucket string) (bal policy.BucketAccessPolicy, err error) { +func (a GatewayUnsupported) GetBucketPolicy(ctx context.Context, bucket string) (bal policy.BucketAccessPolicy, err error) { return bal, errors.Trace(NotImplemented{}) } // DeleteBucketPolicy deletes all policies on bucket -func (a GatewayUnsupported) DeleteBucketPolicy(bucket string) error { +func (a GatewayUnsupported) DeleteBucketPolicy(ctx context.Context, bucket string) error { return errors.Trace(NotImplemented{}) } // HealFormat - Not implemented stub -func (a GatewayUnsupported) HealFormat(dryRun bool) (madmin.HealResultItem, error) { +func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) { return madmin.HealResultItem{}, errors.Trace(NotImplemented{}) } // HealBucket - Not implemented stub -func (a GatewayUnsupported) HealBucket(bucket string, dryRun bool) ([]madmin.HealResultItem, error) { +func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) { return nil, errors.Trace(NotImplemented{}) } // ListBucketsHeal - Not implemented stub -func (a GatewayUnsupported) ListBucketsHeal() (buckets []BucketInfo, err error) { +func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error) { return nil, errors.Trace(NotImplemented{}) } // HealObject - Not implemented stub -func (a GatewayUnsupported) HealObject(bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) { +func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) { return h, errors.Trace(NotImplemented{}) } // ListObjectsV2 - Not implemented stub -func (a GatewayUnsupported) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) { +func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) { return result, errors.Trace(NotImplemented{}) } // ListObjectsHeal - Not implemented stub -func (a GatewayUnsupported) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { +func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) { return loi, errors.Trace(NotImplemented{}) } // CopyObject copies a blob from source container to destination container. -func (a GatewayUnsupported) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string, +func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) { return objInfo, errors.Trace(NotImplemented{}) } @@ -117,17 +118,17 @@ func (a GatewayUnsupported) CopyObject(srcBucket string, srcObject string, destB // Locking operations // ListLocks lists namespace locks held in object layer -func (a GatewayUnsupported) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) { +func (a GatewayUnsupported) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) { return []VolumeLockInfo{}, errors.Trace(NotImplemented{}) } // ClearLocks clears namespace locks held in object layer -func (a GatewayUnsupported) ClearLocks([]VolumeLockInfo) error { +func (a GatewayUnsupported) ClearLocks(ctx context.Context, info []VolumeLockInfo) error { return errors.Trace(NotImplemented{}) } // RefreshBucketPolicy refreshes cache policy with what's on disk. -func (a GatewayUnsupported) RefreshBucketPolicy(bucket string) error { +func (a GatewayUnsupported) RefreshBucketPolicy(ctx context.Context, bucket string) error { return errors.Trace(NotImplemented{}) } diff --git a/cmd/gateway/nas/gateway-nas.go b/cmd/gateway/nas/gateway-nas.go index 7c16796c1..09640811b 100644 --- a/cmd/gateway/nas/gateway-nas.go +++ b/cmd/gateway/nas/gateway-nas.go @@ -17,6 +17,8 @@ package nas import ( + "context" + "github.com/minio/cli" "github.com/minio/minio-go/pkg/policy" minio "github.com/minio/minio/cmd" @@ -71,8 +73,8 @@ EXAMPLES: func nasGatewayMain(ctx *cli.Context) { // Validate gateway arguments. host := ctx.Args().First() - if host == "" { - cli.ShowCommandHelpAndExit(ctx, "nas", 1) + if host == "help" { + cli.ShowCommandHelpAndExit(ctx, nasBackend, 1) } // Validate gateway arguments. minio.StartGateway(ctx, &NAS{host}) @@ -114,6 +116,6 @@ func (l *nasObjects) IsNotificationSupported() bool { } // GetBucketPolicy will get policy on bucket -func (l *nasObjects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { +func (l *nasObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) { return minio.ReadBucketPolicy(bucket, l) } diff --git a/cmd/gateway/oss/gateway-oss.go b/cmd/gateway/oss/gateway-oss.go index cf0f36fb8..f5157a63b 100644 --- a/cmd/gateway/oss/gateway-oss.go +++ b/cmd/gateway/oss/gateway-oss.go @@ -17,6 +17,7 @@ package oss import ( + "context" "encoding/xml" "fmt" "io" @@ -323,12 +324,12 @@ type ossObjects struct { // Shutdown saves any gateway metadata to disk // if necessary and reload upon next restart. -func (l *ossObjects) Shutdown() error { +func (l *ossObjects) Shutdown(ctx context.Context) error { return nil } // StorageInfo is not relevant to OSS backend. -func (l *ossObjects) StorageInfo() (si minio.StorageInfo) { +func (l *ossObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { return } @@ -345,7 +346,7 @@ func ossIsValidBucketName(bucket string) bool { } // MakeBucketWithLocation creates a new container on OSS backend. -func (l *ossObjects) MakeBucketWithLocation(bucket, location string) error { +func (l *ossObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { if !ossIsValidBucketName(bucket) { return errors.Trace(minio.BucketNameInvalid{Bucket: bucket}) } @@ -372,12 +373,12 @@ func ossGeBucketInfo(client *oss.Client, bucket string) (bi minio.BucketInfo, er } // GetBucketInfo gets bucket metadata. -func (l *ossObjects) GetBucketInfo(bucket string) (bi minio.BucketInfo, err error) { +func (l *ossObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) { return ossGeBucketInfo(l.Client, bucket) } // ListBuckets lists all OSS buckets. -func (l *ossObjects) ListBuckets() (buckets []minio.BucketInfo, err error) { +func (l *ossObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) { marker := oss.Marker("") for { lbr, err := l.Client.ListBuckets(marker) @@ -402,7 +403,7 @@ func (l *ossObjects) ListBuckets() (buckets []minio.BucketInfo, err error) { } // DeleteBucket deletes a bucket on OSS. -func (l *ossObjects) DeleteBucket(bucket string) error { +func (l *ossObjects) DeleteBucket(ctx context.Context, bucket string) error { err := l.Client.DeleteBucket(bucket) if err != nil { return ossToObjectError(errors.Trace(err), bucket) @@ -483,12 +484,12 @@ func ossListObjectsV2(client *oss.Client, bucket, prefix, continuationToken, del } // ListObjects lists all blobs in OSS bucket filtered by prefix. -func (l *ossObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { +func (l *ossObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { return ossListObjects(l.Client, bucket, prefix, marker, delimiter, maxKeys) } // ListObjectsV2 lists all blobs in OSS bucket filtered by prefix -func (l *ossObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, +func (l *ossObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) { return ossListObjectsV2(l.Client, bucket, prefix, continuationToken, delimiter, maxKeys, fetchOwner, startAfter) } @@ -532,7 +533,7 @@ func ossGetObject(client *oss.Client, bucket, key string, startOffset, length in // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *ossObjects) GetObject(bucket, key string, startOffset, length int64, writer io.Writer, etag string) error { +func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error { return ossGetObject(l.Client, bucket, key, startOffset, length, writer, etag) } @@ -580,7 +581,7 @@ func ossGetObjectInfo(client *oss.Client, bucket, object string) (objInfo minio. } // GetObjectInfo reads object info and replies back ObjectInfo. -func (l *ossObjects) GetObjectInfo(bucket, object string) (objInfo minio.ObjectInfo, err error) { +func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) { return ossGetObjectInfo(l.Client, bucket, object) } @@ -606,12 +607,12 @@ func ossPutObject(client *oss.Client, bucket, object string, data *hash.Reader, } // PutObject creates a new object with the incoming data. -func (l *ossObjects) PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { +func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { return ossPutObject(l.Client, bucket, object, data, metadata) } // CopyObject copies an object from source bucket to a destination bucket. -func (l *ossObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { +func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { bkt, err := l.Client.Bucket(srcBucket) if err != nil { return objInfo, ossToObjectError(errors.Trace(err), srcBucket, srcObject) @@ -633,11 +634,11 @@ func (l *ossObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject strin if _, err = bkt.CopyObjectTo(dstBucket, dstObject, srcObject, opts...); err != nil { return objInfo, ossToObjectError(errors.Trace(err), srcBucket, srcObject) } - return l.GetObjectInfo(dstBucket, dstObject) + return l.GetObjectInfo(ctx, dstBucket, dstObject) } // DeleteObject deletes a blob in bucket. -func (l *ossObjects) DeleteObject(bucket, object string) error { +func (l *ossObjects) DeleteObject(ctx context.Context, bucket, object string) error { bkt, err := l.Client.Bucket(bucket) if err != nil { return ossToObjectError(errors.Trace(err), bucket, object) @@ -679,7 +680,7 @@ func fromOSSClientListMultipartsInfo(lmur oss.ListMultipartUploadResult) minio.L } // ListMultipartUploads lists all multipart uploads. -func (l *ossObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) { +func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) { bkt, err := l.Client.Bucket(bucket) if err != nil { return lmi, ossToObjectError(errors.Trace(err), bucket) @@ -695,7 +696,7 @@ func (l *ossObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMar } // NewMultipartUpload upload object in multiple parts. -func (l *ossObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) { +func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) { bkt, err := l.Client.Bucket(bucket) if err != nil { return uploadID, ossToObjectError(errors.Trace(err), bucket, object) @@ -716,7 +717,7 @@ func (l *ossObjects) NewMultipartUpload(bucket, object string, metadata map[stri } // PutObjectPart puts a part of object in bucket. -func (l *ossObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, err error) { +func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, err error) { bkt, err := l.Client.Bucket(bucket) if err != nil { return pi, ossToObjectError(errors.Trace(err), bucket, object) @@ -796,7 +797,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par // CopyObjectPart creates a part in a multipart upload by copying // existing object or a part of it. -func (l *ossObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject, uploadID string, +func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo minio.ObjectInfo) (p minio.PartInfo, err error) { bkt, err := l.Client.Bucket(destBucket) @@ -825,7 +826,7 @@ func (l *ossObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject } // ListObjectParts returns all object parts for specified object in specified bucket -func (l *ossObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi minio.ListPartsInfo, err error) { +func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi minio.ListPartsInfo, err error) { lupr, err := ossListObjectParts(l.Client, bucket, object, uploadID, partNumberMarker, maxParts) if err != nil { return lpi, ossToObjectError(errors.Trace(err), bucket, object, uploadID) @@ -835,7 +836,7 @@ func (l *ossObjects) ListObjectParts(bucket, object, uploadID string, partNumber } // AbortMultipartUpload aborts a ongoing multipart upload. -func (l *ossObjects) AbortMultipartUpload(bucket, object, uploadID string) error { +func (l *ossObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error { bkt, err := l.Client.Bucket(bucket) if err != nil { return ossToObjectError(errors.Trace(err), bucket, object) @@ -853,7 +854,7 @@ func (l *ossObjects) AbortMultipartUpload(bucket, object, uploadID string) error } // CompleteMultipartUpload completes ongoing multipart upload and finalizes object. -func (l *ossObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) { +func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) { client := l.Client bkt, err := client.Bucket(bucket) if err != nil { @@ -910,7 +911,7 @@ func (l *ossObjects) CompleteMultipartUpload(bucket, object, uploadID string, up return oi, ossToObjectError(errors.Trace(err), bucket, object) } - return l.GetObjectInfo(bucket, object) + return l.GetObjectInfo(ctx, bucket, object) } // SetBucketPolicy sets policy on bucket. @@ -918,7 +919,7 @@ func (l *ossObjects) CompleteMultipartUpload(bucket, object, uploadID string, up // oss.ACLPublicReadWrite: readwrite in minio terminology // oss.ACLPublicRead: readonly in minio terminology // oss.ACLPrivate: none in minio terminology -func (l *ossObjects) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error { +func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error { bucketPolicies := policy.GetPolicies(policyInfo.Statements, bucket, "") if len(bucketPolicies) != 1 { return errors.Trace(minio.NotImplemented{}) @@ -952,7 +953,7 @@ func (l *ossObjects) SetBucketPolicy(bucket string, policyInfo policy.BucketAcce } // GetBucketPolicy will get policy on bucket. -func (l *ossObjects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { +func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) { result, err := l.Client.GetBucketACL(bucket) if err != nil { return policy.BucketAccessPolicy{}, ossToObjectError(errors.Trace(err)) @@ -975,7 +976,7 @@ func (l *ossObjects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, } // DeleteBucketPolicy deletes all policies on bucket. -func (l *ossObjects) DeleteBucketPolicy(bucket string) error { +func (l *ossObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error { err := l.Client.SetBucketACL(bucket, oss.ACLPrivate) if err != nil { return ossToObjectError(errors.Trace(err), bucket) diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index 83dcee4b1..4866874b6 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -17,6 +17,7 @@ package s3 import ( + "context" "io" "github.com/minio/cli" @@ -143,17 +144,17 @@ type s3Objects struct { // Shutdown saves any gateway metadata to disk // if necessary and reload upon next restart. -func (l *s3Objects) Shutdown() error { +func (l *s3Objects) Shutdown(ctx context.Context) error { return nil } // StorageInfo is not relevant to S3 backend. -func (l *s3Objects) StorageInfo() (si minio.StorageInfo) { +func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) { return si } // MakeBucket creates a new container on S3 backend. -func (l *s3Objects) MakeBucketWithLocation(bucket, location string) error { +func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { err := l.Client.MakeBucket(bucket, location) if err != nil { return minio.ErrorRespToObjectError(errors.Trace(err), bucket) @@ -162,7 +163,7 @@ func (l *s3Objects) MakeBucketWithLocation(bucket, location string) error { } // GetBucketInfo gets bucket metadata.. -func (l *s3Objects) GetBucketInfo(bucket string) (bi minio.BucketInfo, e error) { +func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) { // Verify if bucket name is valid. // We are using a separate helper function here to validate bucket // names instead of IsValidBucketName() because there is a possibility @@ -194,7 +195,7 @@ func (l *s3Objects) GetBucketInfo(bucket string) (bi minio.BucketInfo, e error) } // ListBuckets lists all S3 buckets -func (l *s3Objects) ListBuckets() ([]minio.BucketInfo, error) { +func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) { buckets, err := l.Client.ListBuckets() if err != nil { return nil, minio.ErrorRespToObjectError(errors.Trace(err)) @@ -212,7 +213,7 @@ func (l *s3Objects) ListBuckets() ([]minio.BucketInfo, error) { } // DeleteBucket deletes a bucket on S3 -func (l *s3Objects) DeleteBucket(bucket string) error { +func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string) error { err := l.Client.RemoveBucket(bucket) if err != nil { return minio.ErrorRespToObjectError(errors.Trace(err), bucket) @@ -221,7 +222,7 @@ func (l *s3Objects) DeleteBucket(bucket string) error { } // ListObjects lists all blobs in S3 bucket filtered by prefix -func (l *s3Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, e error) { +func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, e error) { result, err := l.Client.ListObjects(bucket, prefix, marker, delimiter, maxKeys) if err != nil { return loi, minio.ErrorRespToObjectError(errors.Trace(err), bucket) @@ -231,7 +232,7 @@ func (l *s3Objects) ListObjects(bucket string, prefix string, marker string, del } // ListObjectsV2 lists all blobs in S3 bucket filtered by prefix -func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) { +func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) { result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys) if err != nil { return loi, minio.ErrorRespToObjectError(errors.Trace(err), bucket) @@ -246,7 +247,7 @@ func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter s // // startOffset indicates the starting read location of the object. // length indicates the total length of the object. -func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error { +func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error { if length < 0 && length != -1 { return minio.ErrorRespToObjectError(errors.Trace(minio.InvalidRange{}), bucket, key) } @@ -270,7 +271,7 @@ func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, leng } // GetObjectInfo reads object info and replies back ObjectInfo -func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) { +func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) { oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{}) if err != nil { return minio.ObjectInfo{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) @@ -280,7 +281,7 @@ func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo minio.O } // PutObject creates a new object with the incoming data, -func (l *s3Objects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { +func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata)) if err != nil { return objInfo, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) @@ -290,7 +291,7 @@ func (l *s3Objects) PutObject(bucket string, object string, data *hash.Reader, m } // CopyObject copies an object from source bucket to a destination bucket. -func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { +func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { // Set this header such that following CopyObject() always sets the right metadata on the destination. // metadata input is already a trickled down value from interpreting x-amz-metadata-directive at // handler layer. So what we have right now is supposed to be applied on the destination object anyways. @@ -300,11 +301,11 @@ func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket str if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil { return objInfo, minio.ErrorRespToObjectError(errors.Trace(err), srcBucket, srcObject) } - return l.GetObjectInfo(dstBucket, dstObject) + return l.GetObjectInfo(ctx, dstBucket, dstObject) } // DeleteObject deletes a blob in bucket -func (l *s3Objects) DeleteObject(bucket string, object string) error { +func (l *s3Objects) DeleteObject(ctx context.Context, bucket string, object string) error { err := l.Client.RemoveObject(bucket, object) if err != nil { return minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) @@ -314,7 +315,7 @@ func (l *s3Objects) DeleteObject(bucket string, object string) error { } // ListMultipartUploads lists all multipart uploads. -func (l *s3Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, e error) { +func (l *s3Objects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, e error) { result, err := l.Client.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) if err != nil { return lmi, err @@ -324,7 +325,7 @@ func (l *s3Objects) ListMultipartUploads(bucket string, prefix string, keyMarker } // NewMultipartUpload upload object in multiple parts -func (l *s3Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) { +func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) { // Create PutObject options opts := miniogo.PutObjectOptions{UserMetadata: metadata} uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts) @@ -335,7 +336,7 @@ func (l *s3Objects) NewMultipartUpload(bucket string, object string, metadata ma } // PutObjectPart puts a part of object in bucket -func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, e error) { +func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, e error) { info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString()) if err != nil { return pi, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) @@ -346,7 +347,7 @@ func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, // CopyObjectPart creates a part in a multipart upload by copying // existing object or a part of it. -func (l *s3Objects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject, uploadID string, +func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo minio.ObjectInfo) (p minio.PartInfo, err error) { srcInfo.UserDefined = map[string]string{ @@ -363,7 +364,7 @@ func (l *s3Objects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject, } // ListObjectParts returns all object parts for specified object in specified bucket -func (l *s3Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, e error) { +func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, e error) { result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts) if err != nil { return lpi, err @@ -373,23 +374,23 @@ func (l *s3Objects) ListObjectParts(bucket string, object string, uploadID strin } // AbortMultipartUpload aborts a ongoing multipart upload -func (l *s3Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error { +func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error { err := l.Client.AbortMultipartUpload(bucket, object, uploadID) return minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) } // CompleteMultipartUpload completes ongoing multipart upload and finalizes object -func (l *s3Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, e error) { +func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, e error) { err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts)) if err != nil { return oi, minio.ErrorRespToObjectError(errors.Trace(err), bucket, object) } - return l.GetObjectInfo(bucket, object) + return l.GetObjectInfo(ctx, bucket, object) } // SetBucketPolicy sets policy on bucket -func (l *s3Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error { +func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error { if err := l.Client.PutBucketPolicy(bucket, policyInfo); err != nil { return minio.ErrorRespToObjectError(errors.Trace(err), bucket, "") } @@ -398,7 +399,7 @@ func (l *s3Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAcces } // GetBucketPolicy will get policy on bucket -func (l *s3Objects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { +func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) { policyInfo, err := l.Client.GetBucketPolicy(bucket) if err != nil { return policy.BucketAccessPolicy{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket, "") @@ -407,7 +408,7 @@ func (l *s3Objects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, e } // DeleteBucketPolicy deletes all policies on bucket -func (l *s3Objects) DeleteBucketPolicy(bucket string) error { +func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error { if err := l.Client.PutBucketPolicy(bucket, policy.BucketAccessPolicy{}); err != nil { return minio.ErrorRespToObjectError(errors.Trace(err), bucket, "") } diff --git a/cmd/logger/log.go b/cmd/logger/log.go index b74af3958..80fe2ba15 100644 --- a/cmd/logger/log.go +++ b/cmd/logger/log.go @@ -19,7 +19,9 @@ package logger import "context" // Key used for ContextSet/Get -const contextKey = "reqInfo" +type contextKeyType string + +const contextLogKey = contextKeyType("miniolog") // KeyVal - appended to ReqInfo.Tags type KeyVal struct { @@ -48,12 +50,12 @@ func (r *ReqInfo) AppendTags(key string, val string) { // ContextSet sets ReqInfo in the context. func ContextSet(ctx context.Context, req *ReqInfo) context.Context { - return context.WithValue(ctx, contextKey, req) + return context.WithValue(ctx, contextLogKey, req) } // ContextGet returns ReqInfo if set. func ContextGet(ctx context.Context) *ReqInfo { - r, ok := ctx.Value(contextKey).(*ReqInfo) + r, ok := ctx.Value(contextLogKey).(*ReqInfo) if ok { return r } diff --git a/cmd/utils.go b/cmd/utils.go index 18d9d319d..52eda0a3f 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -34,9 +34,10 @@ import ( "strings" "time" + "github.com/minio/minio/cmd/logger" + humanize "github.com/dustin/go-humanize" "github.com/gorilla/mux" - "github.com/minio/minio/cmd/logger" "github.com/pkg/profile" ) @@ -318,6 +319,7 @@ func ceilFrac(numerator, denominator int64) (ceil int64) { return } +// Returns context with ReqInfo details set in the context. func newContext(r *http.Request, api string) context.Context { vars := mux.Vars(r) bucket := vars["bucket"]