oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add context support for gateway b2, manta, gcs

Browse Source
master
kannappanr 7 years ago committed by kannappanr
parent e452377b24
commit bdb1a90720
3 changed files with 88 additions and 99 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 92
      cmd/gateway/b2/gateway-b2.go
  2. 52
      cmd/gateway/gcs/gateway-gcs.go
  3. 43
      cmd/gateway/manta/gateway-manta.go

92
cmd/gateway/b2/gateway-b2.go
Unescape View File

@ -199,18 +199,18 @@ func b2ToObjectError(err error, params ...string) error {
// Shutdown saves any gateway metadata to disk // Shutdown saves any gateway metadata to disk
// if necessary and reload upon next restart. // if necessary and reload upon next restart.
func (l *b2Objects) Shutdown() error { func (l *b2Objects) Shutdown(ctx context.Context) error {
// TODO // TODO
return nil return nil
} }
// StorageInfo is not relevant to B2 backend. // StorageInfo is not relevant to B2 backend.
func (l *b2Objects) StorageInfo() (si minio.StorageInfo) { func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
return si return si
} }
// MakeBucket creates a new container on B2 backend. // MakeBucket creates a new container on B2 backend.
func (l *b2Objects) MakeBucketWithLocation(bucket, location string) error { func (l *b2Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
// location is ignored for B2 backend. // location is ignored for B2 backend.
// All buckets are set to private by default. // All buckets are set to private by default.
@ -218,7 +218,7 @@ func (l *b2Objects) MakeBucketWithLocation(bucket, location string) error {
return b2ToObjectError(errors.Trace(err), bucket) return b2ToObjectError(errors.Trace(err), bucket)
} }
func (l *b2Objects) reAuthorizeAccount() error { func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error {
client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport())) client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
if err != nil { if err != nil {
return err return err
@ -233,18 +233,18 @@ func (l *b2Objects) reAuthorizeAccount() error {
// the account and updates the B2 client safely. Once successfully // the account and updates the B2 client safely. Once successfully
// authorized performs the call again and returns list of buckets. // authorized performs the call again and returns list of buckets.
// For any errors which are not actionable we return an error. // For any errors which are not actionable we return an error.
func (l *b2Objects) listBuckets(err error) ([]*b2.Bucket, error) { func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, error) {
if err != nil { if err != nil {
if b2.Action(err) != b2.ReAuthenticate { if b2.Action(err) != b2.ReAuthenticate {
return nil, err return nil, err
} }
if rerr := l.reAuthorizeAccount(); rerr != nil { if rerr := l.reAuthorizeAccount(ctx); rerr != nil {
return nil, rerr return nil, rerr
} }
} }
bktList, lerr := l.b2Client.ListBuckets(l.ctx) bktList, lerr := l.b2Client.ListBuckets(l.ctx)
if lerr != nil { if lerr != nil {
return l.listBuckets(lerr) return l.listBuckets(ctx, lerr)
} }
return bktList, nil return bktList, nil
} }
@ -253,8 +253,8 @@ func (l *b2Objects) listBuckets(err error) ([]*b2.Bucket, error) {
// for performing an API operation. B2 API doesn't // for performing an API operation. B2 API doesn't
// provide a direct way to access the bucket so we need // provide a direct way to access the bucket so we need
// to employ following technique. // to employ following technique.
func (l *b2Objects) Bucket(bucket string) (*b2.Bucket, error) { func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, error) {
bktList, err := l.listBuckets(nil) bktList, err := l.listBuckets(ctx, nil)
if err != nil { if err != nil {
return nil, b2ToObjectError(errors.Trace(err), bucket) return nil, b2ToObjectError(errors.Trace(err), bucket)
} }
@ -267,8 +267,8 @@ func (l *b2Objects) Bucket(bucket string) (*b2.Bucket, error) {
} }
// GetBucketInfo gets bucket metadata.. // GetBucketInfo gets bucket metadata..
func (l *b2Objects) GetBucketInfo(bucket string) (bi minio.BucketInfo, err error) { func (l *b2Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) {
if _, err = l.Bucket(bucket); err != nil { if _, err = l.Bucket(ctx, bucket); err != nil {
return bi, err return bi, err
} }
return minio.BucketInfo{ return minio.BucketInfo{
@ -278,8 +278,8 @@ func (l *b2Objects) GetBucketInfo(bucket string) (bi minio.BucketInfo, err error
} }
// ListBuckets lists all B2 buckets // ListBuckets lists all B2 buckets
func (l *b2Objects) ListBuckets() ([]minio.BucketInfo, error) { func (l *b2Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) {
bktList, err := l.listBuckets(nil) bktList, err := l.listBuckets(ctx, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -294,8 +294,8 @@ func (l *b2Objects) ListBuckets() ([]minio.BucketInfo, error) {
} }
// DeleteBucket deletes a bucket on B2 // DeleteBucket deletes a bucket on B2
func (l *b2Objects) DeleteBucket(bucket string) error { func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -304,8 +304,8 @@ func (l *b2Objects) DeleteBucket(bucket string) error {
} }
// ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time. // ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time.
func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return loi, err return loi, err
} }
@ -335,10 +335,10 @@ func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, del
} }
// ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time. // ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) { fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
// fetchOwner, startAfter are not supported and unused. // fetchOwner, startAfter are not supported and unused.
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return loi, err return loi, err
} }
@ -374,8 +374,8 @@ func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter s
// //
// startOffset indicates the starting read location of the object. // startOffset indicates the starting read location of the object.
// length indicates the total length of the object. // length indicates the total length of the object.
func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error { func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -389,8 +389,8 @@ func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, l
} }
// GetObjectInfo reads object info and replies back ObjectInfo // GetObjectInfo reads object info and replies back ObjectInfo
func (l *b2Objects) GetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) { func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return objInfo, err return objInfo, err
} }
@ -478,8 +478,8 @@ func (nb *Reader) Read(p []byte) (int, error) {
} }
// PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB. // PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return objInfo, err return objInfo, err
} }
@ -517,8 +517,8 @@ func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, met
} }
// DeleteObject deletes a blob in bucket // DeleteObject deletes a blob in bucket
func (l *b2Objects) DeleteObject(bucket string, object string) error { func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object string) error {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -533,11 +533,11 @@ func (l *b2Objects) DeleteObject(bucket string, object string) error {
} }
// ListMultipartUploads lists all multipart uploads. // ListMultipartUploads lists all multipart uploads.
func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string,
delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) { delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
// keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any // keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any
// of these parameters only equivalent parameter is uploadIDMarker. // of these parameters only equivalent parameter is uploadIDMarker.
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return lmi, err return lmi, err
} }
@ -572,9 +572,9 @@ func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker
// Each large file must consist of at least 2 parts, and all of the parts except the // Each large file must consist of at least 2 parts, and all of the parts except the
// last one must be at least 5MB in size. The last part must contain at least one byte. // last one must be at least 5MB in size. The last part must contain at least one byte.
// For more information - https://www.backblaze.com/b2/docs/large_files.html // For more information - https://www.backblaze.com/b2/docs/large_files.html
func (l *b2Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (string, error) { func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (string, error) {
var uploadID string var uploadID string
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return uploadID, err return uploadID, err
} }
@ -590,8 +590,8 @@ func (l *b2Objects) NewMultipartUpload(bucket string, object string, metadata ma
} }
// PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API. // PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi minio.PartInfo, err error) { func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi minio.PartInfo, err error) {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return pi, err return pi, err
} }
@ -616,8 +616,8 @@ func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string,
} }
// ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API. // ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API.
func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, err error) { func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, err error) {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return lpi, err return lpi, err
} }
@ -649,8 +649,8 @@ func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID strin
} }
// AbortMultipartUpload aborts a on going multipart upload, uses B2's LargeFile upload API. // AbortMultipartUpload aborts a on going multipart upload, uses B2's LargeFile upload API.
func (l *b2Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error { func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -659,8 +659,8 @@ func (l *b2Objects) AbortMultipartUpload(bucket string, object string, uploadID
} }
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API. // CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) { func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return oi, err return oi, err
} }
@ -680,14 +680,14 @@ func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, upload
return oi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID) return oi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
} }
return l.GetObjectInfo(bucket, object) return l.GetObjectInfo(ctx, bucket, object)
} }
// SetBucketPolicy - B2 supports 2 types of bucket policies: // SetBucketPolicy - B2 supports 2 types of bucket policies:
// bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket; // bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
// bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them. // bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
// Default is AllPrivate for all buckets. // Default is AllPrivate for all buckets.
func (l *b2Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error { func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
var policies []minio.BucketAccessPolicy var policies []minio.BucketAccessPolicy
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") { for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") {
@ -706,7 +706,7 @@ func (l *b2Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAcces
if policies[0].Policy != policy.BucketPolicyReadOnly { if policies[0].Policy != policy.BucketPolicyReadOnly {
return errors.Trace(minio.NotImplemented{}) return errors.Trace(minio.NotImplemented{})
} }
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }
@ -717,9 +717,9 @@ func (l *b2Objects) SetBucketPolicy(bucket string, policyInfo policy.BucketAcces
// GetBucketPolicy, returns the current bucketType from B2 backend and convert // GetBucketPolicy, returns the current bucketType from B2 backend and convert
// it into S3 compatible bucket policy info. // it into S3 compatible bucket policy info.
func (l *b2Objects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"} policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return policyInfo, err return policyInfo, err
} }
@ -734,8 +734,8 @@ func (l *b2Objects) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, e
} }
// DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'. // DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'.
func (l *b2Objects) DeleteBucketPolicy(bucket string) error { func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
bkt, err := l.Bucket(bucket) bkt, err := l.Bucket(ctx, bucket)
if err != nil { if err != nil {
return err return err
} }

52
cmd/gateway/gcs/gateway-gcs.go
Unescape View File

@ -401,17 +401,17 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp() {
// Shutdown - save any gateway metadata to disk // Shutdown - save any gateway metadata to disk
// if necessary and reload upon next restart. // if necessary and reload upon next restart.
func (l *gcsGateway) Shutdown() error { func (l *gcsGateway) Shutdown(ctx context.Context) error {
return nil return nil
} }
// StorageInfo - Not relevant to GCS backend. // StorageInfo - Not relevant to GCS backend.
func (l *gcsGateway) StorageInfo() minio.StorageInfo { func (l *gcsGateway) StorageInfo(ctx context.Context) minio.StorageInfo {
return minio.StorageInfo{} return minio.StorageInfo{}
} }
// MakeBucketWithLocation - Create a new container on GCS backend. // MakeBucketWithLocation - Create a new container on GCS backend.
func (l *gcsGateway) MakeBucketWithLocation(bucket, location string) error { func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
bkt := l.client.Bucket(bucket) bkt := l.client.Bucket(bucket)
// we'll default to the us multi-region in case of us-east-1 // we'll default to the us multi-region in case of us-east-1
@ -427,7 +427,7 @@ func (l *gcsGateway) MakeBucketWithLocation(bucket, location string) error {
} }
// GetBucketInfo - Get bucket metadata.. // GetBucketInfo - Get bucket metadata..
func (l *gcsGateway) GetBucketInfo(bucket string) (minio.BucketInfo, error) { func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.BucketInfo, error) {
attrs, err := l.client.Bucket(bucket).Attrs(l.ctx) attrs, err := l.client.Bucket(bucket).Attrs(l.ctx)
if err != nil { if err != nil {
return minio.BucketInfo{}, gcsToObjectError(errors.Trace(err), bucket) return minio.BucketInfo{}, gcsToObjectError(errors.Trace(err), bucket)
@ -440,7 +440,7 @@ func (l *gcsGateway) GetBucketInfo(bucket string) (minio.BucketInfo, error) {
} }
// ListBuckets lists all buckets under your project-id on GCS. // ListBuckets lists all buckets under your project-id on GCS.
func (l *gcsGateway) ListBuckets() (buckets []minio.BucketInfo, err error) { func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
it := l.client.Buckets(l.ctx, l.projectID) it := l.client.Buckets(l.ctx, l.projectID)
// Iterate and capture all the buckets. // Iterate and capture all the buckets.
@ -464,7 +464,7 @@ func (l *gcsGateway) ListBuckets() (buckets []minio.BucketInfo, err error) {
} }
// DeleteBucket delete a bucket on GCS. // DeleteBucket delete a bucket on GCS.
func (l *gcsGateway) DeleteBucket(bucket string) error { func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
itObject := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{ itObject := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
Delimiter: "/", Delimiter: "/",
Versions: false, Versions: false,
@ -537,7 +537,7 @@ func isGCSMarker(marker string) bool {
} }
// ListObjects - lists all blobs in GCS bucket filtered by prefix // ListObjects - lists all blobs in GCS bucket filtered by prefix
func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) { func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) {
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{ it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
Delimiter: delimiter, Delimiter: delimiter,
Prefix: prefix, Prefix: prefix,
@ -640,7 +640,7 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de
} }
// ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix // ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
func (l *gcsGateway) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) { func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{ it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
Delimiter: delimiter, Delimiter: delimiter,
Prefix: prefix, Prefix: prefix,
@ -713,7 +713,7 @@ func (l *gcsGateway) ListObjectsV2(bucket, prefix, continuationToken, delimiter
// //
// startOffset indicates the starting read location of the object. // startOffset indicates the starting read location of the object.
// length indicates the total length of the object. // length indicates the total length of the object.
func (l *gcsGateway) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error { func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
// otherwise gcs will just return object not exist in case of non-existing bucket // otherwise gcs will just return object not exist in case of non-existing bucket
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@ -751,7 +751,7 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
} }
// GetObjectInfo - reads object info and replies back ObjectInfo // GetObjectInfo - reads object info and replies back ObjectInfo
func (l *gcsGateway) GetObjectInfo(bucket string, object string) (minio.ObjectInfo, error) { func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) {
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
// otherwise gcs will just return object not exist in case of non-existing bucket // otherwise gcs will just return object not exist in case of non-existing bucket
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@ -767,7 +767,7 @@ func (l *gcsGateway) GetObjectInfo(bucket string, object string) (minio.ObjectIn
} }
// PutObject - Create a new object with the incoming data, // PutObject - Create a new object with the incoming data,
func (l *gcsGateway) PutObject(bucket string, key string, data *hash.Reader, metadata map[string]string) (minio.ObjectInfo, error) { func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string) (minio.ObjectInfo, error) {
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first, // if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
// otherwise gcs will just return object not exist in case of non-existing bucket // otherwise gcs will just return object not exist in case of non-existing bucket
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil { if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@ -800,7 +800,7 @@ func (l *gcsGateway) PutObject(bucket string, key string, data *hash.Reader, met
} }
// CopyObject - Copies a blob from source container to destination container. // CopyObject - Copies a blob from source container to destination container.
func (l *gcsGateway) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string, func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
srcInfo minio.ObjectInfo) (minio.ObjectInfo, error) { srcInfo minio.ObjectInfo) (minio.ObjectInfo, error) {
src := l.client.Bucket(srcBucket).Object(srcObject) src := l.client.Bucket(srcBucket).Object(srcObject)
@ -818,7 +818,7 @@ func (l *gcsGateway) CopyObject(srcBucket string, srcObject string, destBucket s
} }
// DeleteObject - Deletes a blob in bucket // DeleteObject - Deletes a blob in bucket
func (l *gcsGateway) DeleteObject(bucket string, object string) error { func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error {
err := l.client.Bucket(bucket).Object(object).Delete(l.ctx) err := l.client.Bucket(bucket).Object(object).Delete(l.ctx)
if err != nil { if err != nil {
return gcsToObjectError(errors.Trace(err), bucket, object) return gcsToObjectError(errors.Trace(err), bucket, object)
@ -828,7 +828,7 @@ func (l *gcsGateway) DeleteObject(bucket string, object string) error {
} }
// NewMultipartUpload - upload object in multiple parts // NewMultipartUpload - upload object in multiple parts
func (l *gcsGateway) NewMultipartUpload(bucket string, key string, metadata map[string]string) (uploadID string, err error) { func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string) (uploadID string, err error) {
// generate new uploadid // generate new uploadid
uploadID = minio.MustGetUUID() uploadID = minio.MustGetUUID()
@ -853,7 +853,7 @@ func (l *gcsGateway) NewMultipartUpload(bucket string, key string, metadata map[
} }
// ListMultipartUploads - lists all multipart uploads. // ListMultipartUploads - lists all multipart uploads.
func (l *gcsGateway) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (minio.ListMultipartsInfo, error) { func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (minio.ListMultipartsInfo, error) {
return minio.ListMultipartsInfo{ return minio.ListMultipartsInfo{
KeyMarker: keyMarker, KeyMarker: keyMarker,
UploadIDMarker: uploadIDMarker, UploadIDMarker: uploadIDMarker,
@ -865,14 +865,14 @@ func (l *gcsGateway) ListMultipartUploads(bucket string, prefix string, keyMarke
// Checks if minio.sys.tmp/multipart/v1/<upload-id>/gcs.json exists, returns // Checks if minio.sys.tmp/multipart/v1/<upload-id>/gcs.json exists, returns
// an object layer compatible error upon any error. // an object layer compatible error upon any error.
func (l *gcsGateway) checkUploadIDExists(bucket string, key string, uploadID string) error { func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key string, uploadID string) error {
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx) _, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx)
return gcsToObjectError(errors.Trace(err), bucket, key, uploadID) return gcsToObjectError(errors.Trace(err), bucket, key, uploadID)
} }
// PutObjectPart puts a part of object in bucket // PutObjectPart puts a part of object in bucket
func (l *gcsGateway) PutObjectPart(bucket string, key string, uploadID string, partNumber int, data *hash.Reader) (minio.PartInfo, error) { func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader) (minio.PartInfo, error) {
if err := l.checkUploadIDExists(bucket, key, uploadID); err != nil { if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil {
return minio.PartInfo{}, err return minio.PartInfo{}, err
} }
etag := data.MD5HexString() etag := data.MD5HexString()
@ -902,8 +902,8 @@ func (l *gcsGateway) PutObjectPart(bucket string, key string, uploadID string, p
} }
// ListObjectParts returns all object parts for specified object in specified bucket // ListObjectParts returns all object parts for specified object in specified bucket
func (l *gcsGateway) ListObjectParts(bucket string, key string, uploadID string, partNumberMarker int, maxParts int) (minio.ListPartsInfo, error) { func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key string, uploadID string, partNumberMarker int, maxParts int) (minio.ListPartsInfo, error) {
return minio.ListPartsInfo{}, l.checkUploadIDExists(bucket, key, uploadID) return minio.ListPartsInfo{}, l.checkUploadIDExists(ctx, bucket, key, uploadID)
} }
// Called by AbortMultipartUpload and CompleteMultipartUpload for cleaning up. // Called by AbortMultipartUpload and CompleteMultipartUpload for cleaning up.
@ -931,8 +931,8 @@ func (l *gcsGateway) cleanupMultipartUpload(bucket, key, uploadID string) error
} }
// AbortMultipartUpload aborts a ongoing multipart upload // AbortMultipartUpload aborts a ongoing multipart upload
func (l *gcsGateway) AbortMultipartUpload(bucket string, key string, uploadID string) error { func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, key string, uploadID string) error {
if err := l.checkUploadIDExists(bucket, key, uploadID); err != nil { if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil {
return err return err
} }
return l.cleanupMultipartUpload(bucket, key, uploadID) return l.cleanupMultipartUpload(bucket, key, uploadID)
@ -946,7 +946,7 @@ func (l *gcsGateway) AbortMultipartUpload(bucket string, key string, uploadID st
// to the number of components you can compose per second. This rate counts both the // to the number of components you can compose per second. This rate counts both the
// components being appended to a composite object as well as the components being // components being appended to a composite object as well as the components being
// copied when the composite object of which they are a part is copied. // copied when the composite object of which they are a part is copied.
func (l *gcsGateway) CompleteMultipartUpload(bucket string, key string, uploadID string, uploadedParts []minio.CompletePart) (minio.ObjectInfo, error) { func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, uploadedParts []minio.CompletePart) (minio.ObjectInfo, error) {
meta := gcsMultipartMetaName(uploadID) meta := gcsMultipartMetaName(uploadID)
object := l.client.Bucket(bucket).Object(meta) object := l.client.Bucket(bucket).Object(meta)
@ -1046,7 +1046,7 @@ func (l *gcsGateway) CompleteMultipartUpload(bucket string, key string, uploadID
} }
// SetBucketPolicy - Set policy on bucket // SetBucketPolicy - Set policy on bucket
func (l *gcsGateway) SetBucketPolicy(bucket string, policyInfo policy.BucketAccessPolicy) error { func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
var policies []minio.BucketAccessPolicy var policies []minio.BucketAccessPolicy
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") { for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") {
@ -1091,7 +1091,7 @@ func (l *gcsGateway) SetBucketPolicy(bucket string, policyInfo policy.BucketAcce
} }
// GetBucketPolicy - Get policy on bucket // GetBucketPolicy - Get policy on bucket
func (l *gcsGateway) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
rules, err := l.client.Bucket(bucket).ACL().List(l.ctx) rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
if err != nil { if err != nil {
return policy.BucketAccessPolicy{}, gcsToObjectError(errors.Trace(err), bucket) return policy.BucketAccessPolicy{}, gcsToObjectError(errors.Trace(err), bucket)
@ -1116,7 +1116,7 @@ func (l *gcsGateway) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy,
} }
// DeleteBucketPolicy - Delete all policies on bucket // DeleteBucketPolicy - Delete all policies on bucket
func (l *gcsGateway) DeleteBucketPolicy(bucket string) error { func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error {
// This only removes the storage.AllUsers policies // This only removes the storage.AllUsers policies
if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil { if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
return gcsToObjectError(errors.Trace(err), bucket) return gcsToObjectError(errors.Trace(err), bucket)

43
cmd/gateway/manta/gateway-manta.go
Unescape View File

@ -220,12 +220,12 @@ type tritonObjects struct {
// Shutdown - save any gateway metadata to disk // Shutdown - save any gateway metadata to disk
// if necessary and reload upon next restart. // if necessary and reload upon next restart.
func (t *tritonObjects) Shutdown() error { func (t *tritonObjects) Shutdown(ctx context.Context) error {
return nil return nil
} }
// StorageInfo - Not relevant to Triton backend. // StorageInfo - Not relevant to Triton backend.
func (t *tritonObjects) StorageInfo() (si minio.StorageInfo) { func (t *tritonObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
return si return si
} }
@ -236,8 +236,7 @@ func (t *tritonObjects) StorageInfo() (si minio.StorageInfo) {
// MakeBucketWithLocation - Create a new directory within manta. // MakeBucketWithLocation - Create a new directory within manta.
// //
// https://apidocs.joyent.com/manta/api.html#PutDirectory // https://apidocs.joyent.com/manta/api.html#PutDirectory
func (t *tritonObjects) MakeBucketWithLocation(bucket, location string) error { func (t *tritonObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
ctx := context.Background()
err := t.client.Dir().Put(ctx, &storage.PutDirectoryInput{ err := t.client.Dir().Put(ctx, &storage.PutDirectoryInput{
DirectoryName: path.Join(mantaRoot, bucket), DirectoryName: path.Join(mantaRoot, bucket),
}) })
@ -250,9 +249,8 @@ func (t *tritonObjects) MakeBucketWithLocation(bucket, location string) error {
// GetBucketInfo - Get directory metadata.. // GetBucketInfo - Get directory metadata..
// //
// https://apidocs.joyent.com/manta/api.html#GetObject // https://apidocs.joyent.com/manta/api.html#GetObject
func (t *tritonObjects) GetBucketInfo(bucket string) (bi minio.BucketInfo, e error) { func (t *tritonObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) {
var info minio.BucketInfo var info minio.BucketInfo
ctx := context.Background()
resp, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{ resp, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{
ObjectPath: path.Join(mantaRoot, bucket), ObjectPath: path.Join(mantaRoot, bucket),
}) })
@ -270,8 +268,7 @@ func (t *tritonObjects) GetBucketInfo(bucket string) (bi minio.BucketInfo, e err
// ListDirectories. // ListDirectories.
// //
// https://apidocs.joyent.com/manta/api.html#ListDirectory // https://apidocs.joyent.com/manta/api.html#ListDirectory
func (t *tritonObjects) ListBuckets() (buckets []minio.BucketInfo, err error) { func (t *tritonObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
ctx := context.Background()
dirs, err := t.client.Dir().List(ctx, &storage.ListDirectoryInput{ dirs, err := t.client.Dir().List(ctx, &storage.ListDirectoryInput{
DirectoryName: path.Join(mantaRoot), DirectoryName: path.Join(mantaRoot),
}) })
@ -295,8 +292,7 @@ func (t *tritonObjects) ListBuckets() (buckets []minio.BucketInfo, err error) {
// DeleteDirectory. // DeleteDirectory.
// //
// https://apidocs.joyent.com/manta/api.html#DeleteDirectory // https://apidocs.joyent.com/manta/api.html#DeleteDirectory
func (t *tritonObjects) DeleteBucket(bucket string) error { func (t *tritonObjects) DeleteBucket(ctx context.Context, bucket string) error {
ctx := context.Background()
return t.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{ return t.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{
DirectoryName: path.Join(mantaRoot, bucket), DirectoryName: path.Join(mantaRoot, bucket),
}) })
@ -310,13 +306,12 @@ func (t *tritonObjects) DeleteBucket(bucket string) error {
// and marker, uses Manta equivalent ListDirectory. // and marker, uses Manta equivalent ListDirectory.
// //
// https://apidocs.joyent.com/manta/api.html#ListDirectory // https://apidocs.joyent.com/manta/api.html#ListDirectory
func (t *tritonObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) { func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) {
var ( var (
dirName string dirName string
objs *storage.ListDirectoryOutput objs *storage.ListDirectoryOutput
input *storage.ListDirectoryInput input *storage.ListDirectoryInput
ctx = context.Background()
pathBase = path.Base(prefix) pathBase = path.Base(prefix)
) )
@ -393,13 +388,12 @@ func (t *tritonObjects) ListObjects(bucket, prefix, marker, delimiter string, ma
// and continuationToken, uses Manta equivalent ListDirectory. // and continuationToken, uses Manta equivalent ListDirectory.
// //
// https://apidocs.joyent.com/manta/api.html#ListDirectory // https://apidocs.joyent.com/manta/api.html#ListDirectory
func (t *tritonObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) { func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
var ( var (
dirName string dirName string
objs *storage.ListDirectoryOutput objs *storage.ListDirectoryOutput
input *storage.ListDirectoryInput input *storage.ListDirectoryInput
ctx = context.Background()
pathBase = path.Base(prefix) pathBase = path.Base(prefix)
) )
@ -467,13 +461,12 @@ func (t *tritonObjects) ListObjectsV2(bucket, prefix, continuationToken, delimit
// indicates the total length of the object. // indicates the total length of the object.
// //
// https://apidocs.joyent.com/manta/api.html#GetObject // https://apidocs.joyent.com/manta/api.html#GetObject
func (t *tritonObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error { func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
// Start offset cannot be negative. // Start offset cannot be negative.
if startOffset < 0 { if startOffset < 0 {
return errors.Trace(fmt.Errorf("Unexpected error")) return errors.Trace(fmt.Errorf("Unexpected error"))
} }
ctx := context.Background()
output, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{ output, err := t.client.Objects().Get(ctx, &storage.GetObjectInput{
ObjectPath: path.Join(mantaRoot, bucket, object), ObjectPath: path.Join(mantaRoot, bucket, object),
}) })
@ -499,8 +492,7 @@ func (t *tritonObjects) GetObject(bucket, object string, startOffset int64, leng
// uses Triton equivalent GetBlobProperties. // uses Triton equivalent GetBlobProperties.
// //
// https://apidocs.joyent.com/manta/api.html#GetObject // https://apidocs.joyent.com/manta/api.html#GetObject
func (t *tritonObjects) GetObjectInfo(bucket, object string) (objInfo minio.ObjectInfo, err error) { func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) {
ctx := context.Background()
info, err := t.client.Objects().GetInfo(ctx, &storage.GetInfoInput{ info, err := t.client.Objects().GetInfo(ctx, &storage.GetInfoInput{
ObjectPath: path.Join(mantaRoot, bucket, object), ObjectPath: path.Join(mantaRoot, bucket, object),
}) })
@ -538,8 +530,7 @@ func (d dummySeeker) Seek(offset int64, whence int) (int64, error) {
// CreateBlockBlobFromReader. // CreateBlockBlobFromReader.
// //
// https://apidocs.joyent.com/manta/api.html#PutObject // https://apidocs.joyent.com/manta/api.html#PutObject
func (t *tritonObjects) PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
ctx := context.Background()
if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{ if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{
ContentLength: uint64(data.Size()), ContentLength: uint64(data.Size()),
ObjectPath: path.Join(mantaRoot, bucket, object), ObjectPath: path.Join(mantaRoot, bucket, object),
@ -552,19 +543,18 @@ func (t *tritonObjects) PutObject(bucket, object string, data *hash.Reader, meta
return objInfo, errors.Trace(err) return objInfo, errors.Trace(err)
} }
if err = data.Verify(); err != nil { if err = data.Verify(); err != nil {
t.DeleteObject(bucket, object) t.DeleteObject(ctx, bucket, object)
return objInfo, errors.Trace(err) return objInfo, errors.Trace(err)
} }
return t.GetObjectInfo(bucket, object) return t.GetObjectInfo(ctx, bucket, object)
} }
// CopyObject - Copies a blob from source container to destination container. // CopyObject - Copies a blob from source container to destination container.
// Uses Manta Snaplinks API. // Uses Manta Snaplinks API.
// //
// https://apidocs.joyent.com/manta/api.html#PutSnapLink // https://apidocs.joyent.com/manta/api.html#PutSnapLink
func (t *tritonObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) { func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
ctx := context.Background()
if err = t.client.SnapLinks().Put(ctx, &storage.PutSnapLinkInput{ if err = t.client.SnapLinks().Put(ctx, &storage.PutSnapLinkInput{
SourcePath: path.Join(mantaRoot, srcBucket, srcObject), SourcePath: path.Join(mantaRoot, srcBucket, srcObject),
LinkPath: path.Join(mantaRoot, destBucket, destObject), LinkPath: path.Join(mantaRoot, destBucket, destObject),
@ -572,14 +562,13 @@ func (t *tritonObjects) CopyObject(srcBucket, srcObject, destBucket, destObject
return objInfo, errors.Trace(err) return objInfo, errors.Trace(err)
} }
return t.GetObjectInfo(destBucket, destObject) return t.GetObjectInfo(ctx, destBucket, destObject)
} }
// DeleteObject - Delete a blob in Manta, uses Triton equivalent DeleteBlob API. // DeleteObject - Delete a blob in Manta, uses Triton equivalent DeleteBlob API.
// //
// https://apidocs.joyent.com/manta/api.html#DeleteObject // https://apidocs.joyent.com/manta/api.html#DeleteObject
func (t *tritonObjects) DeleteObject(bucket, object string) error { func (t *tritonObjects) DeleteObject(ctx context.Context, bucket, object string) error {
ctx := context.Background()
if err := t.client.Objects().Delete(ctx, &storage.DeleteObjectInput{ if err := t.client.Objects().Delete(ctx, &storage.DeleteObjectInput{
ObjectPath: path.Join(mantaRoot, bucket, object), ObjectPath: path.Join(mantaRoot, bucket, object),
}); err != nil { }); err != nil {

Write Preview
Loading…
Cancel
Save