From fa4d627b5750770992a12ca4c3f0a5d60f1ed7aa Mon Sep 17 00:00:00 2001 From: Ingmar Runge Date: Sat, 28 Mar 2020 21:59:55 +0100 Subject: [PATCH] B2 gateway S3 compat: return MD5 hash as ETag from PutObject (#9183) - B2 does actually return an MD5 hash for newly uploaded objects so we can use it to provide better compatibility with S3 client libraries that assume the ETag is the MD5 hash such as boto. - depends on change in blazer library. - new behaviour is only enabled if MinIO's --compat mode is active. - behaviour for multipart uploads is unchanged (works fine as is). --- cmd/gateway/b2/gateway-b2.go | 47 ++++++++++++++++++++++++------------ docs/gateway/b2.md | 7 +++--- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/cmd/gateway/b2/gateway-b2.go b/cmd/gateway/b2/gateway-b2.go index dea36bce3..f39f14bbc 100644 --- a/cmd/gateway/b2/gateway-b2.go +++ b/cmd/gateway/b2/gateway-b2.go @@ -87,11 +87,15 @@ EXAMPLES: // Handler for 'minio gateway b2' command line. func b2GatewayMain(ctx *cli.Context) { - minio.StartGateway(ctx, &B2{}) + minio.StartGateway(ctx, &B2{ + strictS3Compat: ctx.IsSet("compat") || ctx.GlobalIsSet("compat"), + }) } // B2 implements MinIO Gateway -type B2 struct{} +type B2 struct { + strictS3Compat bool +} // Name implements Gateway interface. func (g *B2) Name() string { @@ -113,7 +117,8 @@ func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) httpClient: &http.Client{ Transport: minio.NewGatewayHTTPTransport(), }, - ctx: ctx, + ctx: ctx, + strictS3Compat: g.strictS3Compat, }, nil } @@ -125,12 +130,13 @@ func (g *B2) Production() bool { // b2Object implements gateway for MinIO and BackBlaze B2 compatible object storage servers. type b2Objects struct { minio.GatewayUnsupported - mu sync.Mutex - creds auth.Credentials - b2Client *b2.B2 - httpClient *http.Client - ctx context.Context - buckets []*b2.Bucket + mu sync.Mutex + creds auth.Credentials + b2Client *b2.B2 + httpClient *http.Client + ctx context.Context + buckets []*b2.Bucket + strictS3Compat bool } // Convert B2 errors to minio object layer errors. @@ -248,6 +254,15 @@ func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error { return nil } +// getETag returns an S3-compatible md5sum-ETag if requested and possible. +func (l *b2Objects) getETag(f *b2.File, fi *b2.FileInfo) string { + if l.strictS3Compat && fi.MD5 != "" { + return fi.MD5 + } + + return minio.ToS3ETag(f.ID) +} + // listBuckets is a wrapper similar to ListBuckets, which re-authorizes // the account and updates the B2 client safely. Once successfully // authorized performs the call again and returns list of buckets. @@ -374,7 +389,7 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin Name: file.Name, ModTime: file.Timestamp, Size: file.Size, - ETag: minio.ToS3ETag(file.ID), + ETag: l.getETag(file, file.Info), ContentType: file.Info.ContentType, UserDefined: file.Info.Info, }) @@ -427,7 +442,7 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat Name: file.Name, ModTime: file.Timestamp, Size: file.Size, - ETag: minio.ToS3ETag(file.ID), + ETag: l.getETag(file, file.Info), ContentType: file.Info.ContentType, UserDefined: file.Info.Info, }) @@ -452,7 +467,7 @@ func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r pr, pw := io.Pipe() go func() { - err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts) + err := l.GetObject(ctx, bucket, object, startOffset, length, pw, "", opts) pw.CloseWithError(err) }() // Setup cleanup function to cause the above go-routine to @@ -472,7 +487,7 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, if err != nil { return err } - reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length) + reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length, false) if err != nil { l.buckets = []*b2.Bucket{} @@ -481,7 +496,7 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, return err } - reader, err = bkt.DownloadFileByName(l.ctx, object, startOffset, length) + reader, err = bkt.DownloadFileByName(l.ctx, object, startOffset, length, false) if err != nil { logger.LogIf(ctx, err) return b2ToObjectError(err, bucket, object) @@ -543,7 +558,7 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str return minio.ObjectInfo{ Bucket: bucket, Name: object, - ETag: minio.ToS3ETag(f[0].ID), + ETag: l.getETag(f[0], fi), Size: fi.Size, ModTime: fi.Timestamp, ContentType: fi.ContentType, @@ -660,7 +675,7 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, return minio.ObjectInfo{ Bucket: bucket, Name: object, - ETag: minio.ToS3ETag(f.ID), + ETag: l.getETag(f, fi), Size: fi.Size, ModTime: fi.Timestamp, ContentType: fi.ContentType, diff --git a/docs/gateway/b2.md b/docs/gateway/b2.md index 6154665d5..891acae99 100644 --- a/docs/gateway/b2.md +++ b/docs/gateway/b2.md @@ -43,15 +43,16 @@ mc ls myb2 ### Known limitations Gateway inherits the following B2 limitations: -- PutObject S3 API does not return the md5sum of an uploaded file as it's etag response. Applications that check the validity of the uploaded object by comparing these two values will fail. -- No support for CopyObject S3 API (There are no equivalent APIs available on Backblaze B2). -- No support for CopyObjectPart S3 API (There are no equivalent APIs available on Backblaze B2). +- No support for CopyObject S3 API (yet). +- No support for CopyObjectPart S3 API (yet). - Only read-only bucket policy supported at bucket level, all other variations will return API Notimplemented error. - DeleteObject() might not delete the object right away on Backblaze B2, so you might see the object immediately after a Delete request. +- Multipart uploads don't return an MD5-hash ETag, not even with the `--compat` flag. Other limitations: - Bucket notification APIs are not supported. +- The PutObject S3 API only returns the md5sum of an uploaded file as it's ETag response when minio is started in `--compat` mode. Don't use this unless you are stuck with a client library where this check cannot be turned off (e.g. boto). ## Explore Further - [`mc` command-line interface](https://docs.min.io/docs/minio-client-quickstart-guide) diff --git a/go.mod b/go.mod index 00d575633..ee01e8444 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/klauspost/pgzip v1.2.1 github.com/klauspost/readahead v1.3.1 github.com/klauspost/reedsolomon v1.9.3 - github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3 + github.com/kurin/blazer v0.5.4-0.20200327014341-8f90a40f8af7 github.com/lib/pq v1.1.1 github.com/mattn/go-colorable v0.1.1 github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect diff --git a/go.sum b/go.sum index 8af5b121c..83729b6ba 100644 --- a/go.sum +++ b/go.sum @@ -246,6 +246,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3 h1:1sl2HmNtqGnDuydLgCJwZIpDLGqZOdwOkcY8WtUl8Cw= github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= +github.com/kurin/blazer v0.5.4-0.20200327014341-8f90a40f8af7 h1:smZXPopqRVVywwzou4WYWvUbJvSAzIDFizfWElpmAqY= +github.com/kurin/blazer v0.5.4-0.20200327014341-8f90a40f8af7/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5 h1:0x4qcEHDpruK6ML/m/YSlFUUu0UpRD3I2PHsNCuGnyA=