oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Switch to kurin/blazer from minio/blazer fork for b2 gateway (#7879)

Browse Source
master
mzukowski-reef 6 years ago committed by Harshavardhana
parent 8e09374cb8
commit 9d49688c87
4 changed files with 99 additions and 56 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 52
      cmd/gateway/b2/gateway-b2.go
  2. 97
      cmd/gateway/b2/gateway-b2_test.go
  3. 2
      go.mod
  4. 4
      go.sum

52
cmd/gateway/b2/gateway-b2.go
Unescape View File

@ -29,7 +29,7 @@ import (
"sync" "sync"
"time" "time"
b2 "github.com/minio/blazer/base" b2 "github.com/kurin/blazer/base"
"github.com/minio/cli" "github.com/minio/cli"
miniogopolicy "github.com/minio/minio-go/v6/pkg/policy" miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
@ -147,6 +147,29 @@ func b2ToObjectError(err error, params ...string) error {
if err == nil { if err == nil {
return nil return nil
} }
code, msgCode, msg := b2.MsgCode(err)
if code == 0 {
// We don't interpret non B2 errors. B2 errors have statusCode
// to help us convert them to S3 object errors.
return err
}
objErr := b2MsgCodeToObjectError(code, msgCode, msg, params...)
if objErr == nil {
return err
}
return objErr
}
func b2MsgCodeToObjectError(code int, msgCode string, msg string, params ...string) error {
// Following code is a non-exhaustive check to convert
// B2 errors into S3 compatible errors.
//
// For a more complete information - https://www.backblaze.com/b2/docs/
var err error
bucket := "" bucket := ""
object := "" object := ""
uploadID := "" uploadID := ""
@ -160,18 +183,7 @@ func b2ToObjectError(err error, params ...string) error {
uploadID = params[2] uploadID = params[2]
} }
// Following code is a non-exhaustive check to convert switch msgCode {
// B2 errors into S3 compatible errors.
//
// For a more complete information - https://www.backblaze.com/b2/docs/
statusCode, code, msg := b2.Code(err)
if statusCode == 0 {
// We don't interpret non B2 errors. B2 errors have statusCode
// to help us convert them to S3 object errors.
return err
}
switch code {
case "duplicate_bucket_name": case "duplicate_bucket_name":
err = minio.BucketAlreadyOwnedByYou{Bucket: bucket} err = minio.BucketAlreadyOwnedByYou{Bucket: bucket}
case "bad_request": case "bad_request":
@ -343,7 +355,7 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
Name: file.Name, Name: file.Name,
ModTime: file.Timestamp, ModTime: file.Timestamp,
Size: file.Size, Size: file.Size,
ETag: minio.ToS3ETag(file.Info.ID), ETag: minio.ToS3ETag(file.ID),
ContentType: file.Info.ContentType, ContentType: file.Info.ContentType,
UserDefined: file.Info.Info, UserDefined: file.Info.Info,
}) })
@ -386,7 +398,7 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
Name: file.Name, Name: file.Name,
ModTime: file.Timestamp, ModTime: file.Timestamp,
Size: file.Size, Size: file.Size,
ETag: minio.ToS3ETag(file.Info.ID), ETag: minio.ToS3ETag(file.ID),
ContentType: file.Info.ContentType, ContentType: file.Info.ContentType,
UserDefined: file.Info.Info, UserDefined: file.Info.Info,
}) })
@ -462,7 +474,7 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
return minio.ObjectInfo{ return minio.ObjectInfo{
Bucket: bucket, Bucket: bucket,
Name: object, Name: object,
ETag: minio.ToS3ETag(fi.ID), ETag: minio.ToS3ETag(f.ID),
Size: fi.Size, Size: fi.Size,
ModTime: fi.Timestamp, ModTime: fi.Timestamp,
ContentType: fi.ContentType, ContentType: fi.ContentType,
@ -569,7 +581,7 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string,
return minio.ObjectInfo{ return minio.ObjectInfo{
Bucket: bucket, Bucket: bucket,
Name: object, Name: object,
ETag: minio.ToS3ETag(fi.ID), ETag: minio.ToS3ETag(f.ID),
Size: fi.Size, Size: fi.Size,
ModTime: fi.Timestamp, ModTime: fi.Timestamp,
ContentType: fi.ContentType, ContentType: fi.ContentType,
@ -617,7 +629,7 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
if maxUploads > 100 { if maxUploads > 100 {
maxUploads = 100 maxUploads = 100
} }
largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads) largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, maxUploads, uploadIDMarker)
if err != nil { if err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
return lmi, b2ToObjectError(err, bucket) return lmi, b2ToObjectError(err, bucket)
@ -677,7 +689,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
} }
hr := newB2Reader(data, data.Size()) hr := newB2Reader(data, data.Size())
sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID) _, err = fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
if err != nil { if err != nil {
logger.LogIf(ctx, err) logger.LogIf(ctx, err)
return pi, b2ToObjectError(err, bucket, object, uploadID) return pi, b2ToObjectError(err, bucket, object, uploadID)
@ -686,7 +698,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
return minio.PartInfo{ return minio.PartInfo{
PartNumber: partID, PartNumber: partID,
LastModified: minio.UTCNow(), LastModified: minio.UTCNow(),
ETag: minio.ToS3ETag(sha1), ETag: minio.ToS3ETag(fmt.Sprintf("%x", hr.sha1Hash.Sum(nil))),
Size: data.Size(), Size: data.Size(),
}, nil }, nil
} }

97
cmd/gateway/b2/gateway-b2_test.go
Unescape View File

@ -20,8 +20,6 @@ import (
"fmt" "fmt"
"testing" "testing"
b2 "github.com/minio/blazer/base"
minio "github.com/minio/minio/cmd" minio "github.com/minio/minio/cmd"
) )
@ -41,75 +39,104 @@ func TestB2ObjectError(t *testing.T) {
{ {
[]string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"), []string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"),
}, },
}
for i, testCase := range testCases {
actualErr := b2ToObjectError(testCase.b2Err, testCase.params...)
if actualErr != nil {
if actualErr.Error() != testCase.expectedErr.Error() {
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
}
}
}
}
// Test b2 msg code to object error.
func TestB2MsgCodeToObjectError(t *testing.T) {
testCases := []struct {
params []string
code int
msgCode string
msg string
expectedErr error
}{
{ {
[]string{"bucket"}, b2.Error{ []string{"bucket"},
StatusCode: 1, 1,
Code: "duplicate_bucket_name", "duplicate_bucket_name",
}, minio.BucketAlreadyOwnedByYou{ "",
minio.BucketAlreadyOwnedByYou{
Bucket: "bucket", Bucket: "bucket",
}, },
}, },
{ {
[]string{"bucket"}, b2.Error{ []string{"bucket"},
StatusCode: 1, 1,
Code: "bad_request", "bad_request",
}, minio.BucketNotFound{ "",
minio.BucketNotFound{
Bucket: "bucket", Bucket: "bucket",
}, },
}, },
{ {
[]string{"bucket", "object"}, b2.Error{ []string{"bucket", "object"},
StatusCode: 1, 1,
Code: "bad_request", "bad_request",
}, minio.ObjectNameInvalid{ "",
minio.ObjectNameInvalid{
Bucket: "bucket", Bucket: "bucket",
Object: "object", Object: "object",
}, },
}, },
{ {
[]string{"bucket"}, b2.Error{ []string{"bucket"},
StatusCode: 1, 1,
Code: "bad_bucket_id", "bad_bucket_id",
}, minio.BucketNotFound{Bucket: "bucket"}, "",
minio.BucketNotFound{Bucket: "bucket"},
}, },
{ {
[]string{"bucket", "object"}, b2.Error{ []string{"bucket", "object"},
StatusCode: 1, 1,
Code: "file_not_present", "file_not_present",
}, minio.ObjectNotFound{ "",
minio.ObjectNotFound{
Bucket: "bucket", Bucket: "bucket",
Object: "object", Object: "object",
}, },
}, },
{ {
[]string{"bucket", "object"}, b2.Error{ []string{"bucket", "object"},
StatusCode: 1, 1,
Code: "not_found", "not_found",
}, minio.ObjectNotFound{ "",
minio.ObjectNotFound{
Bucket: "bucket", Bucket: "bucket",
Object: "object", Object: "object",
}, },
}, },
{ {
[]string{"bucket"}, b2.Error{ []string{"bucket"},
StatusCode: 1, 1,
Code: "cannot_delete_non_empty_bucket", "cannot_delete_non_empty_bucket",
}, minio.BucketNotEmpty{ "",
minio.BucketNotEmpty{
Bucket: "bucket", Bucket: "bucket",
}, },
}, },
{ {
[]string{"bucket", "object", "uploadID"}, b2.Error{ []string{"bucket", "object", "uploadID"},
StatusCode: 1, 1,
Message: "No active upload for", "",
}, minio.InvalidUploadID{ "No active upload for",
minio.InvalidUploadID{
UploadID: "uploadID", UploadID: "uploadID",
}, },
}, },
} }
for i, testCase := range testCases { for i, testCase := range testCases {
actualErr := b2ToObjectError(testCase.b2Err, testCase.params...) actualErr := b2MsgCodeToObjectError(testCase.code, testCase.msgCode, testCase.msg, testCase.params...)
if actualErr != nil { if actualErr != nil {
if actualErr.Error() != testCase.expectedErr.Error() { if actualErr.Error() != testCase.expectedErr.Error() {
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr) t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)

2
go.mod
Unescape View File

@ -47,11 +47,11 @@ require (
github.com/klauspost/pgzip v1.2.1 github.com/klauspost/pgzip v1.2.1
github.com/klauspost/readahead v1.3.0 github.com/klauspost/readahead v1.3.0
github.com/klauspost/reedsolomon v1.9.1 github.com/klauspost/reedsolomon v1.9.1
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
github.com/lib/pq v1.0.0 github.com/lib/pq v1.0.0
github.com/marstr/guid v1.1.0 // indirect github.com/marstr/guid v1.1.0 // indirect
github.com/mattn/go-isatty v0.0.7 github.com/mattn/go-isatty v0.0.7
github.com/miekg/dns v1.1.8 github.com/miekg/dns v1.1.8
github.com/minio/blazer v0.0.0-20171126203752-2081f5bf0465
github.com/minio/cli v1.20.0 github.com/minio/cli v1.20.0
github.com/minio/dsync/v2 v2.0.0 github.com/minio/dsync/v2 v2.0.0
github.com/minio/hdfs/v3 v3.0.0 github.com/minio/hdfs/v3 v3.0.0

4
go.sum
Unescape View File

@ -376,6 +376,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk=
github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3 h1:1sl2HmNtqGnDuydLgCJwZIpDLGqZOdwOkcY8WtUl8Cw=
github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=

Write Preview
Loading…
Cancel
Save