From 69ee28a08206dce52d275dfcf74a9b52ac2bdfa9 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 18 Apr 2020 22:12:51 -0700 Subject: [PATCH] remove OSS gateway due to lack of licensing (#9390) OSS go sdk lacks licensing terms in their repository, and there has been no activity On the issue here https://github.com/aliyun/aliyun-oss-go-sdk/issues/245 This PR is to ensure we remove any dependency code which lacks explicit license file in their repo. --- CREDITS | 53 -- cmd/api-errors.go | 7 - cmd/gateway/gateway.go | 1 - cmd/gateway/oss/gateway-oss.go | 1107 --------------------------- cmd/gateway/oss/gateway-oss_test.go | 189 ----- docs/gateway/README.md | 1 - docs/gateway/oss.md | 57 -- go.mod | 3 - go.sum | 6 - 9 files changed, 1424 deletions(-) delete mode 100644 cmd/gateway/oss/gateway-oss.go delete mode 100644 cmd/gateway/oss/gateway-oss_test.go delete mode 100644 docs/gateway/oss.md diff --git a/CREDITS b/CREDITS index eed445382..ba2f487e9 100644 --- a/CREDITS +++ b/CREDITS @@ -1306,33 +1306,6 @@ https://github.com/aws/aws-sdk-go ================================================================ -github.com/baiyubin/aliyun-sts-go-sdk -https://github.com/baiyubin/aliyun-sts-go-sdk ----------------------------------------------------------------- -The MIT License (MIT) - -Copyright (c) Aliyun. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -================================================================ - github.com/bcicen/jstream https://github.com/bcicen/jstream ---------------------------------------------------------------- @@ -16597,32 +16570,6 @@ SOFTWARE. ================================================================ -github.com/satori/go.uuid -https://github.com/satori/go.uuid ----------------------------------------------------------------- -Copyright (C) 2013-2018 by Maxim Bublis - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -================================================================ - github.com/secure-io/sio-go https://github.com/secure-io/sio-go ---------------------------------------------------------------- diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 872126d2f..fea254e09 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -25,7 +25,6 @@ import ( "strings" "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/aliyun/aliyun-oss-go-sdk/oss" "google.golang.org/api/googleapi" minio "github.com/minio/minio-go/v6" @@ -1917,12 +1916,6 @@ func toAPIError(ctx context.Context, err error) APIError { Description: e.Error(), HTTPStatusCode: e.Response().StatusCode, } - case oss.ServiceError: - apiErr = APIError{ - Code: e.Code, - Description: e.Message, - HTTPStatusCode: e.StatusCode, - } // Add more Gateway SDKs here if any in future. } } diff --git a/cmd/gateway/gateway.go b/cmd/gateway/gateway.go index 3e61ed6bd..b88a7575c 100644 --- a/cmd/gateway/gateway.go +++ b/cmd/gateway/gateway.go @@ -22,7 +22,6 @@ import ( _ "github.com/minio/minio/cmd/gateway/gcs" _ "github.com/minio/minio/cmd/gateway/hdfs" _ "github.com/minio/minio/cmd/gateway/nas" - _ "github.com/minio/minio/cmd/gateway/oss" _ "github.com/minio/minio/cmd/gateway/s3" // B2 is specifically kept here to avoid re-ordering by goimports, diff --git a/cmd/gateway/oss/gateway-oss.go b/cmd/gateway/oss/gateway-oss.go deleted file mode 100644 index eea00104c..000000000 --- a/cmd/gateway/oss/gateway-oss.go +++ /dev/null @@ -1,1107 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package oss - -import ( - "context" - "encoding/xml" - "fmt" - "io" - "net/http" - "strconv" - "strings" - - "github.com/aliyun/aliyun-oss-go-sdk/oss" - humanize "github.com/dustin/go-humanize" - - "github.com/minio/cli" - miniogopolicy "github.com/minio/minio-go/v6/pkg/policy" - "github.com/minio/minio-go/v6/pkg/s3utils" - minio "github.com/minio/minio/cmd" - xhttp "github.com/minio/minio/cmd/http" - "github.com/minio/minio/cmd/logger" - "github.com/minio/minio/pkg/auth" - "github.com/minio/minio/pkg/bucket/policy" - "github.com/minio/minio/pkg/bucket/policy/condition" - "github.com/minio/minio/pkg/hash" -) - -const ( - ossS3MinPartSize = 5 * humanize.MiByte - ossMaxParts = 1000 - ossMaxKeys = 1000 - ossBackend = "oss" -) - -func init() { - const ossGatewayTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT] -{{if .VisibleFlags}} -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}{{end}} -ENDPOINT: - oss server endpoint. Default ENDPOINT is https://oss.aliyuncs.com - -EXAMPLES: - 1. Start minio gateway server for Aliyun OSS backend - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey - {{.Prompt}} {{.HelpName}} - - 2. Start minio gateway server for Aliyun OSS backend with edge caching enabled - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4" - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png" - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3 - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75 - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85 - {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90 - {{.Prompt}} {{.HelpName}} -` - - minio.RegisterGatewayCommand(cli.Command{ - Name: "oss", - Usage: "Alibaba Cloud (Aliyun) Object Storage Service (OSS)", - Action: ossGatewayMain, - CustomHelpTemplate: ossGatewayTemplate, - HideHelpCommand: true, - }) -} - -// Handler for 'minio gateway oss' command line. -func ossGatewayMain(ctx *cli.Context) { - if ctx.Args().First() == "help" { - cli.ShowCommandHelpAndExit(ctx, ossBackend, 1) - } - - // Validate gateway arguments. - host := ctx.Args().First() - logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") - - minio.StartGateway(ctx, &OSS{host}) -} - -// OSS implements Gateway. -type OSS struct { - host string -} - -// Name implements Gateway interface. -func (g *OSS) Name() string { - return ossBackend -} - -// NewGatewayLayer implements Gateway interface and returns OSS ObjectLayer. -func (g *OSS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) { - var err error - - // Regions and endpoints - // https://www.alibabacloud.com/help/doc-detail/31837.htm - if g.host == "" { - g.host = "https://oss.aliyuncs.com" - } - - // Initialize oss client object. - client, err := oss.New(g.host, creds.AccessKey, creds.SecretKey) - if err != nil { - return nil, err - } - client.HTTPClient = &http.Client{ - Transport: minio.NewGatewayHTTPTransport(), - } - return &ossObjects{ - Client: client, - }, nil -} - -// Production - oss is production ready. -func (g *OSS) Production() bool { - return true -} - -// appendS3MetaToOSSOptions converts metadata meant for S3 PUT/COPY -// object into oss.Option. -// -// S3 user-metadata is translated to OSS metadata by removing the -// `X-Amz-Meta-` prefix and converted into `X-Oss-Meta-`. -// -// Header names are canonicalized as in http.Header. -func appendS3MetaToOSSOptions(ctx context.Context, opts []oss.Option, s3Metadata map[string]string) ([]oss.Option, error) { - if opts == nil { - opts = make([]oss.Option, 0, len(s3Metadata)) - } - for k, v := range s3Metadata { - k = http.CanonicalHeaderKey(k) - - switch { - case strings.HasPrefix(k, "X-Amz-Meta-"): - metaKey := k[len("X-Amz-Meta-"):] - // NOTE(timonwong): OSS won't allow headers with underscore(_). - if strings.Contains(metaKey, "_") { - logger.LogIf(ctx, minio.UnsupportedMetadata{}) - return nil, minio.UnsupportedMetadata{} - } - opts = append(opts, oss.Meta(metaKey, v)) - case k == "X-Amz-Acl": - // Valid values: public-read, private, and public-read-write - opts = append(opts, oss.ObjectACL(oss.ACLType(v))) - case k == "X-Amz-Server-Side-Encryption": - opts = append(opts, oss.ServerSideEncryption(v)) - case k == "X-Amz-Copy-Source-If-Match": - opts = append(opts, oss.CopySourceIfMatch(v)) - case k == "X-Amz-Copy-Source-If-None-Match": - opts = append(opts, oss.CopySourceIfNoneMatch(v)) - case k == "X-Amz-Copy-Source-If-Unmodified-Since": - if v, err := http.ParseTime(v); err == nil { - opts = append(opts, oss.CopySourceIfUnmodifiedSince(v)) - } - case k == "X-Amz-Copy-Source-If-Modified-Since": - if v, err := http.ParseTime(v); err == nil { - opts = append(opts, oss.CopySourceIfModifiedSince(v)) - } - case k == "Accept-Encoding": - opts = append(opts, oss.AcceptEncoding(v)) - case k == "Cache-Control": - opts = append(opts, oss.CacheControl(v)) - case k == "Content-Disposition": - opts = append(opts, oss.ContentDisposition(v)) - case k == "Content-Encoding": - opts = append(opts, oss.ContentEncoding(v)) - case k == "Content-Length": - if v, err := strconv.ParseInt(v, 10, 64); err == nil { - opts = append(opts, oss.ContentLength(v)) - } - case k == "Content-MD5": - opts = append(opts, oss.ContentMD5(v)) - case k == "Content-Type": - opts = append(opts, oss.ContentType(v)) - case k == "Expires": - if v, err := http.ParseTime(v); err == nil { - opts = append(opts, oss.Expires(v)) - } - } - } - - return opts, nil -} - -// ossMetaToS3Meta converts OSS metadata to S3 metadata. -// It is the reverse of appendS3MetaToOSSOptions. -func ossHeaderToS3Meta(header http.Header) map[string]string { - // Decoding technique for each key is used here is as follows - // Each '_' is converted to '-' - // Each '__' is converted to '_' - // With this basic assumption here are some of the expected - // translations for these keys. - // i: 'x_s3cmd__attrs' -> o: 'x-s3cmd_attrs' (mixed) - // i: 'x____test____value' -> o: 'x__test__value' (double '_') - decodeKey := func(key string) string { - tokens := strings.Split(key, "__") - for i := range tokens { - tokens[i] = strings.Replace(tokens[i], "_", "-", -1) - } - return strings.Join(tokens, "_") - } - - s3Metadata := make(map[string]string) - for k := range header { - k = http.CanonicalHeaderKey(k) - switch { - case strings.HasPrefix(k, oss.HTTPHeaderOssMetaPrefix): - // Add amazon s3 meta prefix - metaKey := k[len(oss.HTTPHeaderOssMetaPrefix):] - metaKey = "X-Amz-Meta-" + decodeKey(metaKey) - metaKey = http.CanonicalHeaderKey(metaKey) - s3Metadata[metaKey] = header.Get(k) - case k == "Cache-Control": - fallthrough - case k == "Content-Encoding": - fallthrough - case k == "Content-Disposition": - fallthrough - case k == "Content-Length": - fallthrough - case k == "Content-MD5": - fallthrough - case k == "Content-Type": - s3Metadata[k] = header.Get(k) - } - } - - return s3Metadata -} - -// ossToObjectError converts OSS errors to minio object layer errors. -func ossToObjectError(err error, params ...string) error { - if err == nil { - return nil - } - - bucket := "" - object := "" - uploadID := "" - switch len(params) { - case 3: - uploadID = params[2] - fallthrough - case 2: - object = params[1] - fallthrough - case 1: - bucket = params[0] - } - - ossErr, ok := err.(oss.ServiceError) - if !ok { - // We don't interpret non OSS errors. As oss errors will - // have StatusCode to help to convert to object errors. - return err - } - - switch ossErr.Code { - case "BucketAlreadyExists": - err = minio.BucketAlreadyOwnedByYou{Bucket: bucket} - case "BucketNotEmpty": - err = minio.BucketNotEmpty{Bucket: bucket} - case "InvalidBucketName": - err = minio.BucketNameInvalid{Bucket: bucket} - case "NoSuchBucket": - err = minio.BucketNotFound{Bucket: bucket} - case "NoSuchKey": - if object != "" { - err = minio.ObjectNotFound{Bucket: bucket, Object: object} - } else { - err = minio.BucketNotFound{Bucket: bucket} - } - case "InvalidObjectName": - err = minio.ObjectNameInvalid{Bucket: bucket, Object: object} - case "AccessDenied": - err = minio.PrefixAccessDenied{Bucket: bucket, Object: object} - case "NoSuchUpload": - err = minio.InvalidUploadID{UploadID: uploadID} - case "EntityTooSmall": - err = minio.PartTooSmall{} - case "SignatureDoesNotMatch": - err = minio.SignatureDoesNotMatch{} - case "InvalidPart": - err = minio.InvalidPart{} - } - - return err -} - -// ossObjects implements gateway for Aliyun Object Storage Service. -type ossObjects struct { - minio.GatewayUnsupported - Client *oss.Client -} - -// Shutdown saves any gateway metadata to disk -// if necessary and reload upon next restart. -func (l *ossObjects) Shutdown(ctx context.Context) error { - return nil -} - -// StorageInfo is not relevant to OSS backend. -func (l *ossObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) { - si.Backend.Type = minio.BackendGateway - si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint) - return si -} - -// ossIsValidBucketName verifies whether a bucket name is valid. -func ossIsValidBucketName(bucket string) bool { - // dot is not allowed in bucket name - if strings.Contains(bucket, ".") { - return false - } - if s3utils.CheckValidBucketNameStrict(bucket) != nil { - return false - } - return true -} - -// MakeBucketWithLocation creates a new container on OSS backend. -func (l *ossObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error { - if !ossIsValidBucketName(bucket) { - logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket}) - return minio.BucketNameInvalid{Bucket: bucket} - } - - err := l.Client.CreateBucket(bucket) - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket) -} - -// ossGeBucketInfo gets bucket metadata. -func ossGeBucketInfo(ctx context.Context, client *oss.Client, bucket string) (bi minio.BucketInfo, err error) { - bgir, err := client.GetBucketInfo(bucket) - if err != nil { - logger.LogIf(ctx, err) - return bi, ossToObjectError(err, bucket) - } - - return minio.BucketInfo{ - Name: bgir.BucketInfo.Name, - Created: bgir.BucketInfo.CreationDate, - }, nil -} - -// GetBucketInfo gets bucket metadata. -func (l *ossObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) { - return ossGeBucketInfo(ctx, l.Client, bucket) -} - -// ListBuckets lists all OSS buckets. -func (l *ossObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) { - marker := oss.Marker("") - for { - lbr, err := l.Client.ListBuckets(marker) - if err != nil { - logger.LogIf(ctx, err) - return nil, ossToObjectError(err) - } - - for _, bi := range lbr.Buckets { - buckets = append(buckets, minio.BucketInfo{ - Name: bi.Name, - Created: bi.CreationDate, - }) - } - - marker = oss.Marker(lbr.NextMarker) - if !lbr.IsTruncated { - break - } - } - - return buckets, nil -} - -// DeleteBucket deletes a bucket on OSS. -func (l *ossObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error { - err := l.Client.DeleteBucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket) - } - return nil -} - -// fromOSSClientObjectProperties converts oss ObjectProperties to ObjectInfo. -func fromOSSClientObjectProperties(bucket string, o oss.ObjectProperties) minio.ObjectInfo { - // NOTE(timonwong): No Content-Type and user defined metadata. - // https://www.alibabacloud.com/help/doc-detail/31965.htm - - return minio.ObjectInfo{ - Bucket: bucket, - Name: o.Key, - ModTime: o.LastModified, - Size: o.Size, - ETag: o.ETag, - } -} - -// fromOSSClientListObjectsResult converts oss ListBucketResult to ListObjectsInfo. -func fromOSSClientListObjectsResult(bucket string, lor oss.ListObjectsResult) minio.ListObjectsInfo { - objects := make([]minio.ObjectInfo, len(lor.Objects)) - for i, oi := range lor.Objects { - objects[i] = fromOSSClientObjectProperties(bucket, oi) - } - - prefixes := make([]string, len(lor.CommonPrefixes)) - copy(prefixes, lor.CommonPrefixes) - - return minio.ListObjectsInfo{ - IsTruncated: lor.IsTruncated, - NextMarker: lor.NextMarker, - Objects: objects, - Prefixes: prefixes, - } -} - -// ossListObjects lists all blobs in OSS bucket filtered by prefix. -func ossListObjects(ctx context.Context, client *oss.Client, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { - buck, err := client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return loi, ossToObjectError(err, bucket) - } - - // maxKeys should default to 1000 or less. - if maxKeys == 0 || maxKeys > ossMaxKeys { - maxKeys = ossMaxKeys - } - - lor, err := buck.ListObjects(oss.Prefix(prefix), oss.Marker(marker), oss.Delimiter(delimiter), oss.MaxKeys(maxKeys)) - if err != nil { - logger.LogIf(ctx, err) - return loi, ossToObjectError(err, bucket) - } - - return fromOSSClientListObjectsResult(bucket, lor), nil -} - -// ossListObjectsV2 lists all blobs in OSS bucket filtered by prefix. -func ossListObjectsV2(ctx context.Context, client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int, - fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) { - // fetchOwner is not supported and unused. - marker := continuationToken - if marker == "" { - marker = startAfter - } - - resultV1, err := ossListObjects(ctx, client, bucket, prefix, marker, delimiter, maxKeys) - if err != nil { - return loi, err - } - - return minio.ListObjectsV2Info{ - Objects: resultV1.Objects, - Prefixes: resultV1.Prefixes, - ContinuationToken: continuationToken, - NextContinuationToken: resultV1.NextMarker, - IsTruncated: resultV1.IsTruncated, - }, nil -} - -// ListObjects lists all blobs in OSS bucket filtered by prefix. -func (l *ossObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) { - return ossListObjects(ctx, l.Client, bucket, prefix, marker, delimiter, maxKeys) -} - -// ListObjectsV2 lists all blobs in OSS bucket filtered by prefix -func (l *ossObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, - fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) { - return ossListObjectsV2(ctx, l.Client, bucket, prefix, continuationToken, delimiter, maxKeys, fetchOwner, startAfter) -} - -// ossGetObject reads an object on OSS. Supports additional -// parameters like offset and length which are synonymous with -// HTTP Range requests. -// -// startOffset indicates the starting read location of the object. -// length indicates the total length of the object. -func ossGetObject(ctx context.Context, client *oss.Client, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error { - if length < 0 && length != -1 { - logger.LogIf(ctx, fmt.Errorf("Invalid argument"), logger.Application) - return ossToObjectError(fmt.Errorf("Invalid argument"), bucket, key) - } - - bkt, err := client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err, logger.Application) - return ossToObjectError(err, bucket, key) - } - - var opts []oss.Option - if startOffset >= 0 && length >= 0 { - opts = append(opts, oss.Range(startOffset, startOffset+length-1)) - } - - object, err := bkt.GetObject(key, opts...) - if err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket, key) - } - defer object.Close() - - if _, err := io.Copy(writer, object); err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket, key) - } - return nil -} - -// GetObjectNInfo - returns object info and locked object ReadCloser -func (l *ossObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) { - var objInfo minio.ObjectInfo - objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts) - if err != nil { - return nil, err - } - - var startOffset, length int64 - startOffset, length, err = rs.GetOffsetLength(objInfo.Size) - if err != nil { - return nil, err - } - - pr, pw := io.Pipe() - go func() { - err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts) - pw.CloseWithError(err) - }() - // Setup cleanup function to cause the above go-routine to - // exit in case of partial read - pipeCloser := func() { pr.Close() } - return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser) -} - -// GetObject reads an object on OSS. Supports additional -// parameters like offset and length which are synonymous with -// HTTP Range requests. -// -// startOffset indicates the starting read location of the object. -// length indicates the total length of the object. -func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error { - return ossGetObject(ctx, l.Client, bucket, key, startOffset, length, writer, etag) -} - -func translatePlainError(err error) error { - errString := err.Error() - - switch errString { - case "oss: service returned without a response body (404 Not Found)": - return oss.ServiceError{Code: "NoSuchKey"} - case "oss: service returned without a response body (400 Bad Request)": - return oss.ServiceError{Code: "AccessDenied"} - } - - return err -} - -// ossGetObjectInfo reads object info and replies back ObjectInfo. -func ossGetObjectInfo(ctx context.Context, client *oss.Client, bucket, object string) (objInfo minio.ObjectInfo, err error) { - bkt, err := client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return objInfo, ossToObjectError(err, bucket, object) - } - - header, err := bkt.GetObjectDetailedMeta(object) - if err != nil { - logger.LogIf(ctx, translatePlainError(err)) - return objInfo, ossToObjectError(translatePlainError(err), bucket, object) - } - - // Build S3 metadata from OSS metadata - userDefined := ossHeaderToS3Meta(header) - - modTime, _ := http.ParseTime(header.Get("Last-Modified")) - size, _ := strconv.ParseInt(header.Get("Content-Length"), 10, 64) - - return minio.ObjectInfo{ - Bucket: bucket, - Name: object, - ModTime: modTime, - Size: size, - ETag: header.Get("ETag"), - UserDefined: userDefined, - ContentType: header.Get("Content-Type"), - ContentEncoding: header.Get("Content-Encoding"), - }, nil -} - -// GetObjectInfo reads object info and replies back ObjectInfo. -func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { - return ossGetObjectInfo(ctx, l.Client, bucket, object) -} - -// ossPutObject creates a new object with the incoming data. -func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) { - bkt, err := client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return objInfo, ossToObjectError(err, bucket, object) - } - - // Build OSS metadata - opts, err := appendS3MetaToOSSOptions(ctx, nil, metadata) - if err != nil { - return objInfo, ossToObjectError(err, bucket, object) - } - - err = bkt.PutObject(object, data, opts...) - if err != nil { - logger.LogIf(ctx, err) - return objInfo, ossToObjectError(err, bucket, object) - } - - return ossGetObjectInfo(ctx, client, bucket, object) -} - -// PutObject creates a new object with the incoming data. -func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { - data := r.Reader - - return ossPutObject(ctx, l.Client, bucket, object, data, opts.UserDefined) -} - -// CopyObject copies an object from source bucket to a destination bucket. -func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) { - if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") { - return minio.ObjectInfo{}, minio.PreConditionFailed{} - } - bkt, err := l.Client.Bucket(srcBucket) - if err != nil { - logger.LogIf(ctx, err) - return objInfo, ossToObjectError(err, srcBucket, srcObject) - } - - opts := make([]oss.Option, 0, len(srcInfo.UserDefined)+1) - // Set this header such that following CopyObject() always sets the right metadata on the destination. - // metadata input is already a trickled down value from interpreting x-oss-metadata-directive at - // handler layer. So what we have right now is supposed to be applied on the destination object anyways. - // So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API. - opts = append(opts, oss.MetadataDirective(oss.MetaReplace)) - - // Build OSS metadata - opts, err = appendS3MetaToOSSOptions(ctx, opts, srcInfo.UserDefined) - if err != nil { - return objInfo, ossToObjectError(err, srcBucket, srcObject) - } - - if _, err = bkt.CopyObjectTo(dstBucket, dstObject, srcObject, opts...); err != nil { - logger.LogIf(ctx, err) - return objInfo, ossToObjectError(err, srcBucket, srcObject) - } - return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts) -} - -// DeleteObject deletes a blob in bucket. -func (l *ossObjects) DeleteObject(ctx context.Context, bucket, object string) error { - bkt, err := l.Client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket, object) - } - - err = bkt.DeleteObject(object) - if err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket, object) - } - return nil -} - -func (l *ossObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) { - errs := make([]error, len(objects)) - for idx, object := range objects { - errs[idx] = l.DeleteObject(ctx, bucket, object) - } - return errs, nil -} - -// fromOSSClientListMultipartsInfo converts oss ListMultipartUploadResult to ListMultipartsInfo -func fromOSSClientListMultipartsInfo(lmur oss.ListMultipartUploadResult) minio.ListMultipartsInfo { - uploads := make([]minio.MultipartInfo, len(lmur.Uploads)) - for i, um := range lmur.Uploads { - uploads[i] = minio.MultipartInfo{ - Object: um.Key, - UploadID: um.UploadID, - Initiated: um.Initiated, - } - } - - commonPrefixes := make([]string, len(lmur.CommonPrefixes)) - copy(commonPrefixes, lmur.CommonPrefixes) - - return minio.ListMultipartsInfo{ - KeyMarker: lmur.KeyMarker, - UploadIDMarker: lmur.UploadIDMarker, - NextKeyMarker: lmur.NextKeyMarker, - NextUploadIDMarker: lmur.NextUploadIDMarker, - MaxUploads: lmur.MaxUploads, - IsTruncated: lmur.IsTruncated, - Uploads: uploads, - Prefix: lmur.Prefix, - Delimiter: lmur.Delimiter, - CommonPrefixes: commonPrefixes, - } -} - -// ListMultipartUploads lists all multipart uploads. -func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) { - bkt, err := l.Client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return lmi, ossToObjectError(err, bucket) - } - - lmur, err := bkt.ListMultipartUploads(oss.Prefix(prefix), oss.KeyMarker(keyMarker), oss.UploadIDMarker(uploadIDMarker), - oss.Delimiter(delimiter), oss.MaxUploads(maxUploads)) - if err != nil { - logger.LogIf(ctx, err) - return lmi, ossToObjectError(err, bucket) - } - - return fromOSSClientListMultipartsInfo(lmur), nil -} - -// NewMultipartUpload upload object in multiple parts. -func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, o minio.ObjectOptions) (uploadID string, err error) { - bkt, err := l.Client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return uploadID, ossToObjectError(err, bucket, object) - } - - // Build OSS metadata - opts, err := appendS3MetaToOSSOptions(ctx, nil, o.UserDefined) - if err != nil { - return uploadID, ossToObjectError(err, bucket, object) - } - - lmur, err := bkt.InitiateMultipartUpload(object, opts...) - if err != nil { - logger.LogIf(ctx, err) - return uploadID, ossToObjectError(err, bucket, object) - } - - return lmur.UploadID, nil -} - -// PutObjectPart puts a part of object in bucket. -func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) { - data := r.Reader - bkt, err := l.Client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return pi, ossToObjectError(err, bucket, object) - } - - imur := oss.InitiateMultipartUploadResult{ - Bucket: bucket, - Key: object, - UploadID: uploadID, - } - size := data.Size() - up, err := bkt.UploadPart(imur, data, size, partID) - if err != nil { - logger.LogIf(ctx, err) - return pi, ossToObjectError(err, bucket, object) - } - - return minio.PartInfo{ - Size: size, - ETag: up.ETag, - // NOTE(timonwong): LastModified is not supported - PartNumber: up.PartNumber, - }, nil -} - -func ossBuildListObjectPartsParams(uploadID string, partNumberMarker, maxParts int) map[string]interface{} { - return map[string]interface{}{ - "uploadId": uploadID, - "part-number-marker": strconv.Itoa(partNumberMarker), - "max-parts": strconv.Itoa(maxParts), - } -} - -// fromOSSClientListPartsInfo converts OSS ListUploadedPartsResult to ListPartsInfo -func fromOSSClientListPartsInfo(lupr oss.ListUploadedPartsResult, partNumberMarker int) minio.ListPartsInfo { - parts := make([]minio.PartInfo, len(lupr.UploadedParts)) - for i, up := range lupr.UploadedParts { - parts[i] = minio.PartInfo{ - PartNumber: up.PartNumber, - LastModified: up.LastModified, - ETag: up.ETag, - Size: int64(up.Size), - } - } - - nextPartNumberMarker, _ := strconv.Atoi(lupr.NextPartNumberMarker) - return minio.ListPartsInfo{ - Bucket: lupr.Bucket, - Object: lupr.Key, - UploadID: lupr.UploadID, - PartNumberMarker: partNumberMarker, - NextPartNumberMarker: nextPartNumberMarker, - MaxParts: lupr.MaxParts, - IsTruncated: lupr.IsTruncated, - Parts: parts, - } -} - -func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, partNumberMarker, maxParts int) (lupr oss.ListUploadedPartsResult, err error) { - params := ossBuildListObjectPartsParams(uploadID, partNumberMarker, maxParts) - resp, err := client.Conn.Do("GET", bucket, object, params, nil, nil, 0, nil) - if err != nil { - return lupr, err - } - - // always drain output (response body) - defer xhttp.DrainBody(resp.Body) - - err = xml.NewDecoder(resp.Body).Decode(&lupr) - if err != nil { - return lupr, err - } - return lupr, nil -} - -// CopyObjectPart creates a part in a multipart upload by copying -// existing object or a part of it. -func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, - partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) { - - bkt, err := l.Client.Bucket(destBucket) - if err != nil { - logger.LogIf(ctx, err) - return p, ossToObjectError(err, destBucket) - } - - // Build OSS metadata - opts, err := appendS3MetaToOSSOptions(ctx, nil, srcInfo.UserDefined) - if err != nil { - return p, ossToObjectError(err, srcBucket, srcObject) - } - - completePart, err := bkt.UploadPartCopy(oss.InitiateMultipartUploadResult{ - Key: destObject, - UploadID: uploadID, - }, srcBucket, srcObject, startOffset, length, partID, opts...) - - if err != nil { - logger.LogIf(ctx, err) - return p, ossToObjectError(err, srcBucket, srcObject) - } - - p.PartNumber = completePart.PartNumber - p.ETag = completePart.ETag - return p, nil -} - -// ListObjectParts returns all object parts for specified object in specified bucket -func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, err error) { - lupr, err := ossListObjectParts(l.Client, bucket, object, uploadID, partNumberMarker, maxParts) - if err != nil { - logger.LogIf(ctx, err) - return lpi, ossToObjectError(err, bucket, object, uploadID) - } - - return fromOSSClientListPartsInfo(lupr, partNumberMarker), nil -} - -// AbortMultipartUpload aborts a ongoing multipart upload. -func (l *ossObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error { - bkt, err := l.Client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket, object) - } - - err = bkt.AbortMultipartUpload(oss.InitiateMultipartUploadResult{ - Bucket: bucket, - Key: object, - UploadID: uploadID, - }) - if err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket, object) - } - return nil -} - -// CompleteMultipartUpload completes ongoing multipart upload and finalizes object. -func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) { - client := l.Client - bkt, err := client.Bucket(bucket) - if err != nil { - logger.LogIf(ctx, err) - return oi, ossToObjectError(err, bucket, object) - } - - // Error out if uploadedParts except last part sizing < 5MiB. - // NOTE(timonwong): Actually, OSS wont't throw EntityTooSmall error, doing this check just for mint :( - var partNumberMarker int - lupr := oss.ListUploadedPartsResult{IsTruncated: true} - for lupr.IsTruncated { - lupr, err = ossListObjectParts(client, bucket, object, uploadID, partNumberMarker, ossMaxParts) - if err != nil { - logger.LogIf(ctx, err) - return oi, ossToObjectError(err, bucket, object, uploadID) - } - - uploadedParts := lupr.UploadedParts - if !lupr.IsTruncated { - if len(uploadedParts) < 1 { - uploadedParts = nil - } else { - uploadedParts = uploadedParts[:len(uploadedParts)-1] - } - } - - for _, part := range uploadedParts { - if part.Size < ossS3MinPartSize { - logger.LogIf(ctx, minio.PartTooSmall{ - PartNumber: part.PartNumber, - PartSize: int64(part.Size), - PartETag: part.ETag, - }) - return oi, minio.PartTooSmall{ - PartNumber: part.PartNumber, - PartSize: int64(part.Size), - PartETag: part.ETag, - } - } - } - - partNumberMarker, _ = strconv.Atoi(lupr.NextPartNumberMarker) - } - - imur := oss.InitiateMultipartUploadResult{ - Bucket: bucket, - Key: object, - UploadID: uploadID, - } - parts := make([]oss.UploadPart, len(uploadedParts)) - for i, up := range uploadedParts { - parts[i] = oss.UploadPart{ - PartNumber: up.PartNumber, - ETag: strings.TrimSuffix(up.ETag, "-1"), // Trim "-1" suffix in ETag as PutObjectPart(). - } - } - - _, err = bkt.CompleteMultipartUpload(imur, parts) - if err != nil { - logger.LogIf(ctx, err) - return oi, ossToObjectError(err, bucket, object) - } - - return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{}) -} - -// SetBucketPolicy sets policy on bucket. -// OSS supports three types of bucket policies: -// oss.ACLPublicReadWrite: readwrite in minio terminology -// oss.ACLPublicRead: readonly in minio terminology -// oss.ACLPrivate: none in minio terminology -func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error { - policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy) - if err != nil { - // This should not happen. - return ossToObjectError(err, bucket) - } - - bucketPolicies := miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") - if len(bucketPolicies) != 1 { - logger.LogIf(ctx, minio.NotImplemented{}) - return minio.NotImplemented{} - } - - prefix := bucket + "/*" // For all objects inside the bucket. - for policyPrefix, bucketPolicy := range bucketPolicies { - if policyPrefix != prefix { - logger.LogIf(ctx, minio.NotImplemented{}) - return minio.NotImplemented{} - } - - var acl oss.ACLType - switch bucketPolicy { - case miniogopolicy.BucketPolicyNone: - acl = oss.ACLPrivate - case miniogopolicy.BucketPolicyReadOnly: - acl = oss.ACLPublicRead - case miniogopolicy.BucketPolicyReadWrite: - acl = oss.ACLPublicReadWrite - default: - logger.LogIf(ctx, minio.NotImplemented{}) - return minio.NotImplemented{} - } - - err := l.Client.SetBucketACL(bucket, acl) - if err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket) - } - } - - return nil -} - -// GetBucketPolicy will get policy on bucket. -func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) { - result, err := l.Client.GetBucketACL(bucket) - if err != nil { - logger.LogIf(ctx, err) - return nil, ossToObjectError(err) - } - - var readOnly, readWrite bool - switch result.ACL { - case string(oss.ACLPrivate): - // By default, all buckets starts with a "private" policy. - return nil, ossToObjectError(minio.BucketPolicyNotFound{}, bucket) - case string(oss.ACLPublicRead): - readOnly = true - case string(oss.ACLPublicReadWrite): - readWrite = true - default: - logger.LogIf(ctx, minio.NotImplemented{}) - return nil, minio.NotImplemented{} - } - - actionSet := policy.NewActionSet() - if readOnly { - actionSet.Add(policy.GetBucketLocationAction) - actionSet.Add(policy.ListBucketAction) - actionSet.Add(policy.GetObjectAction) - } - if readWrite { - actionSet.Add(policy.GetBucketLocationAction) - actionSet.Add(policy.ListBucketAction) - actionSet.Add(policy.GetObjectAction) - actionSet.Add(policy.ListBucketMultipartUploadsAction) - actionSet.Add(policy.AbortMultipartUploadAction) - actionSet.Add(policy.DeleteObjectAction) - actionSet.Add(policy.ListMultipartUploadPartsAction) - actionSet.Add(policy.PutObjectAction) - } - - return &policy.Policy{ - Version: policy.DefaultVersion, - Statements: []policy.Statement{ - policy.NewStatement( - policy.Allow, - policy.NewPrincipal("*"), - actionSet, - policy.NewResourceSet( - policy.NewResource(bucket, ""), - policy.NewResource(bucket, "*"), - ), - condition.NewFunctions(), - ), - }, - }, nil -} - -// DeleteBucketPolicy deletes all policies on bucket. -func (l *ossObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error { - err := l.Client.SetBucketACL(bucket, oss.ACLPrivate) - if err != nil { - logger.LogIf(ctx, err) - return ossToObjectError(err, bucket) - } - return nil -} - -// IsCompressionSupported returns whether compression is applicable for this layer. -func (l *ossObjects) IsCompressionSupported() bool { - return false -} - -// IsReady returns whether the layer is ready to take requests. -func (l *ossObjects) IsReady(ctx context.Context) bool { - return minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint) -} diff --git a/cmd/gateway/oss/gateway-oss_test.go b/cmd/gateway/oss/gateway-oss_test.go deleted file mode 100644 index 91dc3e69e..000000000 --- a/cmd/gateway/oss/gateway-oss_test.go +++ /dev/null @@ -1,189 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package oss - -import ( - "fmt" - "net/http" - "reflect" - "testing" - - "github.com/aliyun/aliyun-oss-go-sdk/oss" - - minio "github.com/minio/minio/cmd" -) - -func ossErrResponse(code string) error { - return oss.ServiceError{ - Code: code, - } -} - -func TestOSSToObjectError(t *testing.T) { - testCases := []struct { - inputErr error - expectedErr error - bucket, object string - }{ - { - inputErr: ossErrResponse("BucketAlreadyExists"), - expectedErr: minio.BucketAlreadyOwnedByYou{}, - }, - { - inputErr: ossErrResponse("BucketNotEmpty"), - expectedErr: minio.BucketNotEmpty{}, - }, - { - inputErr: ossErrResponse("InvalidBucketName"), - expectedErr: minio.BucketNameInvalid{}, - }, - { - inputErr: ossErrResponse("NoSuchBucket"), - expectedErr: minio.BucketNotFound{}, - }, - // with empty object, NoSuchKey is interpreted as BucketNotFound - { - inputErr: ossErrResponse("NoSuchKey"), - expectedErr: minio.BucketNotFound{}, - }, - { - inputErr: ossErrResponse("NoSuchUpload"), - expectedErr: minio.InvalidUploadID{}, - }, - { - inputErr: ossErrResponse("InvalidObjectName"), - expectedErr: minio.ObjectNameInvalid{}, - }, - { - inputErr: ossErrResponse("AccessDenied"), - expectedErr: minio.PrefixAccessDenied{}, - }, - { - inputErr: ossErrResponse("NoSuchUpload"), - expectedErr: minio.InvalidUploadID{}, - }, - { - inputErr: ossErrResponse("EntityTooSmall"), - expectedErr: minio.PartTooSmall{}, - }, - { - inputErr: nil, - expectedErr: nil, - }, - // Special test case for NoSuchKey with object name - { - inputErr: ossErrResponse("NoSuchKey"), - expectedErr: minio.ObjectNotFound{Bucket: "bucket", Object: "object"}, - bucket: "bucket", - object: "object", - }, - - // Special test case for error value that is not of - // type (*Error) - { - inputErr: fmt.Errorf("not a *Error"), - expectedErr: fmt.Errorf("not a *Error"), - }, - } - - for i, tc := range testCases { - actualErr := ossToObjectError(tc.inputErr, tc.bucket, tc.object) - if actualErr != nil && tc.expectedErr != nil && actualErr.Error() != tc.expectedErr.Error() { - t.Errorf("Test case %d: Expected error '%v' but received error '%v'", i+1, tc.expectedErr, actualErr) - } - } -} - -func TestS3MetaToOSSOptions(t *testing.T) { - var err error - - headers := map[string]string{ - "x-amz-meta-invalid_meta": "value", - } - _, err = appendS3MetaToOSSOptions(minio.GlobalContext, nil, headers) - if err != nil { - if _, ok := err.(minio.UnsupportedMetadata); !ok { - t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err) - } - } - - headers = map[string]string{ - "accept-encoding": "gzip", // not this - "content-encoding": "gzip", - "X-Amz-Meta-Hdr": "value", - "X-Amz-Meta-X-test-key": "value", - "X-Amz-Meta-X--test--key": "value", - "X-Amz-Meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=", - "X-Amz-Meta-X-Amz-Matdesc": "{}", - "X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==", - } - opts, err := appendS3MetaToOSSOptions(minio.GlobalContext, nil, headers) - if err != nil { - t.Fatalf("Test failed, with %s", err) - } - if len(opts) != len(headers) { - t.Fatalf("Test failed, S3 metdata is not fully transformed. expeted: %d, actual: %d", len(headers)-1, len(opts)) - } -} - -func TestOSSHeaderToS3Meta(t *testing.T) { - meta := map[string]string{ - "x-oss-meta-first_name": "myname", - "X-OSS-Meta-x_test_key": "value", - "X-Oss-Meta-x_test__key": "value", - "X-Oss-Meta-x__test__key": "value", - "X-Oss-Meta-x____test____key": "value", - "X-Oss-Meta-x_amz_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=", - "X-Oss-Meta-x_amz_matdesc": "{}", - "x-oss-meta-x_amz_iv": "eWmyryl8kq+EVnnsE7jpOg==", - } - header := make(http.Header) - for k, v := range meta { - header.Set(k, v) - } - - expectedMeta := map[string]string{ - "X-Amz-Meta-First-Name": "myname", - "X-Amz-Meta-X-Test-Key": "value", - "X-Amz-Meta-X-Test_key": "value", - "X-Amz-Meta-X_test_key": "value", - "X-Amz-Meta-X__test__key": "value", - "X-Amz-Meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=", - "X-Amz-Meta-X-Amz-Matdesc": "{}", - "X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==", - } - actualMeta := ossHeaderToS3Meta(header) - for k, v := range expectedMeta { - if v2, ok := actualMeta[k]; !ok { - t.Errorf("Test failed for key %s: missing key", k) - } else if v != v2 { - t.Errorf("Test failed for key %s, expected '%s', got '%s'", k, v, v2) - } - } -} - -func TestOSSBuildListObjectPartsParams(t *testing.T) { - expected := map[string]interface{}{ - "uploadId": "test", - "part-number-marker": "123", - "max-parts": "456", - } - actual := ossBuildListObjectPartsParams("test", 123, 456) - if !reflect.DeepEqual(expected, actual) { - t.Fatalf("Test failed, expected %v, got %v", expected, actual) - } -} diff --git a/docs/gateway/README.md b/docs/gateway/README.md index aee09ac98..9f4190f55 100644 --- a/docs/gateway/README.md +++ b/docs/gateway/README.md @@ -4,6 +4,5 @@ MinIO Gateway adds Amazon S3 compatibility to third party cloud storage provider - [Microsoft Azure Blob Storage](https://github.com/minio/minio/blob/master/docs/gateway/azure.md) - [S3](https://github.com/minio/minio/blob/master/docs/gateway/s3.md) - [Google Cloud Storage](https://github.com/minio/minio/blob/master/docs/gateway/gcs.md) -- [Alibaba Cloud Storage](https://github.com/minio/minio/blob/master/docs/gateway/oss.md) - [Backblaze B2](https://github.com/minio/minio/blob/master/docs/gateway/b2.md) diff --git a/docs/gateway/oss.md b/docs/gateway/oss.md deleted file mode 100644 index 58224d7eb..000000000 --- a/docs/gateway/oss.md +++ /dev/null @@ -1,57 +0,0 @@ -# MinIO OSS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) -MinIO Gateway adds Amazon S3 compatibility to Alibaba Cloud Object Storage Service (OSS). - -## Run MinIO Gateway for OSS. - -### Using Docker -``` -docker run -p 9000:9000 --name azure-s3 \ - -e "MINIO_ACCESS_KEY=ossaccesskey" \ - -e "MINIO_SECRET_KEY=osssecretkey" \ - minio/minio gateway oss -``` - -### Using Binary -``` -export MINIO_ACCESS_KEY=ossaccesskey -export MINIO_SECRET_KEY=osssecretkey -minio gateway oss -``` - -## Test using MinIO Browser -MinIO Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully. - -![Screenshot](https://raw.githubusercontent.com/minio/minio/master/docs/screenshots/minio-browser-gateway.png) - -## Test using MinIO Client `mc` -`mc` provides a modern alternative to UNIX commands such as ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. - -### Configure `mc` -``` -mc config host add myoss http://gateway-ip:9000 ossaccesskey osssecretkey -``` - -### List buckets on OSS -``` -mc ls myb2 -[2017-02-22 01:50:43 PST] 0B ferenginar/ -[2017-02-26 21:43:51 PST] 0B my-bucket/ -[2017-02-26 22:10:11 PST] 0B test-bucket1/ -``` - -### Known limitations - -Gateway inherits the following OSS limitations: - -- Bucket names with "." in the bucket name are not supported. -- Custom metadata with "_" in the key is not supported. - -Other limitations: - -- Bucket notification APIs are not supported. - -## Explore Further - -- [`mc` command-line interface](https://docs.min.io/docs/minio-client-quickstart-guide) -- [`aws` command-line interface](https://docs.min.io/docs/aws-cli-with-minio) -- [`minio-go` Go SDK](https://docs.min.io/docs/golang-client-quickstart-guide) diff --git a/go.mod b/go.mod index 8686d79cd..e29429f42 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,7 @@ require ( github.com/Shopify/sarama v1.24.1 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/alecthomas/participle v0.2.1 - github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5 github.com/aws/aws-sdk-go v1.20.21 - github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 github.com/beevik/ntp v0.2.0 github.com/cespare/xxhash/v2 v2.1.1 @@ -96,7 +94,6 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81 // indirect github.com/rjeczalik/notify v0.9.2 github.com/rs/cors v1.6.0 - github.com/satori/go.uuid v1.2.0 // indirect github.com/secure-io/sio-go v0.3.0 github.com/shirou/gopsutil v2.20.3-0.20200314133625-53cec6b37e6a+incompatible github.com/sirupsen/logrus v1.5.0 diff --git a/go.sum b/go.sum index 3d163222e..05df1e6c1 100644 --- a/go.sum +++ b/go.sum @@ -25,16 +25,12 @@ github.com/alecthomas/participle v0.2.1 h1:4AVLj1viSGa4LG5HDXKXrm5xRx19SB/rS/skP github.com/alecthomas/participle v0.2.1/go.mod h1:SW6HZGeZgSIpcUWX3fXpfZhuaWHnmoD5KCVaqSaNTkk= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5 h1:nWDRPCyCltiTsANwC/n3QZH7Vww33Npq9MKqlwRzI/c= -github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.20.21 h1:22vHWL9rur+SRTYPHAXlxJMFIA9OSYsYDIAHFDhQ7Z0= github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= -github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 h1:M+TYzBcNIRyzPRg66ndEqUMd7oWDmhvdQmaPC6EZNwM= github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M= github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw= @@ -385,8 +381,6 @@ github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/secure-io/sio-go v0.3.0 h1:QKGb6rGJeiExac9wSWxnWPYo8O8OFN7lxXQvHshX6vo= github.com/secure-io/sio-go v0.3.0/go.mod h1:D3KmXgKETffyYxBdFRN+Hpd2WzhzqS0EQwT3XWsAcBU= github.com/shirou/gopsutil v2.20.3-0.20200314133625-53cec6b37e6a+incompatible h1:YiKUe2ZOmfpDBH4OSyxwkx/mjNqHHnNhOtZ2mPyRme8=