diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index 9d3776b57..0de8a7886 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -148,6 +148,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) { initNSLock(false) // Enable local namespace lock. + // Set when gateway is enabled + globalIsGateway = true + router := mux.NewRouter().SkipClean(true) if globalEtcdClient != nil { @@ -304,9 +307,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) { printGatewayStartupMessage(getAPIEndpoints(), gatewayName) } - // Set when gateway is enabled - globalIsGateway = true - // Set uptime time after object layer has initialized. globalBootTime = UTCNow() diff --git a/cmd/gateway/s3/gateway-s3.go b/cmd/gateway/s3/gateway-s3.go index c9f040208..9fae4ba1d 100644 --- a/cmd/gateway/s3/gateway-s3.go +++ b/cmd/gateway/s3/gateway-s3.go @@ -169,14 +169,6 @@ func randString(n int, src rand.Source, prefix string) string { return prefix + string(b[0:30-len(prefix)]) } -func isAmazonS3Endpoint(urlStr string) bool { - u, err := url.Parse(urlStr) - if err != nil { - panic(err) - } - return s3utils.IsAmazonEndpoint(*u) -} - // Chains all credential types, in the following order: // - AWS env vars (i.e. AWS_ACCESS_KEY_ID) // - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) @@ -210,6 +202,11 @@ func newS3(urlStr string) (*miniogo.Core, error) { urlStr = "https://s3.amazonaws.com" } + u, err := url.Parse(urlStr) + if err != nil { + return nil, err + } + // Override default params if the host is provided endpoint, secure, err := minio.ParseGatewayEndpoint(urlStr) if err != nil { @@ -217,15 +214,23 @@ func newS3(urlStr string) (*miniogo.Core, error) { } var creds *credentials.Credentials - if isAmazonS3Endpoint(urlStr) { + if s3utils.IsAmazonEndpoint(*u) { // If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials. // Specifically IAM style rotating credentials are only supported with AWS S3 endpoint. creds = credentials.NewChainCredentials(defaultAWSCredProviders) + } else { creds = credentials.NewChainCredentials(defaultProviders) } - clnt, err := miniogo.NewWithCredentials(endpoint, creds, secure, "") + options := miniogo.Options{ + Creds: creds, + Secure: secure, + Region: s3utils.GetRegionFromURL(*u), + BucketLookup: miniogo.BucketLookupAuto, + } + + clnt, err := miniogo.NewWithOptions(endpoint, &options) if err != nil { return nil, err } @@ -234,9 +239,15 @@ func newS3(urlStr string) (*miniogo.Core, error) { clnt.SetCustomTransport(minio.NewCustomHTTPTransport()) probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-") + + // Set maximum retry operations. + miniogo.MaxRetry = 1 + // Check if the provided keys are valid. if _, err = clnt.BucketExists(probeBucketName); err != nil { - return nil, err + if miniogo.ToErrorResponse(err).Code != "AccessDenied" { + return nil, err + } } return &miniogo.Core{Client: clnt}, nil @@ -313,7 +324,19 @@ func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) { buckets, err := l.Client.ListBuckets() if err != nil { - return bi, minio.ErrorRespToObjectError(err, bucket) + // Listbuckets may be disallowed, proceed to check if + // bucket indeed exists, if yes return success. + var ok bool + if ok, err = l.Client.BucketExists(bucket); err != nil { + return bi, minio.ErrorRespToObjectError(err, bucket) + } + if !ok { + return bi, minio.BucketNotFound{Bucket: bucket} + } + return minio.BucketInfo{ + Name: bi.Name, + Created: time.Now().UTC(), + }, nil } for _, bi := range buckets { diff --git a/docs/gateway/s3.md b/docs/gateway/s3.md index 6ba98f0a9..5bea4cf19 100644 --- a/docs/gateway/s3.md +++ b/docs/gateway/s3.md @@ -34,6 +34,31 @@ MinIO gateway will automatically look for list of credential styles in following - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) - IAM profile based credentials. (performs an HTTP call to a pre-defined endpoint, only valid inside configured ec2 instances) +Minimum permissions required if you wish to provide restricted access with your AWS credentials, please make sure you have following IAM policies attached for your AWS user or roles. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "readonly", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::testbucket" + }, + { + "Sid": "readonly", + "Effect": "Allow", + "Action": "s3:HeadBucket", + "Resource": "arn:aws:s3:::testbucket" + } + ] +} +``` + ## Run MinIO Gateway for AWS S3 compatible services As a prerequisite to run MinIO S3 gateway on an AWS S3 compatible service, you need valid access key, secret key and service endpoint. diff --git a/go.mod b/go.mod index 594c07c6f..97d59a0e3 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/minio/highwayhash v1.0.0 github.com/minio/lsync v0.1.0 github.com/minio/mc v0.0.0-20190529152718-f4bb0b8850cb - github.com/minio/minio-go/v6 v6.0.27 + github.com/minio/minio-go/v6 v6.0.28 github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679 github.com/minio/sha256-simd v0.1.0 github.com/minio/sio v0.2.0 @@ -63,7 +63,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nats-io/gnatsd v1.4.1 // indirect - github.com/nats-io/go-nats-streaming v0.4.2 github.com/nats-io/nats v1.7.2 github.com/nats-io/nats-streaming-server v0.14.2 // indirect github.com/nats-io/nats.go v1.8.0 @@ -81,6 +80,7 @@ require ( github.com/streadway/amqp v0.0.0-20190402114354-16ed540749f6 github.com/tidwall/gjson v1.2.1 github.com/tidwall/match v1.0.1 // indirect + github.com/tidwall/pretty v1.0.0 // indirect github.com/tidwall/sjson v1.0.4 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a go.uber.org/atomic v1.3.2 diff --git a/go.sum b/go.sum index 097c607bd..7af4877f1 100644 --- a/go.sum +++ b/go.sum @@ -431,6 +431,8 @@ github.com/minio/minio-go/v6 v6.0.26 h1:nHLr1A+sJBv/sQu6zc5BrHLFAStCXxloC+jmZp4F github.com/minio/minio-go/v6 v6.0.26/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y= github.com/minio/minio-go/v6 v6.0.27 h1:UxEyhY10E9qog45XmUQyX4npPao0uwLHcoqZcF9gjOI= github.com/minio/minio-go/v6 v6.0.27/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y= +github.com/minio/minio-go/v6 v6.0.28 h1:RIsoxMaljP2euGbu2r0gBC0UNn70l2gzHjifU6DhE0c= +github.com/minio/minio-go/v6 v6.0.28/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y= github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679 h1:OMKaN/82sBHUZPvjYNBFituHExa1OGY63eACDGtetKs= github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679/go.mod h1:J+goXSuzlte5imWMqb6cUWC/tbYYysUHctwmKXomYzM= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -609,6 +611,8 @@ github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 h1:rQ229MBgvW68s1/g6f1/63TgYwYxfF4E+bi/KC19P8g= github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=