From 669c9da85d229755a33217b4834cdd3a311a6495 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 29 Dec 2019 08:56:45 -0800 Subject: [PATCH] Disable federated buckets when etcd is namespaced (#8709) This is to ensure that when we have multiple tenants deployed all sharing the same etcd for global bucket should avoid listing each others buckets, this leads to information leak which should be avoided unless etcd is not namespaced for IAM assets in which case it can be assumed that its a federated setup. Federated setup and namespaced IAM assets on etcd is not supported since namespacing is only useful when you wish to separate the tenants as isolated instances of MinIO. This PR allows a new type of behavior, primarily driven by the usecase of m3(mkube) multi-tenant deployments with global bucket support. --- cmd/bucket-handlers.go | 2 +- cmd/config-current.go | 7 +++++++ cmd/generic-handlers.go | 3 ++- cmd/globals.go | 4 ++++ cmd/object-handlers.go | 8 ++++++-- cmd/web-handlers.go | 6 ++---- docs/config/README.md | 32 ++++++++++++++++++++++++++++++++ 7 files changed, 54 insertions(+), 8 deletions(-) diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 3e738d3c7..a8311beaa 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -278,7 +278,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R // If etcd, dns federation configured list buckets from etcd. var bucketsInfo []BucketInfo - if globalDNSConfig != nil { + if globalDNSConfig != nil && globalBucketFederation { dnsBuckets, err := globalDNSConfig.List() if err != nil && err != dns.ErrNoEntriesFound { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) diff --git a/cmd/config-current.go b/cmd/config-current.go index 7594d9f6b..afdd8b7c0 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -315,6 +315,13 @@ func lookupConfigs(s config.Config) { } } + // Bucket federation is 'true' only when IAM assets are not namespaced + // per tenant and all tenants interested in globally available users + // if namespace was requested such as specifying etcdPathPrefix then + // we assume that users are interested in global bucket support + // but not federation. + globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled + if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil { globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config, dns.DomainNames(globalDomainNames), diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 8c7ecc4b8..49a82ef06 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -619,7 +619,8 @@ type bucketForwardingHandler struct { func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if globalDNSConfig == nil || len(globalDomainNames) == 0 || guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || - guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) { + guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) || + !globalBucketFederation { f.handler.ServeHTTP(w, r) return } diff --git a/cmd/globals.go b/cmd/globals.go index 9740a963c..1357c4f37 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -212,6 +212,10 @@ var ( // Allocated etcd endpoint for config and bucket DNS. globalEtcdClient *etcd.Client + // Is set to true when Bucket federation is requested + // and is 'true' when etcdConfig.PathPrefix is empty + globalBucketFederation bool + // Allocated DNS config wrapper over etcd client. globalDNSConfig *dns.CoreDNS diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index f92394c7b..e81cc5787 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -640,8 +640,11 @@ func isRemoteCallRequired(ctx context.Context, bucket string, objAPI ObjectLayer if globalDNSConfig == nil { return false } - _, err := objAPI.GetBucketInfo(ctx, bucket) - return err == toObjectErr(errVolumeNotFound, bucket) + if globalBucketFederation { + _, err := objAPI.GetBucketInfo(ctx, bucket) + return err == toObjectErr(errVolumeNotFound, bucket) + } + return false } // CopyObjectHandler - Copy Object @@ -2421,6 +2424,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. writeErrorResponse(ctx, w, errorCodes.ToAPIErr(err), r.URL, guessIsBrowserReq(r)) return } + if globalDNSConfig != nil { _, err := globalDNSConfig.Get(bucket) if err != nil { diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 343e74925..4ef80a6d7 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -258,10 +258,6 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs, return toJSONError(ctx, err, args.BucketName) } - globalNotificationSys.RemoveNotification(args.BucketName) - globalPolicySys.Remove(args.BucketName) - globalNotificationSys.DeleteBucket(ctx, args.BucketName) - if globalDNSConfig != nil { if err := globalDNSConfig.Delete(args.BucketName); err != nil { // Deleting DNS entry failed, attempt to create the bucket again. @@ -270,6 +266,8 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs, } } + globalNotificationSys.DeleteBucket(ctx, args.BucketName) + return nil } diff --git a/docs/config/README.md b/docs/config/README.md index debb9bf63..1eca25351 100644 --- a/docs/config/README.md +++ b/docs/config/README.md @@ -137,6 +137,38 @@ MINIO_CACHE_EXCLUDE (csv) comma separated wildcard exclusion patterns e.g MINIO_CACHE_COMMENT (sentence) optionally add a comment to this setting ``` +#### Etcd +MinIO supports storing encrypted IAM assets and bucket DNS records on etcd. + +> NOTE: if *path_prefix* is set then MinIO will not federate your buckets, namespaced IAM assets are assumed as isolated tenants, only buckets are considered globally unique but performing a lookup with a *bucket* which belongs to a different tenant will fail unlike federated setups where MinIO would port-forward and route the request to relevant cluster accordingly. This is a special feature, federated deployments should not need to set *path_prefix*. + +``` +KEY: +etcd federate multiple clusters for IAM and Bucket DNS + +ARGS: +endpoints* (csv) comma separated list of etcd endpoints e.g. "http://localhost:2379" +path_prefix (path) namespace prefix to isolate tenants e.g. "customer1/" +coredns_path (path) shared bucket DNS records, default is "/skydns" +client_cert (path) client cert for mTLS authentication +client_cert_key (path) client cert key for mTLS authentication +comment (sentence) optionally add a comment to this setting +``` + +or environment variables +``` +KEY: +etcd federate multiple clusters for IAM and Bucket DNS + +ARGS: +MINIO_ETCD_ENDPOINTS* (csv) comma separated list of etcd endpoints e.g. "http://localhost:2379" +MINIO_ETCD_PATH_PREFIX (path) namespace prefix to isolate tenants e.g. "customer1/" +MINIO_ETCD_COREDNS_PATH (path) shared bucket DNS records, default is "/skydns" +MINIO_ETCD_CLIENT_CERT (path) client cert for mTLS authentication +MINIO_ETCD_CLIENT_CERT_KEY (path) client cert key for mTLS authentication +MINIO_ETCD_COMMENT (sentence) optionally add a comment to this setting +``` + #### Notifications Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://docs.min.io/docs/minio-bucket-notification-guide.html)