optimize Listen bucket notification implementation (#9444)

this commit avoids lots of tiny allocations, repeated
channel creates which are performed when filtering
the incoming events, unescaping a key just for matching.

also remove deprecated code which is not needed
anymore, avoids unexpected data structure transformations
from the map to slice.

cmd/bucket-notification-handlers.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"io"
 	"net/http"
-	"net/url"
 	"path"
 	"reflect"
 	"time"
@@ -296,11 +295,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
 			return false
 		}
-		objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
+		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
-		if uerr != nil {
-			objectName = ev.S3.Object.Key
-		}
-		return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0
 	})
 
 	for _, peer := range peers {

cmd/gateway-main.go

@@ -131,6 +131,14 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
 	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
 
+	globalMinioEndpoint = func() string {
+		host := globalMinioHost
+		if host == "" {
+			host = sortIPs(localIP4.ToSlice())[0]
+		}
+		return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
+	}()
+
 	// Handle gateway specific env
 	gatewayHandleEnvVars()
 

cmd/globals.go

@@ -138,6 +138,8 @@ var (
 	globalMinioPort = globalMinioDefaultPort
 	// Holds the host that was passed using --address
 	globalMinioHost = ""
+	// Holds the possible host endpoint.
+	globalMinioEndpoint = ""
 
 	// globalConfigSys server config system.
 	globalConfigSys *ConfigSys

cmd/notification.go

@@ -22,7 +22,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net"
 	"net/url"
 	"path"
 	"sort"
@@ -50,6 +49,7 @@ import (
 type NotificationSys struct {
 	sync.RWMutex
 	targetList                 *event.TargetList
+	targetResCh                chan event.TargetIDResult
 	bucketRulesMap             map[string]event.RulesMap
 	bucketRemoteTargetRulesMap map[string]map[event.TargetID]event.RulesMap
 	peerClients                []*peerRESTClient
@@ -662,19 +662,6 @@ func (sys *NotificationSys) AddRemoteTarget(bucketName string, target event.Targ
 	return nil
 }
 
-// RemoteTargetExist - checks whether given target ID is a HTTP/PeerRPC client target or not.
-func (sys *NotificationSys) RemoteTargetExist(bucketName string, targetID event.TargetID) bool {
-	sys.Lock()
-	defer sys.Unlock()
-
-	targetMap, ok := sys.bucketRemoteTargetRulesMap[bucketName]
-	if ok {
-		_, ok = targetMap[targetID]
-	}
-
-	return ok
-}
-
 // Loads notification policies for all buckets into NotificationSys.
 func (sys *NotificationSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
 	for _, bucket := range buckets {
@@ -713,6 +700,17 @@ func (sys *NotificationSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error
 		}
 	}
 
+	go func() {
+		for res := range sys.targetResCh {
+			if res.Err != nil {
+				reqInfo := &logger.ReqInfo{}
+				reqInfo.AppendTags("targetID", res.ID.Name)
+				ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+				logger.LogOnceIf(ctx, res.Err, res.ID)
+			}
+		}
+	}()
+
 	return sys.load(buckets, objAPI)
 }
 
@@ -759,7 +757,9 @@ func (sys *NotificationSys) ConfiguredTargetIDs() []event.TargetID {
 	for _, rmap := range sys.bucketRulesMap {
 		for _, rules := range rmap {
 			for _, targetSet := range rules {
-				targetIDs = append(targetIDs, targetSet.ToSlice()...)
+				for id := range targetSet {
+					targetIDs = append(targetIDs, id)
+				}
 			}
 		}
 	}
@@ -780,69 +780,41 @@ func (sys *NotificationSys) RemoveNotification(bucketName string) {
 
 	delete(sys.bucketRulesMap, bucketName)
 
+	targetIDSet := event.NewTargetIDSet()
 	for targetID := range sys.bucketRemoteTargetRulesMap[bucketName] {
-		sys.targetList.Remove(targetID)
+		targetIDSet[targetID] = struct{}{}
 		delete(sys.bucketRemoteTargetRulesMap[bucketName], targetID)
 	}
+	sys.targetList.Remove(targetIDSet)
 
 	delete(sys.bucketRemoteTargetRulesMap, bucketName)
 }
 
-// RemoveAllRemoteTargets - closes and removes all HTTP/PeerRPC client targets.
+// RemoveAllRemoteTargets - closes and removes all notification targets.
 func (sys *NotificationSys) RemoveAllRemoteTargets() {
 	sys.Lock()
 	defer sys.Unlock()
 
 	for _, targetMap := range sys.bucketRemoteTargetRulesMap {
-		for targetID := range targetMap {
+		targetIDSet := event.NewTargetIDSet()
-			sys.targetList.Remove(targetID)
+		for k := range targetMap {
-		}
+			targetIDSet[k] = struct{}{}
-	}
-}
-
-// RemoveRemoteTarget - closes and removes target by target ID.
-func (sys *NotificationSys) RemoveRemoteTarget(bucketName string, targetID event.TargetID) {
-	for terr := range sys.targetList.Remove(targetID) {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("targetID", terr.ID.Name)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-		logger.LogIf(ctx, terr.Err)
-	}
-
-	sys.Lock()
-	defer sys.Unlock()
-
-	if _, ok := sys.bucketRemoteTargetRulesMap[bucketName]; ok {
-		delete(sys.bucketRemoteTargetRulesMap[bucketName], targetID)
-		if len(sys.bucketRemoteTargetRulesMap[bucketName]) == 0 {
-			delete(sys.bucketRemoteTargetRulesMap, bucketName)
-		}
-	}
-}
-
-func (sys *NotificationSys) send(bucketName string, eventData event.Event, targetIDs ...event.TargetID) (errs []event.TargetIDErr) {
-	errCh := sys.targetList.Send(eventData, targetIDs...)
-	for terr := range errCh {
-		errs = append(errs, terr)
-		if sys.RemoteTargetExist(bucketName, terr.ID) {
-			sys.RemoveRemoteTarget(bucketName, terr.ID)
 		}
+		sys.targetList.Remove(targetIDSet)
 	}
-
-	return errs
 }
 
 // Send - sends event data to all matching targets.
-func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
+func (sys *NotificationSys) Send(args eventArgs) {
 	sys.RLock()
 	targetIDSet := sys.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name)
 	sys.RUnlock()
 
 	if len(targetIDSet) == 0 {
-		return nil
+		return
 	}
 
-	targetIDs := targetIDSet.ToSlice()
+	sys.targetList.Send(args.ToEvent(true), targetIDSet, sys.targetResCh)
-	return sys.send(args.BucketName, args.ToEvent(), targetIDs...)
 }
 
 // PutBucketObjectLockConfig - put bucket object lock configuration to all peers.
@@ -1204,6 +1176,7 @@ func NewNotificationSys(endpoints EndpointZones) *NotificationSys {
 	// bucketRulesMap/bucketRemoteTargetRulesMap are initialized by NotificationSys.Init()
 	return &NotificationSys{
 		targetList:                 event.NewTargetList(),
+		targetResCh:                make(chan event.TargetIDResult),
 		bucketRulesMap:             make(map[string]event.RulesMap),
 		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
 		peerClients:                newPeerRestClients(endpoints),
@@ -1221,23 +1194,13 @@ type eventArgs struct {
 }
 
 // ToEvent - converts to notification event.
-func (args eventArgs) ToEvent() event.Event {
+func (args eventArgs) ToEvent(escape bool) event.Event {
-	getOriginEndpoint := func() string {
-		host := globalMinioHost
-		if host == "" {
-			// FIXME: Send FQDN or hostname of this machine than sending IP address.
-			host = sortIPs(localIP4.ToSlice())[0]
-		}
-
-		return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
-	}
-
 	eventTime := UTCNow()
 	uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
 
 	respElements := map[string]string{
 		"x-amz-request-id":        args.RespElements["requestId"],
-		"x-minio-origin-endpoint": getOriginEndpoint(), // MinIO specific custom elements.
+		"x-minio-origin-endpoint": globalMinioEndpoint, // MinIO specific custom elements.
 	}
 	// Add deployment as part of
 	if globalDeploymentID != "" {
@@ -1246,6 +1209,10 @@ func (args eventArgs) ToEvent() event.Event {
 	if args.RespElements["content-length"] != "" {
 		respElements["content-length"] = args.RespElements["content-length"]
 	}
+	keyName := args.Object.Name
+	if escape {
+		keyName = url.QueryEscape(args.Object.Name)
+	}
 	newEvent := event.Event{
 		EventVersion:      "2.0",
 		EventSource:       "minio:s3",
@@ -1264,7 +1231,7 @@ func (args eventArgs) ToEvent() event.Event {
 				ARN:           policy.ResourceARNPrefix + args.BucketName,
 			},
 			Object: event.Object{
-				Key:       url.QueryEscape(args.Object.Name),
+				Key:       keyName,
 				VersionID: "1",
 				Sequencer: uniqueID,
 			},
@@ -1308,19 +1275,10 @@ func sendEvent(args eventArgs) {
 	}
 
 	if globalHTTPListen.HasSubscribers() {
-		globalHTTPListen.Publish(args.ToEvent())
+		globalHTTPListen.Publish(args.ToEvent(false))
 	}
 
-	notifyCh := globalNotificationSys.Send(args)
+	globalNotificationSys.Send(args)
-	go func() {
-		for _, err := range notifyCh {
-			reqInfo := &logger.ReqInfo{BucketName: args.BucketName, ObjectName: args.Object.Name}
-			reqInfo.AppendTags("EventName", args.EventName.String())
-			reqInfo.AppendTags("targetID", err.ID.Name)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogOnceIf(ctx, err.Err, err.ID)
-		}
-	}()
 }
 
 func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) (*event.Config, error) {

cmd/peer-rest-client-target.go

@@ -1,72 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018, 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import "github.com/minio/minio/pkg/event"
-
-// PeerRESTClientTarget - RPCClient is an event.Target which sends event to target of remote peer.
-type PeerRESTClientTarget struct {
-	id             event.TargetID
-	remoteTargetID event.TargetID
-	restClient     *peerRESTClient
-	bucketName     string
-}
-
-// ID - returns target ID.
-func (target *PeerRESTClientTarget) ID() event.TargetID {
-	return target.id
-}
-
-// IsActive - does nothing and available for interface compatibility.
-func (target *PeerRESTClientTarget) IsActive() (bool, error) {
-	return true, nil
-}
-
-// HasQueueStore - No-Op. Added for interface compatibility
-func (target PeerRESTClientTarget) HasQueueStore() bool {
-	return false
-}
-
-// Save - Sends event directly without persisting.
-func (target *PeerRESTClientTarget) Save(eventData event.Event) error {
-	return target.send(eventData)
-}
-
-// Send - interface compatible method does no-op.
-func (target *PeerRESTClientTarget) Send(eventKey string) error {
-	return nil
-}
-
-// sends event to remote peer by making RPC call.
-func (target *PeerRESTClientTarget) send(eventData event.Event) error {
-	return target.restClient.SendEvent(target.bucketName, target.id, target.remoteTargetID, eventData)
-}
-
-// Close - does nothing and available for interface compatibility.
-func (target *PeerRESTClientTarget) Close() error {
-	return nil
-}
-
-// NewPeerRESTClientTarget - creates RPCClient target with given target ID available in remote peer.
-func NewPeerRESTClientTarget(bucketName string, targetID event.TargetID, restClient *peerRESTClient) *PeerRESTClientTarget {
-	return &PeerRESTClientTarget{
-		id:             event.TargetID{ID: targetID.ID, Name: targetID.Name + "+" + mustGetUUID()},
-		remoteTargetID: targetID,
-		bucketName:     bucketName,
-		restClient:     restClient,
-	}
-}

cmd/peer-rest-client.go

@@ -500,78 +500,6 @@ func (client *peerRESTClient) ReloadFormat(dryRun bool) error {
 	return nil
 }
 
-// SendEvent - calls send event RPC.
-func (client *peerRESTClient) SendEvent(bucket string, targetID, remoteTargetID event.TargetID, eventData event.Event) error {
-	numTries := 10
-	for {
-		err := client.sendEvent(bucket, targetID, remoteTargetID, eventData)
-		if err == nil {
-			return nil
-		}
-		if numTries == 0 {
-			return err
-		}
-		numTries--
-		time.Sleep(5 * time.Second)
-	}
-}
-
-func (client *peerRESTClient) sendEvent(bucket string, targetID, remoteTargetID event.TargetID, eventData event.Event) error {
-	args := sendEventRequest{
-		TargetID: remoteTargetID,
-		Event:    eventData,
-	}
-
-	values := make(url.Values)
-	values.Set(peerRESTBucket, bucket)
-
-	var reader bytes.Buffer
-	err := gob.NewEncoder(&reader).Encode(args)
-	if err != nil {
-		return err
-	}
-	respBody, err := client.call(peerRESTMethodSendEvent, values, &reader, -1)
-	if err != nil {
-		return err
-	}
-
-	var eventResp sendEventResp
-	defer http.DrainBody(respBody)
-	err = gob.NewDecoder(respBody).Decode(&eventResp)
-
-	if err != nil || !eventResp.Success {
-		reqInfo := &logger.ReqInfo{BucketName: bucket}
-		reqInfo.AppendTags("targetID", targetID.Name)
-		reqInfo.AppendTags("event", eventData.EventName.String())
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-		logger.LogIf(ctx, err)
-		globalNotificationSys.RemoveRemoteTarget(bucket, targetID)
-	}
-
-	return err
-}
-
-// RemoteTargetExist - calls remote target ID exist REST API.
-func (client *peerRESTClient) RemoteTargetExist(bucket string, targetID event.TargetID) (bool, error) {
-	values := make(url.Values)
-	values.Set(peerRESTBucket, bucket)
-
-	var reader bytes.Buffer
-	err := gob.NewEncoder(&reader).Encode(targetID)
-	if err != nil {
-		return false, err
-	}
-
-	respBody, err := client.call(peerRESTMethodTargetExists, values, &reader, -1)
-	if err != nil {
-		return false, err
-	}
-	defer http.DrainBody(respBody)
-	var targetExists remoteTargetExistsResp
-	err = gob.NewDecoder(respBody).Decode(&targetExists)
-	return targetExists.Exists, err
-}
-
 // RemoveBucketPolicy - Remove bucket policy on the peer node.
 func (client *peerRESTClient) RemoveBucketPolicy(bucket string) error {
 	values := make(url.Values)

cmd/peer-rest-common.go

@@ -17,7 +17,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v7"
+	peerRESTVersion       = "v8"
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -52,17 +52,14 @@ const (
 	peerRESTMethodDownloadProfilingData        = "/downloadprofilingdata"
 	peerRESTMethodBucketPolicySet              = "/setbucketpolicy"
 	peerRESTMethodBucketNotificationPut        = "/putbucketnotification"
-	peerRESTMethodBucketNotificationListen     = "/listenbucketnotification"
 	peerRESTMethodReloadFormat                 = "/reloadformat"
-	peerRESTMethodTargetExists                 = "/targetexists"
-	peerRESTMethodSendEvent                    = "/sendevent"
 	peerRESTMethodTrace                        = "/trace"
 	peerRESTMethodListen                       = "/listen"
+	peerRESTMethodLog                          = "/log"
 	peerRESTMethodBucketLifecycleSet           = "/setbucketlifecycle"
 	peerRESTMethodBucketLifecycleRemove        = "/removebucketlifecycle"
 	peerRESTMethodBucketEncryptionSet          = "/setbucketencryption"
 	peerRESTMethodBucketEncryptionRemove       = "/removebucketencryption"
-	peerRESTMethodLog                          = "/log"
 	peerRESTMethodPutBucketObjectLockConfig    = "/putbucketobjectlockconfig"
 	peerRESTMethodBucketObjectLockConfigRemove = "/removebucketobjectlockconfig"
 )

cmd/peer-rest-server.go

@@ -24,7 +24,6 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -774,97 +773,6 @@ func (s *peerRESTServer) SetBucketSSEConfigHandler(w http.ResponseWriter, r *htt
 	w.(http.Flusher).Flush()
 }
 
-type remoteTargetExistsResp struct {
-	Exists bool
-}
-
-// TargetExistsHandler - Check if Target exists.
-func (s *peerRESTServer) TargetExistsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "TargetExists")
-	if !s.IsValid(w, r) {
-		s.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	vars := mux.Vars(r)
-	bucketName := vars[peerRESTBucket]
-	if bucketName == "" {
-		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
-		return
-	}
-	var targetID event.TargetID
-	if r.ContentLength <= 0 {
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
-	}
-
-	err := gob.NewDecoder(r.Body).Decode(&targetID)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	var targetExists remoteTargetExistsResp
-	targetExists.Exists = globalNotificationSys.RemoteTargetExist(bucketName, targetID)
-
-	defer w.(http.Flusher).Flush()
-	logger.LogIf(ctx, gob.NewEncoder(w).Encode(&targetExists))
-}
-
-type sendEventRequest struct {
-	Event    event.Event
-	TargetID event.TargetID
-}
-
-type sendEventResp struct {
-	Success bool
-}
-
-// SendEventHandler - Send Event.
-func (s *peerRESTServer) SendEventHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		s.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	ctx := newContext(r, w, "SendEvent")
-
-	vars := mux.Vars(r)
-	bucketName := vars[peerRESTBucket]
-	if bucketName == "" {
-		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
-		return
-	}
-	var eventReq sendEventRequest
-	if r.ContentLength <= 0 {
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
-	}
-
-	err := gob.NewDecoder(r.Body).Decode(&eventReq)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	var eventResp sendEventResp
-	eventResp.Success = true
-	errs := globalNotificationSys.send(bucketName, eventReq.Event, eventReq.TargetID)
-
-	for i := range errs {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("Event", eventReq.Event.EventName.String())
-		reqInfo.AppendTags("targetName", eventReq.TargetID.Name)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-		logger.LogIf(ctx, errs[i].Err)
-
-		eventResp.Success = false
-		s.writeErrorResponse(w, errs[i].Err)
-		return
-	}
-	logger.LogIf(ctx, gob.NewEncoder(w).Encode(&eventResp))
-	w.(http.Flusher).Flush()
-}
-
 // PutBucketNotificationHandler - Set bucket policy.
 func (s *peerRESTServer) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -1077,11 +985,7 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 		if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
 			return false
 		}
-		objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
+		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
-		if uerr != nil {
-			objectName = ev.S3.Object.Key
-		}
-		return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0
 	})
 
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
@@ -1239,8 +1143,6 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(httpTraceAll(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(httpTraceHdrs(server.DownloadProfilingDataHandler))
 
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodTargetExists).HandlerFunc(httpTraceHdrs(server.TargetExistsHandler)).Queries(restQueries(peerRESTBucket)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSendEvent).HandlerFunc(httpTraceHdrs(server.SendEventHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBucketNotificationPut).HandlerFunc(httpTraceHdrs(server.PutBucketNotificationHandler)).Queries(restQueries(peerRESTBucket)...)
 
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)

cmd/server-main.go

@@ -111,7 +111,6 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	globalMinioAddr = globalCLIContext.Addr
 
 	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
-
 	endpoints := strings.Fields(env.Get(config.EnvEndpoints, ""))
 	if len(endpoints) > 0 {
 		globalEndpoints, globalXLSetDriveCount, setupType, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
@@ -344,6 +343,14 @@ func serverMain(ctx *cli.Context) {
 	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
 	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
 
+	globalMinioEndpoint = func() string {
+		host := globalMinioHost
+		if host == "" {
+			host = sortIPs(localIP4.ToSlice())[0]
+		}
+		return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
+	}()
+
 	// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
 	if globalIsDistXL {
 		if globalEndpoints.HTTPS() && !globalIsSSL {

go.mod

@@ -67,7 +67,7 @@ require (
 	github.com/minio/hdfs/v3 v3.0.1
 	github.com/minio/highwayhash v1.0.0
 	github.com/minio/lsync v1.0.1
-	github.com/minio/minio-go/v6 v6.0.53
+	github.com/minio/minio-go/v6 v6.0.55-0.20200424204115-7506d2996b22
 	github.com/minio/parquet-go v0.0.0-20200414234858-838cfa8aae61
 	github.com/minio/sha256-simd v0.1.1
 	github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37

go.sum

@@ -276,6 +276,8 @@ github.com/minio/lsync v1.0.1 h1:AVvILxA976xc27hstd1oR+X9PQG0sPSom1MNb1ImfUs=
 github.com/minio/lsync v1.0.1/go.mod h1:tCFzfo0dlvdGl70IT4IAK/5Wtgb0/BrTmo/jE8pArKA=
 github.com/minio/minio-go/v6 v6.0.53 h1:8jzpwiOzZ5Iz7/goFWqNZRICbyWYShbb5rARjrnSCNI=
 github.com/minio/minio-go/v6 v6.0.53/go.mod h1:DIvC/IApeHX8q1BAMVCXSXwpmrmM+I+iBvhvztQorfI=
+github.com/minio/minio-go/v6 v6.0.55-0.20200424204115-7506d2996b22 h1:nZEve4vdUhwHBoV18zRvPDgjL6NYyDJE5QJvz3l9bRs=
+github.com/minio/minio-go/v6 v6.0.55-0.20200424204115-7506d2996b22/go.mod h1:KQMM+/44DSlSGSQWSfRrAZ12FVMmpWNuX37i2AX0jfI=
 github.com/minio/parquet-go v0.0.0-20200414234858-838cfa8aae61 h1:pUSI/WKPdd77gcuoJkSzhJ4wdS8OMDOsOu99MtpXEQA=
 github.com/minio/parquet-go v0.0.0-20200414234858-838cfa8aae61/go.mod h1:4trzEJ7N1nBTd5Tt7OCZT5SEin+WiAXpdJ/WgPkESA8=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=

pkg/event/rules.go

@@ -53,6 +53,16 @@ func (rules Rules) Add(pattern string, targetID TargetID) {
 	rules[pattern] = NewTargetIDSet(targetID).Union(rules[pattern])
 }
 
+// MatchSimple - returns true one of the matching object name in rules.
+func (rules Rules) MatchSimple(objectName string) bool {
+	for pattern := range rules {
+		if wildcard.MatchSimple(pattern, objectName) {
+			return true
+		}
+	}
+	return false
+}
+
 // Match - returns TargetIDSet matching object name in rules.
 func (rules Rules) Match(objectName string) TargetIDSet {
 	targetIDs := NewTargetIDSet()

pkg/event/rulesmap.go

@@ -60,6 +60,11 @@ func (rulesMap RulesMap) Remove(rulesMap2 RulesMap) {
 	}
 }
 
+// MatchSimple - returns true if matching object name and event name in rules map.
+func (rulesMap RulesMap) MatchSimple(eventName Name, objectName string) bool {
+	return rulesMap[eventName].MatchSimple(objectName)
+}
+
 // Match - returns TargetIDSet matching object name and event name in rules map.
 func (rulesMap RulesMap) Match(eventName Name, objectName string) TargetIDSet {
 	return rulesMap[eventName].Match(objectName)

pkg/event/targetidset.go

@@ -16,23 +16,12 @@
 
 package event
 
-import "fmt"
-
 // TargetIDSet - Set representation of TargetIDs.
 type TargetIDSet map[TargetID]struct{}
 
-// ToSlice - returns TargetID slice from TargetIDSet.
+// IsEmpty returns true if the set is empty.
-func (set TargetIDSet) ToSlice() []TargetID {
+func (set TargetIDSet) IsEmpty() bool {
-	keys := make([]TargetID, 0, len(set))
+	return len(set) != 0
-	for k := range set {
-		keys = append(keys, k)
-	}
-	return keys
-}
-
-// String - returns string representation.
-func (set TargetIDSet) String() string {
-	return fmt.Sprintf("%v", set.ToSlice())
 }
 
 // Clone - returns copy of this set.

pkg/event/targetidset_test.go

@@ -21,56 +21,6 @@ import (
 	"testing"
 )
 
-func TestTargetIDSetToSlice(t *testing.T) {
-	testCases := []struct {
-		set            TargetIDSet
-		expectedResult []TargetID
-	}{
-		{NewTargetIDSet(), []TargetID{}},
-		{NewTargetIDSet(TargetID{"1", "webhook"}), []TargetID{{"1", "webhook"}}},
-		{NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"}), []TargetID{{"1", "webhook"}, {"2", "amqp"}}},
-	}
-
-	for i, testCase := range testCases {
-		result := testCase.set.ToSlice()
-
-		if len(result) != len(testCase.expectedResult) {
-			t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
-		}
-
-		for _, targetID1 := range result {
-			var found bool
-			for _, targetID2 := range testCase.expectedResult {
-				if reflect.DeepEqual(targetID1, targetID2) {
-					found = true
-					break
-				}
-			}
-			if !found {
-				t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result)
-			}
-		}
-	}
-}
-
-func TestTargetIDSetString(t *testing.T) {
-	testCases := []struct {
-		set            TargetIDSet
-		expectedResult string
-	}{
-		{NewTargetIDSet(), "[]"},
-		{NewTargetIDSet(TargetID{"1", "webhook"}), "[1:webhook]"},
-	}
-
-	for i, testCase := range testCases {
-		result := testCase.set.String()
-
-		if result != testCase.expectedResult {
-			t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
-		}
-	}
-}
-
 func TestTargetIDSetClone(t *testing.T) {
 	testCases := []struct {
 		set           TargetIDSet

pkg/event/targetlist.go

@@ -61,8 +61,9 @@ func (list *TargetList) Exists(id TargetID) bool {
 	return found
 }
 
-// TargetIDErr returns error associated for a targetID
+// TargetIDResult returns result of Remove/Send operation, sets err if
-type TargetIDErr struct {
+// any for the associated TargetID
+type TargetIDResult struct {
 	// ID where the remove or send were initiated.
 	ID TargetID
 	// Stores any error while removing a target or while sending an event.
@@ -70,40 +71,17 @@ type TargetIDErr struct {
 }
 
 // Remove - closes and removes targets by given target IDs.
-func (list *TargetList) Remove(targetids ...TargetID) <-chan TargetIDErr {
+func (list *TargetList) Remove(targetIDSet TargetIDSet) {
-	errCh := make(chan TargetIDErr)
+	list.Lock()
-
+	defer list.Unlock()
-	go func() {
-		defer close(errCh)
-
-		var wg sync.WaitGroup
-		for _, id := range targetids {
-			list.RLock()
-			target, ok := list.targets[id]
-			list.RUnlock()
-			if ok {
-				wg.Add(1)
-				go func(id TargetID, target Target) {
-					defer wg.Done()
-					if err := target.Close(); err != nil {
-						errCh <- TargetIDErr{
-							ID:  id,
-							Err: err,
-						}
-					}
-				}(id, target)
-			}
-		}
-		wg.Wait()
 
-		list.Lock()
+	for id := range targetIDSet {
-		for _, id := range targetids {
+		target, ok := list.targets[id]
+		if ok {
+			target.Close()
 			delete(list.targets, id)
 		}
-		list.Unlock()
+	}
-	}()
-
-	return errCh
 }
 
 // Targets - list all targets
@@ -140,14 +118,10 @@ func (list *TargetList) TargetMap() map[TargetID]Target {
 }
 
 // Send - sends events to targets identified by target IDs.
-func (list *TargetList) Send(event Event, targetIDs ...TargetID) <-chan TargetIDErr {
+func (list *TargetList) Send(event Event, targetIDset TargetIDSet, resCh chan<- TargetIDResult) {
-	errCh := make(chan TargetIDErr)
-
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
-		for _, id := range targetIDs {
+		for id := range targetIDset {
 			list.RLock()
 			target, ok := list.targets[id]
 			list.RUnlock()
@@ -155,19 +129,18 @@ func (list *TargetList) Send(event Event, targetIDs ...TargetID) <-chan TargetID
 				wg.Add(1)
 				go func(id TargetID, target Target) {
 					defer wg.Done()
+					tgtRes := TargetIDResult{ID: id}
 					if err := target.Save(event); err != nil {
-						errCh <- TargetIDErr{
+						tgtRes.Err = err
-							ID:  id,
-							Err: err,
-						}
 					}
+					resCh <- tgtRes
 				}(id, target)
+			} else {
+				resCh <- TargetIDResult{ID: id}
 			}
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }
 
 // NewTargetList - creates TargetList.

pkg/event/targetlist_test.go

@@ -163,40 +163,6 @@ func TestTargetListExists(t *testing.T) {
 	}
 }
 
-func TestTargetListRemove(t *testing.T) {
-	targetListCase1 := NewTargetList()
-
-	targetListCase2 := NewTargetList()
-	if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil {
-		panic(err)
-	}
-
-	targetListCase3 := NewTargetList()
-	if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, true}); err != nil {
-		panic(err)
-	}
-
-	testCases := []struct {
-		targetList *TargetList
-		targetID   TargetID
-		expectErr  bool
-	}{
-		{targetListCase1, TargetID{"1", "webhook"}, false},
-		{targetListCase2, TargetID{"1", "webhook"}, false},
-		{targetListCase3, TargetID{"3", "testcase"}, true},
-	}
-
-	for i, testCase := range testCases {
-		errCh := testCase.targetList.Remove(testCase.targetID)
-		err := <-errCh
-		expectErr := (err.Err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
 func TestTargetListList(t *testing.T) {
 	targetListCase1 := NewTargetList()
 
@@ -273,10 +239,13 @@ func TestTargetListSend(t *testing.T) {
 		{targetListCase4, TargetID{"4", "testcase"}, true},
 	}
 
+	resCh := make(chan TargetIDResult)
 	for i, testCase := range testCases {
-		errCh := testCase.targetList.Send(Event{}, testCase.targetID)
+		testCase.targetList.Send(Event{}, map[TargetID]struct{}{
-		err := <-errCh
+			testCase.targetID: {},
-		expectErr := (err.Err != nil)
+		}, resCh)
+		res := <-resCh
+		expectErr := (res.Err != nil)
 
 		if expectErr != testCase.expectErr {
 			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)