You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 
minio/cmd/s3-peer-client.go

230 lines
6.3 KiB

/*
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"encoding/json"
"fmt"
"net/rpc"
"path"
"sync"
"time"
"github.com/minio/minio-go/pkg/set"
)
type s3Peers struct {
// A map of peer server address (in `host:port` format) to RPC
// client connections.
rpcClients map[string]*AuthRPCClient
mutex *sync.RWMutex
// Slice of all peer addresses (in `host:port` format).
peers []string
}
func initGlobalS3Peers(disks []string) {
// Get list of de-duplicated peers.
peers := getAllPeers(disks)
// Initialize global state.
globalS3Peers = s3Peers{
rpcClients: make(map[string]*AuthRPCClient),
mutex: &sync.RWMutex{},
}
// Initialize each peer connection.
for _, peer := range peers {
globalS3Peers.InitS3PeerClient(peer)
}
// Additionally setup a local peer if one does not exist.
if globalS3Peers.GetPeerClient(globalMinioAddr) == nil {
globalS3Peers.InitS3PeerClient(globalMinioAddr)
peers = append(peers, globalMinioAddr)
}
globalS3Peers.peers = peers
}
func (s3p *s3Peers) GetPeers() []string {
return s3p.peers
}
func (s3p *s3Peers) GetPeerClient(peer string) *AuthRPCClient {
// Take a read lock
s3p.mutex.RLock()
defer s3p.mutex.RUnlock()
return s3p.rpcClients[peer]
}
// Initializes a new RPC connection (or closes and re-opens if it
// already exists) to a peer. Note that peer address is in `host:port`
// format.
func (s3p *s3Peers) InitS3PeerClient(peer string) {
// Take a write lock
s3p.mutex.Lock()
defer s3p.mutex.Unlock()
if s3p.rpcClients[peer] != nil {
_ = s3p.rpcClients[peer].Close()
delete(s3p.rpcClients, peer)
}
authCfg := &authConfig{
accessKey: serverConfig.GetCredential().AccessKeyID,
secretKey: serverConfig.GetCredential().SecretAccessKey,
address: peer,
secureConn: isSSL(),
path: path.Join(reservedBucket, s3Path),
loginMethod: "S3.LoginHandler",
}
s3p.rpcClients[peer] = newAuthClient(authCfg)
}
func (s3p *s3Peers) Close() error {
// Take a write lock
s3p.mutex.Lock()
defer s3p.mutex.Unlock()
for _, v := range s3p.rpcClients {
if err := v.Close(); err != nil {
return err
}
}
s3p.rpcClients = nil
s3p.peers = nil
return nil
}
// returns the network addresses of all Minio servers in the cluster
// in `host:port` format.
func getAllPeers(disks []string) []string {
res := []string{}
// use set to de-duplicate
sset := set.NewStringSet()
for _, disk := range disks {
netAddr, _, err := splitNetPath(disk)
if err != nil || netAddr == "" {
errorIf(err, "Unexpected error - most likely a bug.")
continue
}
if !sset.Contains(netAddr) {
res = append(
res,
fmt.Sprintf("%s:%d", netAddr, globalMinioPort),
)
sset.Add(netAddr)
}
}
return res
}
// Make RPC calls with the given method and arguments to all the given
// peers (in parallel), and collects the results. Since the methods
// intended for use here, have only a success or failure response, we
// do not return/inspect the `reply` parameter in the RPC call. The
// function attempts to connect to a peer only once, and returns a map
// of peer address to error response. If the error is nil, it means
// the RPC succeeded.
func (s3p *s3Peers) SendRPC(peers []string, method string, args interface {
SetToken(token string)
SetTimestamp(tstamp time.Time)
}) map[string]error {
// peer error responses array
errArr := make([]error, len(peers))
// Start a wait group and make RPC requests to peers.
var wg sync.WaitGroup
for i, target := range peers {
wg.Add(1)
go func(ix int, target string) {
defer wg.Done()
reply := &GenericReply{}
// Get RPC client object safely.
client := s3p.GetPeerClient(target)
var err error
if client == nil {
err = fmt.Errorf("Requested client was not initialized - %v",
target)
} else {
err = client.Call(method, args, reply)
// Check for network errors and try
// again just once.
if err != nil {
if err.Error() == rpc.ErrShutdown.Error() {
err = client.Call(method, args, reply)
}
}
}
errArr[ix] = err
}(i, target)
}
// Wait for requests to complete.
wg.Wait()
// Map of errors
errsMap := make(map[string]error)
for i, errVal := range errArr {
if errVal != nil {
errsMap[peers[i]] = errVal
}
}
return errsMap
}
// S3PeersUpdateBucketNotification - Sends Update Bucket notification
// request to all peers. Currently we log an error and continue.
func S3PeersUpdateBucketNotification(bucket string, ncfg *notificationConfig) {
setBNPArgs := &SetBNPArgs{Bucket: bucket, NCfg: ncfg}
peers := globalS3Peers.GetPeers()
errsMap := globalS3Peers.SendRPC(peers, "S3.SetBucketNotificationPeer",
setBNPArgs)
for peer, err := range errsMap {
errorIf(err, "Error sending peer update bucket notification to %s - %v", peer, err)
}
}
// S3PeersUpdateBucketListener - Sends Update Bucket listeners request
// to all peers. Currently we log an error and continue.
func S3PeersUpdateBucketListener(bucket string, lcfg []listenerConfig) {
setBLPArgs := &SetBLPArgs{Bucket: bucket, LCfg: lcfg}
peers := globalS3Peers.GetPeers()
errsMap := globalS3Peers.SendRPC(peers, "S3.SetBucketListenerPeer",
setBLPArgs)
for peer, err := range errsMap {
errorIf(err, "Error sending peer update bucket listener to %s - %v", peer, err)
}
}
// S3PeersUpdateBucketPolicy - Sends update bucket policy request to
// all peers. Currently we log an error and continue.
func S3PeersUpdateBucketPolicy(bucket string, pCh policyChange) {
byts, err := json.Marshal(pCh)
if err != nil {
errorIf(err, "Failed to marshal policyChange - this is a BUG!")
return
}
setBPPArgs := &SetBPPArgs{Bucket: bucket, PChBytes: byts}
peers := globalS3Peers.GetPeers()
errsMap := globalS3Peers.SendRPC(peers, "S3.SetBucketPolicyPeer", setBPPArgs)
for peer, err := range errsMap {
errorIf(err, "Error sending peer update bucket policy to %s - %v", peer, err)
}
}