oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix a bug in dsync initialization and communication (#5428)

Browse Source 
In current implementation we used as many dsync clients
as per number of endpoints(along with path) which is not
the expected implementation. The implementation of Dsync
was expected to be just for the endpoint Host alone such
that if you have 4 servers and each with 4 disks we need
to only have 4 dsync clients and 4 dsync servers. But
we currently had 8 clients, servers which in-fact is
unexpected and should be avoided.

This PR brings the implementation back to its original
intention. This issue was found #5160
master
Harshavardhana 7 years ago committed by kannappanr
parent bb73c84b10
commit f3f09ed14e
8 changed files with 159 additions and 148 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 59
      cmd/lock-rpc-server.go
  2. 25
      cmd/lock-rpc-server_test.go
  3. 38
      cmd/namespace-lock.go
  4. 4
      cmd/server-main.go
  5. 24
      vendor/github.com/minio/dsync/README.md
  6. 88
      vendor/github.com/minio/dsync/drwmutex.go
  7. 63
      vendor/github.com/minio/dsync/dsync.go
  8. 6
      vendor/vendor.json

59
cmd/lock-rpc-server.go
Unescape View File

@ -19,7 +19,6 @@ package cmd
import ( import (
"fmt" "fmt"
"math/rand" "math/rand"
"path"
"sync" "sync"
"time" "time"
@ -64,49 +63,45 @@ type lockServer struct {
} }
// Start lock maintenance from all lock servers. // Start lock maintenance from all lock servers.
func startLockMaintenance(lockServers []*lockServer) { func startLockMaintenance(lkSrv *lockServer) {
for _, locker := range lockServers { // Start loop for stale lock maintenance
// Start loop for stale lock maintenance go func(lk *lockServer) {
go func(lk *lockServer) { // Initialize a new ticker with a minute between each ticks.
// Initialize a new ticker with a minute between each ticks. ticker := time.NewTicker(lockMaintenanceInterval)
ticker := time.NewTicker(lockMaintenanceInterval)
// Start with random sleep time, so as to avoid "synchronous checks" between servers
// Start with random sleep time, so as to avoid "synchronous checks" between servers time.Sleep(time.Duration(rand.Float64() * float64(lockMaintenanceInterval)))
time.Sleep(time.Duration(rand.Float64() * float64(lockMaintenanceInterval))) for {
for { // Verifies every minute for locks held more than 2minutes.
// Verifies every minute for locks held more than 2minutes. select {
select { case <-globalServiceDoneCh:
case <-globalServiceDoneCh: // Stop the timer upon service closure and cleanup the go-routine.
// Stop the timer upon service closure and cleanup the go-routine. ticker.Stop()
ticker.Stop() return
return case <-ticker.C:
case <-ticker.C: lk.lockMaintenance(lockValidityCheckInterval)
lk.lockMaintenance(lockValidityCheckInterval)
}
} }
}(locker) }
} }(lkSrv)
} }
// Register distributed NS lock handlers. // Register distributed NS lock handlers.
func registerDistNSLockRouter(mux *router.Router, endpoints EndpointList) error { func registerDistNSLockRouter(mux *router.Router, endpoints EndpointList) error {
// Start lock maintenance from all lock servers. // Start lock maintenance from all lock servers.
startLockMaintenance(globalLockServers) startLockMaintenance(globalLockServer)
// Register initialized lock servers to their respective rpc endpoints. // Register initialized lock servers to their respective rpc endpoints.
return registerStorageLockers(mux, globalLockServers) return registerStorageLockers(mux, globalLockServer)
} }
// registerStorageLockers - register locker rpc handlers for net/rpc library clients // registerStorageLockers - register locker rpc handlers for net/rpc library clients
func registerStorageLockers(mux *router.Router, lockServers []*lockServer) error { func registerStorageLockers(mux *router.Router, lkSrv *lockServer) error {
for _, lockServer := range lockServers { lockRPCServer := newRPCServer()
lockRPCServer := newRPCServer() if err := lockRPCServer.RegisterName(lockServiceName, lkSrv); err != nil {
if err := lockRPCServer.RegisterName(lockServiceName, lockServer); err != nil { return errors.Trace(err)
return errors.Trace(err)
}
lockRouter := mux.PathPrefix(minioReservedBucketPath).Subrouter()
lockRouter.Path(path.Join(lockServicePath, lockServer.ll.serviceEndpoint)).Handler(lockRPCServer)
} }
lockRouter := mux.PathPrefix(minioReservedBucketPath).Subrouter()
lockRouter.Path(lockServicePath).Handler(lockRPCServer)
return nil return nil
} }

25
cmd/lock-rpc-server_test.go
Unescape View File

@ -425,8 +425,8 @@ func TestLockRpcServerExpired(t *testing.T) {
} }
} }
// Test initialization of lock servers. // Test initialization of lock server.
func TestLockServers(t *testing.T) { func TestLockServerInit(t *testing.T) {
if runtime.GOOS == globalWindowsOSName { if runtime.GOOS == globalWindowsOSName {
return return
} }
@ -438,8 +438,10 @@ func TestLockServers(t *testing.T) {
defer os.RemoveAll(rootPath) defer os.RemoveAll(rootPath)
currentIsDistXL := globalIsDistXL currentIsDistXL := globalIsDistXL
currentLockServer := globalLockServer
defer func() { defer func() {
globalIsDistXL = currentIsDistXL globalIsDistXL = currentIsDistXL
globalLockServer = currentLockServer
}() }()
case1Endpoints := mustGetNewEndpointList( case1Endpoints := mustGetNewEndpointList(
@ -468,26 +470,27 @@ func TestLockServers(t *testing.T) {
globalMinioHost = "" globalMinioHost = ""
testCases := []struct { testCases := []struct {
isDistXL bool isDistXL bool
endpoints EndpointList endpoints EndpointList
totalLockServers int
}{ }{
// Test - 1 one lock server initialized. // Test - 1 one lock server initialized.
{true, case1Endpoints, 1}, {true, case1Endpoints},
// Test - 2 two servers possible. // Test - similar endpoint hosts should
{true, case2Endpoints, 2}, // converge to single lock server
// initialized.
{true, case2Endpoints},
} }
// Validates lock server initialization. // Validates lock server initialization.
for i, testCase := range testCases { for i, testCase := range testCases {
globalIsDistXL = testCase.isDistXL globalIsDistXL = testCase.isDistXL
globalLockServers = nil globalLockServer = nil
_, _ = newDsyncNodes(testCase.endpoints) _, _ = newDsyncNodes(testCase.endpoints)
if err != nil { if err != nil {
t.Fatalf("Got unexpected error initializing lock servers: %v", err) t.Fatalf("Got unexpected error initializing lock servers: %v", err)
} }
if len(globalLockServers) != testCase.totalLockServers { if globalLockServer == nil && testCase.isDistXL {
t.Fatalf("Test %d: Expected total %d, got %d", i+1, testCase.totalLockServers, len(globalLockServers)) t.Errorf("Test %d: Expected initialized lockServer, but got uninitialized", i+1)
} }
} }
} }

38
cmd/namespace-lock.go
Unescape View File

@ -28,13 +28,17 @@ import (
"github.com/minio/dsync" "github.com/minio/dsync"
"github.com/minio/lsync" "github.com/minio/lsync"
"github.com/minio/minio-go/pkg/set"
) )
// Global name space lock. // Global name space lock.
var globalNSMutex *nsLockMap var globalNSMutex *nsLockMap
// Global lock servers // Global lock server one per server.
var globalLockServers []*lockServer var globalLockServer *lockServer
// Instance of dsync for distributed clients.
var globalDsync *dsync.Dsync
// RWLocker - locker interface to introduce GetRLock, RUnlock. // RWLocker - locker interface to introduce GetRLock, RUnlock.
type RWLocker interface { type RWLocker interface {
@ -56,39 +60,41 @@ type RWLockerSync interface {
// Returns lock clients and the node index for the current server. // Returns lock clients and the node index for the current server.
func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int) { func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int) {
cred := globalServerConfig.GetCredential() cred := globalServerConfig.GetCredential()
clnts = make([]dsync.NetLocker, len(endpoints))
myNode = -1 myNode = -1
for index, endpoint := range endpoints { seenHosts := set.NewStringSet()
for _, endpoint := range endpoints {
if seenHosts.Contains(endpoint.Host) {
continue
}
seenHosts.Add(endpoint.Host)
if !endpoint.IsLocal { if !endpoint.IsLocal {
// For a remote endpoints setup a lock RPC client. // For a remote endpoints setup a lock RPC client.
clnts[index] = newLockRPCClient(authConfig{ clnts = append(clnts, newLockRPCClient(authConfig{
accessKey: cred.AccessKey, accessKey: cred.AccessKey,
secretKey: cred.SecretKey, secretKey: cred.SecretKey,
serverAddr: endpoint.Host, serverAddr: endpoint.Host,
secureConn: globalIsSSL, secureConn: globalIsSSL,
serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath, endpoint.Path), serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath),
serviceName: lockServiceName, serviceName: lockServiceName,
}) }))
continue continue
} }
// Local endpoint // Local endpoint
if myNode == -1 { myNode = len(clnts)
myNode = index
}
// For a local endpoint, setup a local lock server to // For a local endpoint, setup a local lock server to
// avoid network requests. // avoid network requests.
localLockServer := lockServer{ localLockServer := lockServer{
AuthRPCServer: AuthRPCServer{}, AuthRPCServer: AuthRPCServer{},
ll: localLocker{ ll: localLocker{
mutex: sync.Mutex{},
serviceEndpoint: endpoint.Path,
serverAddr: endpoint.Host, serverAddr: endpoint.Host,
serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath),
lockMap: make(map[string][]lockRequesterInfo), lockMap: make(map[string][]lockRequesterInfo),
}, },
} }
globalLockServers = append(globalLockServers, &localLockServer) globalLockServer = &localLockServer
clnts[index] = &(localLockServer.ll) clnts = append(clnts, &(localLockServer.ll))
} }
return clnts, myNode return clnts, myNode
@ -149,7 +155,7 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
nsLk = &nsLock{ nsLk = &nsLock{
RWLockerSync: func() RWLockerSync { RWLockerSync: func() RWLockerSync {
if n.isDistXL { if n.isDistXL {
return dsync.NewDRWMutex(pathJoin(volume, path)) return dsync.NewDRWMutex(pathJoin(volume, path), globalDsync)
} }
return &lsync.LRWMutex{} return &lsync.LRWMutex{}
}(), }(),
@ -303,7 +309,7 @@ func (n *nsLockMap) ForceUnlock(volume, path string) {
// are blocking can now proceed as normal and any new locks will also // are blocking can now proceed as normal and any new locks will also
// participate normally. // participate normally.
if n.isDistXL { // For distributed mode, broadcast ForceUnlock message. if n.isDistXL { // For distributed mode, broadcast ForceUnlock message.
dsync.NewDRWMutex(pathJoin(volume, path)).ForceUnlock() dsync.NewDRWMutex(pathJoin(volume, path), globalDsync).ForceUnlock()
} }
param := nsParam{volume, path} param := nsParam{volume, path}

4
cmd/server-main.go
Unescape View File

@ -186,8 +186,8 @@ func serverMain(ctx *cli.Context) {
// Set nodes for dsync for distributed setup. // Set nodes for dsync for distributed setup.
if globalIsDistXL { if globalIsDistXL {
clnts, myNode := newDsyncNodes(globalEndpoints) globalDsync, err = dsync.New(newDsyncNodes(globalEndpoints))
fatalIf(dsync.Init(clnts, myNode), "Unable to initialize distributed locking clients") fatalIf(err, "Unable to initialize distributed locking clients")
} }
// Initialize name space lock. // Initialize name space lock.

24
vendor/github.com/minio/dsync/README.md generated vendored
Unescape View File

@ -87,22 +87,26 @@ The system can be pushed to 75K locks/sec at 50% CPU load.
Usage Usage
----- -----
> NOTE: Previously if you were using `dsync.Init([]NetLocker, nodeIndex)` to initialize dsync has
been changed to `dsync.New([]NetLocker, nodeIndex)` which returns a `*Dsync` object to be used in
every instance of `NewDRWMutex("test", *Dsync)`
### Exclusive lock ### Exclusive lock
Here is a simple example showing how to protect a single resource (drop-in replacement for `sync.Mutex`): Here is a simple example showing how to protect a single resource (drop-in replacement for `sync.Mutex`):
``` ```go
import ( import (
"github.com/minio/dsync" "github.com/minio/dsync"
) )
func lockSameResource() { func lockSameResource() {
// Create distributed mutex to protect resource 'test' // Create distributed mutex to protect resource 'test'
dm := dsync.NewDRWMutex("test") dm := dsync.NewDRWMutex("test", ds)
dm.Lock() dm.Lock()
log.Println("first lock granted") log.Println("first lock granted")
// Release 1st lock after 5 seconds // Release 1st lock after 5 seconds
go func() { go func() {
@ -111,10 +115,10 @@ func lockSameResource() {
dm.Unlock() dm.Unlock()
}() }()
// Try to acquire lock again, will block until initial lock is released // Try to acquire lock again, will block until initial lock is released
log.Println("about to lock same resource again...") log.Println("about to lock same resource again...")
dm.Lock() dm.Lock()
log.Println("second lock granted") log.Println("second lock granted")
time.Sleep(2 * time.Second) time.Sleep(2 * time.Second)
dm.Unlock() dm.Unlock()
@ -137,7 +141,7 @@ DRWMutex also supports multiple simultaneous read locks as shown below (analogou
``` ```
func twoReadLocksAndSingleWriteLock() { func twoReadLocksAndSingleWriteLock() {
drwm := dsync.NewDRWMutex("resource") drwm := dsync.NewDRWMutex("resource", ds)
drwm.RLock() drwm.RLock()
log.Println("1st read lock acquired, waiting...") log.Println("1st read lock acquired, waiting...")
@ -160,7 +164,7 @@ func twoReadLocksAndSingleWriteLock() {
log.Println("Trying to acquire write lock, waiting...") log.Println("Trying to acquire write lock, waiting...")
drwm.Lock() drwm.Lock()
log.Println("Write lock acquired, waiting...") log.Println("Write lock acquired, waiting...")
time.Sleep(3 * time.Second) time.Sleep(3 * time.Second)
drwm.Unlock() drwm.Unlock()

88
vendor/github.com/minio/dsync/drwmutex.go generated vendored
Unescape View File

@ -49,6 +49,7 @@ type DRWMutex struct {
writeLocks []string // Array of nodes that granted a write lock writeLocks []string // Array of nodes that granted a write lock
readersLocks [][]string // Array of array of nodes that granted reader locks readersLocks [][]string // Array of array of nodes that granted reader locks
m sync.Mutex // Mutex to prevent multiple simultaneous locks from this node m sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
clnt *Dsync
} }
// Granted - represents a structure of a granted lock. // Granted - represents a structure of a granted lock.
@ -66,10 +67,11 @@ func isLocked(uid string) bool {
} }
// NewDRWMutex - initializes a new dsync RW mutex. // NewDRWMutex - initializes a new dsync RW mutex.
func NewDRWMutex(name string) *DRWMutex { func NewDRWMutex(name string, clnt *Dsync) *DRWMutex {
return &DRWMutex{ return &DRWMutex{
Name: name, Name: name,
writeLocks: make([]string, dnodeCount), writeLocks: make([]string, clnt.dNodeCount),
clnt: clnt,
} }
} }
@ -128,10 +130,10 @@ func (dm *DRWMutex) lockBlocking(timeout time.Duration, isReadLock bool) (locked
// Use incremental back-off algorithm for repeated attempts to acquire the lock // Use incremental back-off algorithm for repeated attempts to acquire the lock
for range newRetryTimerSimple(doneCh) { for range newRetryTimerSimple(doneCh) {
// Create temp array on stack. // Create temp array on stack.
locks := make([]string, dnodeCount) locks := make([]string, dm.clnt.dNodeCount)
// Try to acquire the lock. // Try to acquire the lock.
success := lock(clnts, &locks, dm.Name, isReadLock) success := lock(dm.clnt, &locks, dm.Name, isReadLock)
if success { if success {
dm.m.Lock() dm.m.Lock()
defer dm.m.Unlock() defer dm.m.Unlock()
@ -139,7 +141,7 @@ func (dm *DRWMutex) lockBlocking(timeout time.Duration, isReadLock bool) (locked
// If success, copy array to object // If success, copy array to object
if isReadLock { if isReadLock {
// Append new array of strings at the end // Append new array of strings at the end
dm.readersLocks = append(dm.readersLocks, make([]string, dnodeCount)) dm.readersLocks = append(dm.readersLocks, make([]string, dm.clnt.dNodeCount))
// and copy stack array into last spot // and copy stack array into last spot
copy(dm.readersLocks[len(dm.readersLocks)-1], locks[:]) copy(dm.readersLocks[len(dm.readersLocks)-1], locks[:])
} else { } else {
@ -158,14 +160,14 @@ func (dm *DRWMutex) lockBlocking(timeout time.Duration, isReadLock bool) (locked
} }
// lock tries to acquire the distributed lock, returning true or false. // lock tries to acquire the distributed lock, returning true or false.
func lock(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool) bool { func lock(ds *Dsync, locks *[]string, lockName string, isReadLock bool) bool {
// Create buffered channel of size equal to total number of nodes. // Create buffered channel of size equal to total number of nodes.
ch := make(chan Granted, dnodeCount) ch := make(chan Granted, ds.dNodeCount)
defer close(ch) defer close(ch)
var wg sync.WaitGroup var wg sync.WaitGroup
for index, c := range clnts { for index, c := range ds.rpcClnts {
wg.Add(1) wg.Add(1)
// broadcast lock request to all nodes // broadcast lock request to all nodes
@ -181,8 +183,8 @@ func lock(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool)
args := LockArgs{ args := LockArgs{
UID: uid, UID: uid,
Resource: lockName, Resource: lockName,
ServerAddr: clnts[ownNode].ServerAddr(), ServerAddr: ds.rpcClnts[ds.ownNode].ServerAddr(),
ServiceEndpoint: clnts[ownNode].ServiceEndpoint(), ServiceEndpoint: ds.rpcClnts[ds.ownNode].ServiceEndpoint(),
} }
var locked bool var locked bool
@ -222,7 +224,7 @@ func lock(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool)
done := false done := false
timeout := time.After(DRWMutexAcquireTimeout) timeout := time.After(DRWMutexAcquireTimeout)
for ; i < dnodeCount; i++ { // Loop until we acquired all locks for ; i < ds.dNodeCount; i++ { // Loop until we acquired all locks
select { select {
case grant := <-ch: case grant := <-ch:
@ -231,22 +233,22 @@ func lock(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool)
(*locks)[grant.index] = grant.lockUID (*locks)[grant.index] = grant.lockUID
} else { } else {
locksFailed++ locksFailed++
if !isReadLock && locksFailed > dnodeCount-dquorum || if !isReadLock && locksFailed > ds.dNodeCount-ds.dquorum ||
isReadLock && locksFailed > dnodeCount-dquorumReads { isReadLock && locksFailed > ds.dNodeCount-ds.dquorumReads {
// We know that we are not going to get the lock anymore, // We know that we are not going to get the lock anymore,
// so exit out and release any locks that did get acquired // so exit out and release any locks that did get acquired
done = true done = true
// Increment the number of grants received from the buffered channel. // Increment the number of grants received from the buffered channel.
i++ i++
releaseAll(clnts, locks, lockName, isReadLock) releaseAll(ds, locks, lockName, isReadLock)
} }
} }
case <-timeout: case <-timeout:
done = true done = true
// timeout happened, maybe one of the nodes is slow, count // timeout happened, maybe one of the nodes is slow, count
// number of locks to check whether we have quorum or not // number of locks to check whether we have quorum or not
if !quorumMet(locks, isReadLock) { if !quorumMet(locks, isReadLock, ds.dquorum, ds.dquorumReads) {
releaseAll(clnts, locks, lockName, isReadLock) releaseAll(ds, locks, lockName, isReadLock)
} }
} }
@ -256,7 +258,7 @@ func lock(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool)
} }
// Count locks in order to determine whether we have quorum or not // Count locks in order to determine whether we have quorum or not
quorum = quorumMet(locks, isReadLock) quorum = quorumMet(locks, isReadLock, ds.dquorum, ds.dquorumReads)
// Signal that we have the quorum // Signal that we have the quorum
wg.Done() wg.Done()
@ -264,11 +266,11 @@ func lock(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool)
// Wait for the other responses and immediately release the locks // Wait for the other responses and immediately release the locks
// (do not add them to the locks array because the DRWMutex could // (do not add them to the locks array because the DRWMutex could
// already has been unlocked again by the original calling thread) // already has been unlocked again by the original calling thread)
for ; i < dnodeCount; i++ { for ; i < ds.dNodeCount; i++ {
grantToBeReleased := <-ch grantToBeReleased := <-ch
if grantToBeReleased.isLocked() { if grantToBeReleased.isLocked() {
// release lock // release lock
sendRelease(clnts[grantToBeReleased.index], lockName, grantToBeReleased.lockUID, isReadLock) sendRelease(ds, ds.rpcClnts[grantToBeReleased.index], lockName, grantToBeReleased.lockUID, isReadLock)
} }
} }
}(isReadLock) }(isReadLock)
@ -276,9 +278,9 @@ func lock(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool)
wg.Wait() wg.Wait()
// Verify that localhost server is actively participating in the lock (the lock maintenance relies on this fact) // Verify that localhost server is actively participating in the lock (the lock maintenance relies on this fact)
if quorum && !isLocked((*locks)[ownNode]) { if quorum && !isLocked((*locks)[ds.ownNode]) {
// If not, release lock (and try again later) // If not, release lock (and try again later)
releaseAll(clnts, locks, lockName, isReadLock) releaseAll(ds, locks, lockName, isReadLock)
quorum = false quorum = false
} }
@ -286,7 +288,7 @@ func lock(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool)
} }
// quorumMet determines whether we have acquired the required quorum of underlying locks or not // quorumMet determines whether we have acquired the required quorum of underlying locks or not
func quorumMet(locks *[]string, isReadLock bool) bool { func quorumMet(locks *[]string, isReadLock bool, quorum, quorumReads int) bool {
count := 0 count := 0
for _, uid := range *locks { for _, uid := range *locks {
@ -295,21 +297,21 @@ func quorumMet(locks *[]string, isReadLock bool) bool {
} }
} }
var quorum bool var metQuorum bool
if isReadLock { if isReadLock {
quorum = count >= dquorumReads metQuorum = count >= quorumReads
} else { } else {
quorum = count >= dquorum metQuorum = count >= quorum
} }
return quorum return metQuorum
} }
// releaseAll releases all locks that are marked as locked // releaseAll releases all locks that are marked as locked
func releaseAll(clnts []NetLocker, locks *[]string, lockName string, isReadLock bool) { func releaseAll(ds *Dsync, locks *[]string, lockName string, isReadLock bool) {
for lock := 0; lock < dnodeCount; lock++ { for lock := 0; lock < ds.dNodeCount; lock++ {
if isLocked((*locks)[lock]) { if isLocked((*locks)[lock]) {
sendRelease(clnts[lock], lockName, (*locks)[lock], isReadLock) sendRelease(ds, ds.rpcClnts[lock], lockName, (*locks)[lock], isReadLock)
(*locks)[lock] = "" (*locks)[lock] = ""
} }
} }
@ -321,7 +323,7 @@ func releaseAll(clnts []NetLocker, locks *[]string, lockName string, isReadLock
func (dm *DRWMutex) Unlock() { func (dm *DRWMutex) Unlock() {
// create temp array on stack // create temp array on stack
locks := make([]string, dnodeCount) locks := make([]string, dm.clnt.dNodeCount)
{ {
dm.m.Lock() dm.m.Lock()
@ -342,11 +344,11 @@ func (dm *DRWMutex) Unlock() {
// Copy write locks to stack array // Copy write locks to stack array
copy(locks, dm.writeLocks[:]) copy(locks, dm.writeLocks[:])
// Clear write locks array // Clear write locks array
dm.writeLocks = make([]string, dnodeCount) dm.writeLocks = make([]string, dm.clnt.dNodeCount)
} }
isReadLock := false isReadLock := false
unlock(locks, dm.Name, isReadLock) unlock(dm.clnt, locks, dm.Name, isReadLock)
} }
// RUnlock releases a read lock held on dm. // RUnlock releases a read lock held on dm.
@ -355,7 +357,7 @@ func (dm *DRWMutex) Unlock() {
func (dm *DRWMutex) RUnlock() { func (dm *DRWMutex) RUnlock() {
// create temp array on stack // create temp array on stack
locks := make([]string, dnodeCount) locks := make([]string, dm.clnt.dNodeCount)
{ {
dm.m.Lock() dm.m.Lock()
@ -370,19 +372,19 @@ func (dm *DRWMutex) RUnlock() {
} }
isReadLock := true isReadLock := true
unlock(locks, dm.Name, isReadLock) unlock(dm.clnt, locks, dm.Name, isReadLock)
} }
func unlock(locks []string, name string, isReadLock bool) { func unlock(ds *Dsync, locks []string, name string, isReadLock bool) {
// We don't need to synchronously wait until we have released all the locks (or the quorum) // We don't need to synchronously wait until we have released all the locks (or the quorum)
// (a subsequent lock will retry automatically in case it would fail to get quorum) // (a subsequent lock will retry automatically in case it would fail to get quorum)
for index, c := range clnts { for index, c := range ds.rpcClnts {
if isLocked(locks[index]) { if isLocked(locks[index]) {
// broadcast lock release to all nodes that granted the lock // broadcast lock release to all nodes that granted the lock
sendRelease(c, name, locks[index], isReadLock) sendRelease(ds, c, name, locks[index], isReadLock)
} }
} }
} }
@ -394,24 +396,24 @@ func (dm *DRWMutex) ForceUnlock() {
defer dm.m.Unlock() defer dm.m.Unlock()
// Clear write locks array // Clear write locks array
dm.writeLocks = make([]string, dnodeCount) dm.writeLocks = make([]string, dm.clnt.dNodeCount)
// Clear read locks array // Clear read locks array
dm.readersLocks = nil dm.readersLocks = nil
} }
for _, c := range clnts { for _, c := range dm.clnt.rpcClnts {
// broadcast lock release to all nodes that granted the lock // broadcast lock release to all nodes that granted the lock
sendRelease(c, dm.Name, "", false) sendRelease(dm.clnt, c, dm.Name, "", false)
} }
} }
// sendRelease sends a release message to a node that previously granted a lock // sendRelease sends a release message to a node that previously granted a lock
func sendRelease(c NetLocker, name, uid string, isReadLock bool) { func sendRelease(ds *Dsync, c NetLocker, name, uid string, isReadLock bool) {
args := LockArgs{ args := LockArgs{
UID: uid, UID: uid,
Resource: name, Resource: name,
ServerAddr: clnts[ownNode].ServerAddr(), ServerAddr: ds.rpcClnts[ds.ownNode].ServerAddr(),
ServiceEndpoint: clnts[ownNode].ServiceEndpoint(), ServiceEndpoint: ds.rpcClnts[ds.ownNode].ServiceEndpoint(),
} }
if len(uid) == 0 { if len(uid) == 0 {
if _, err := c.ForceUnlock(args); err != nil { if _, err := c.ForceUnlock(args); err != nil {

63
vendor/github.com/minio/dsync/dsync.go generated vendored
Unescape View File

@ -16,51 +16,52 @@
package dsync package dsync
import "errors" import (
"errors"
)
// Number of nodes participating in the distributed locking. // Dsync represents dsync client object which is initialized with
var dnodeCount int // authenticated clients, used to initiate lock RPC calls.
type Dsync struct {
// Number of nodes participating in the distributed locking.
dNodeCount int
// List of rpc client objects, one per lock server. // List of rpc client objects, one per lock server.
var clnts []NetLocker rpcClnts []NetLocker
// Index into rpc client array for server running on localhost // Index into rpc client array for server running on localhost
var ownNode int ownNode int
// Simple majority based quorum, set to dNodeCount/2+1 // Simple majority based quorum, set to dNodeCount/2+1
var dquorum int dquorum int
// Simple quorum for read operations, set to dNodeCount/2 // Simple quorum for read operations, set to dNodeCount/2
var dquorumReads int dquorumReads int
}
// Init - initializes package-level global state variables such as clnts.
// N B - This function should be called only once inside any program
// that uses dsync.
func Init(rpcClnts []NetLocker, rpcOwnNode int) (err error) {
// Validate if number of nodes is within allowable range. // New - initializes a new dsync object with input rpcClnts.
if dnodeCount != 0 { func New(rpcClnts []NetLocker, rpcOwnNode int) (*Dsync, error) {
return errors.New("Cannot reinitialize dsync package")
}
if len(rpcClnts) < 4 { if len(rpcClnts) < 4 {
return errors.New("Dsync is not designed for less than 4 nodes") return nil, errors.New("Dsync is not designed for less than 4 nodes")
} else if len(rpcClnts) > 16 { } else if len(rpcClnts) > 16 {
return errors.New("Dsync is not designed for more than 16 nodes") return nil, errors.New("Dsync is not designed for more than 16 nodes")
} else if len(rpcClnts)%2 != 0 { } else if len(rpcClnts)%2 != 0 {
return errors.New("Dsync is not designed for an uneven number of nodes") return nil, errors.New("Dsync is not designed for an uneven number of nodes")
} }
if rpcOwnNode > len(rpcClnts) { if rpcOwnNode > len(rpcClnts) {
return errors.New("Index for own node is too large") return nil, errors.New("Index for own node is too large")
} }
dnodeCount = len(rpcClnts) ds := &Dsync{}
dquorum = dnodeCount/2 + 1 ds.dNodeCount = len(rpcClnts)
dquorumReads = dnodeCount / 2 ds.dquorum = ds.dNodeCount/2 + 1
ds.dquorumReads = ds.dNodeCount / 2
ds.ownNode = rpcOwnNode
// Initialize node name and rpc path for each NetLocker object. // Initialize node name and rpc path for each NetLocker object.
clnts = make([]NetLocker, dnodeCount) ds.rpcClnts = make([]NetLocker, ds.dNodeCount)
copy(clnts, rpcClnts) copy(ds.rpcClnts, rpcClnts)
ownNode = rpcOwnNode return ds, nil
return nil
} }

6
vendor/vendor.json vendored
Unescape View File

@ -380,10 +380,10 @@
"revisionTime": "2017-02-27T07:32:28Z" "revisionTime": "2017-02-27T07:32:28Z"
}, },
{ {
"checksumSHA1": "hQ8i4UPTbFW68oPJP3uFxYTLfxk=", "checksumSHA1": "qhWQM7xmqaxFqADNTj8YPjE/8Ws=",
"path": "github.com/minio/dsync", "path": "github.com/minio/dsync",
"revision": "a26b9de6c8006208d10a9517720d3212b42c374e", "revision": "ed0989bc6c7b199f749fa6be0b7ee98d689b88c7",
"revisionTime": "2017-05-25T17:53:53Z" "revisionTime": "2017-11-22T09:16:00Z"
}, },
{ {
"path": "github.com/minio/go-homedir", "path": "github.com/minio/go-homedir",

Write Preview
Loading…
Cancel
Save