Support MinIO to be deployed on more than 32 nodes (#8492)
This PR implements locking from a global entity into a more localized set level entity, allowing for locks to be held only on the resources which are writing to a collection of disks rather than a global level. In this process this PR also removes the top-level limit of 32 nodes to an unlimited number of nodes. This is a precursor change before bring in bucket expansion.master
Harshavardhana
5 years ago
committed by
kannappanr
parent
069b8ee8ff
commit
e9b2bf00ad
@ -0,0 +1,3 @@ |
|||||||
|
dsync.test |
||||||
|
coverage.txt |
||||||
|
*.out |
||||||
@ -0,0 +1,440 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package dsync |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
golog "log" |
||||||
|
"math/rand" |
||||||
|
"os" |
||||||
|
"path" |
||||||
|
"runtime" |
||||||
|
"sync" |
||||||
|
"time" |
||||||
|
) |
||||||
|
|
||||||
|
// Indicator if logging is enabled.
|
||||||
|
var dsyncLog bool |
||||||
|
|
||||||
|
func init() { |
||||||
|
// Check for MINIO_DSYNC_TRACE env variable, if set logging will be enabled for failed REST operations.
|
||||||
|
dsyncLog = os.Getenv("MINIO_DSYNC_TRACE") == "1" |
||||||
|
rand.Seed(time.Now().UnixNano()) |
||||||
|
} |
||||||
|
|
||||||
|
func log(msg ...interface{}) { |
||||||
|
if dsyncLog { |
||||||
|
golog.Println(msg...) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// DRWMutexAcquireTimeout - tolerance limit to wait for lock acquisition before.
|
||||||
|
const DRWMutexAcquireTimeout = 1 * time.Second // 1 second.
|
||||||
|
const drwMutexInfinite = time.Duration(1<<63 - 1) |
||||||
|
|
||||||
|
// A DRWMutex is a distributed mutual exclusion lock.
|
||||||
|
type DRWMutex struct { |
||||||
|
Name string |
||||||
|
writeLocks []string // Array of nodes that granted a write lock
|
||||||
|
readersLocks [][]string // Array of array of nodes that granted reader locks
|
||||||
|
m sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
|
||||||
|
clnt *Dsync |
||||||
|
ctx context.Context |
||||||
|
} |
||||||
|
|
||||||
|
// Granted - represents a structure of a granted lock.
|
||||||
|
type Granted struct { |
||||||
|
index int |
||||||
|
lockUID string // Locked if set with UID string, unlocked if empty
|
||||||
|
} |
||||||
|
|
||||||
|
func (g *Granted) isLocked() bool { |
||||||
|
return isLocked(g.lockUID) |
||||||
|
} |
||||||
|
|
||||||
|
func isLocked(uid string) bool { |
||||||
|
return len(uid) > 0 |
||||||
|
} |
||||||
|
|
||||||
|
// NewDRWMutex - initializes a new dsync RW mutex.
|
||||||
|
func NewDRWMutex(ctx context.Context, name string, clnt *Dsync) *DRWMutex { |
||||||
|
return &DRWMutex{ |
||||||
|
Name: name, |
||||||
|
writeLocks: make([]string, clnt.dNodeCount), |
||||||
|
clnt: clnt, |
||||||
|
ctx: ctx, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Lock holds a write lock on dm.
|
||||||
|
//
|
||||||
|
// If the lock is already in use, the calling go routine
|
||||||
|
// blocks until the mutex is available.
|
||||||
|
func (dm *DRWMutex) Lock(id, source string) { |
||||||
|
|
||||||
|
isReadLock := false |
||||||
|
dm.lockBlocking(drwMutexInfinite, id, source, isReadLock) |
||||||
|
} |
||||||
|
|
||||||
|
// GetLock tries to get a write lock on dm before the timeout elapses.
|
||||||
|
//
|
||||||
|
// If the lock is already in use, the calling go routine
|
||||||
|
// blocks until either the mutex becomes available and return success or
|
||||||
|
// more time has passed than the timeout value and return false.
|
||||||
|
func (dm *DRWMutex) GetLock(id, source string, timeout time.Duration) (locked bool) { |
||||||
|
|
||||||
|
isReadLock := false |
||||||
|
return dm.lockBlocking(timeout, id, source, isReadLock) |
||||||
|
} |
||||||
|
|
||||||
|
// RLock holds a read lock on dm.
|
||||||
|
//
|
||||||
|
// If one or more read locks are already in use, it will grant another lock.
|
||||||
|
// Otherwise the calling go routine blocks until the mutex is available.
|
||||||
|
func (dm *DRWMutex) RLock(id, source string) { |
||||||
|
|
||||||
|
isReadLock := true |
||||||
|
dm.lockBlocking(drwMutexInfinite, id, source, isReadLock) |
||||||
|
} |
||||||
|
|
||||||
|
// GetRLock tries to get a read lock on dm before the timeout elapses.
|
||||||
|
//
|
||||||
|
// If one or more read locks are already in use, it will grant another lock.
|
||||||
|
// Otherwise the calling go routine blocks until either the mutex becomes
|
||||||
|
// available and return success or more time has passed than the timeout
|
||||||
|
// value and return false.
|
||||||
|
func (dm *DRWMutex) GetRLock(id, source string, timeout time.Duration) (locked bool) { |
||||||
|
|
||||||
|
isReadLock := true |
||||||
|
return dm.lockBlocking(timeout, id, source, isReadLock) |
||||||
|
} |
||||||
|
|
||||||
|
// lockBlocking will try to acquire either a read or a write lock
|
||||||
|
//
|
||||||
|
// The function will loop using a built-in timing randomized back-off
|
||||||
|
// algorithm until either the lock is acquired successfully or more
|
||||||
|
// time has elapsed than the timeout value.
|
||||||
|
func (dm *DRWMutex) lockBlocking(timeout time.Duration, id, source string, isReadLock bool) (locked bool) { |
||||||
|
doneCh, start := make(chan struct{}), time.Now().UTC() |
||||||
|
defer close(doneCh) |
||||||
|
|
||||||
|
// Use incremental back-off algorithm for repeated attempts to acquire the lock
|
||||||
|
for range newRetryTimerSimple(doneCh) { |
||||||
|
select { |
||||||
|
case <-dm.ctx.Done(): |
||||||
|
return |
||||||
|
default: |
||||||
|
} |
||||||
|
|
||||||
|
// Create temp array on stack.
|
||||||
|
locks := make([]string, dm.clnt.dNodeCount) |
||||||
|
|
||||||
|
// Try to acquire the lock.
|
||||||
|
success := lock(dm.clnt, &locks, dm.Name, id, source, isReadLock) |
||||||
|
if success { |
||||||
|
dm.m.Lock() |
||||||
|
|
||||||
|
// If success, copy array to object
|
||||||
|
if isReadLock { |
||||||
|
// Append new array of strings at the end
|
||||||
|
dm.readersLocks = append(dm.readersLocks, make([]string, dm.clnt.dNodeCount)) |
||||||
|
// and copy stack array into last spot
|
||||||
|
copy(dm.readersLocks[len(dm.readersLocks)-1], locks[:]) |
||||||
|
} else { |
||||||
|
copy(dm.writeLocks, locks[:]) |
||||||
|
} |
||||||
|
|
||||||
|
dm.m.Unlock() |
||||||
|
return true |
||||||
|
} |
||||||
|
if time.Now().UTC().Sub(start) >= timeout { // Are we past the timeout?
|
||||||
|
break |
||||||
|
} |
||||||
|
// Failed to acquire the lock on this attempt, incrementally wait
|
||||||
|
// for a longer back-off time and try again afterwards.
|
||||||
|
} |
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
// lock tries to acquire the distributed lock, returning true or false.
|
||||||
|
func lock(ds *Dsync, locks *[]string, lockName, id, source string, isReadLock bool) bool { |
||||||
|
|
||||||
|
// Create buffered channel of size equal to total number of nodes.
|
||||||
|
ch := make(chan Granted, ds.dNodeCount) |
||||||
|
defer close(ch) |
||||||
|
|
||||||
|
var wg sync.WaitGroup |
||||||
|
for index, c := range ds.restClnts { |
||||||
|
|
||||||
|
wg.Add(1) |
||||||
|
// broadcast lock request to all nodes
|
||||||
|
go func(index int, isReadLock bool, c NetLocker) { |
||||||
|
defer wg.Done() |
||||||
|
|
||||||
|
args := LockArgs{ |
||||||
|
UID: id, |
||||||
|
Resource: lockName, |
||||||
|
Source: source, |
||||||
|
} |
||||||
|
|
||||||
|
var locked bool |
||||||
|
var err error |
||||||
|
if isReadLock { |
||||||
|
if locked, err = c.RLock(args); err != nil { |
||||||
|
log("Unable to call RLock", err) |
||||||
|
} |
||||||
|
} else { |
||||||
|
if locked, err = c.Lock(args); err != nil { |
||||||
|
log("Unable to call Lock", err) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
g := Granted{index: index} |
||||||
|
if locked { |
||||||
|
g.lockUID = args.UID |
||||||
|
} |
||||||
|
|
||||||
|
ch <- g |
||||||
|
|
||||||
|
}(index, isReadLock, c) |
||||||
|
} |
||||||
|
|
||||||
|
quorum := false |
||||||
|
|
||||||
|
wg.Add(1) |
||||||
|
go func(isReadLock bool) { |
||||||
|
|
||||||
|
// Wait until we have either
|
||||||
|
//
|
||||||
|
// a) received all lock responses
|
||||||
|
// b) received too many 'non-'locks for quorum to be still possible
|
||||||
|
// c) time out
|
||||||
|
//
|
||||||
|
i, locksFailed := 0, 0 |
||||||
|
done := false |
||||||
|
timeout := time.After(DRWMutexAcquireTimeout) |
||||||
|
|
||||||
|
for ; i < ds.dNodeCount; i++ { // Loop until we acquired all locks
|
||||||
|
|
||||||
|
select { |
||||||
|
case grant := <-ch: |
||||||
|
if grant.isLocked() { |
||||||
|
// Mark that this node has acquired the lock
|
||||||
|
(*locks)[grant.index] = grant.lockUID |
||||||
|
} else { |
||||||
|
locksFailed++ |
||||||
|
if !isReadLock && locksFailed > ds.dNodeCount-ds.dquorum || |
||||||
|
isReadLock && locksFailed > ds.dNodeCount-ds.dquorumReads { |
||||||
|
// We know that we are not going to get the lock anymore,
|
||||||
|
// so exit out and release any locks that did get acquired
|
||||||
|
done = true |
||||||
|
// Increment the number of grants received from the buffered channel.
|
||||||
|
i++ |
||||||
|
releaseAll(ds, locks, lockName, isReadLock) |
||||||
|
} |
||||||
|
} |
||||||
|
case <-timeout: |
||||||
|
done = true |
||||||
|
// timeout happened, maybe one of the nodes is slow, count
|
||||||
|
// number of locks to check whether we have quorum or not
|
||||||
|
if !quorumMet(locks, isReadLock, ds.dquorum, ds.dquorumReads) { |
||||||
|
releaseAll(ds, locks, lockName, isReadLock) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if done { |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Count locks in order to determine whether we have quorum or not
|
||||||
|
quorum = quorumMet(locks, isReadLock, ds.dquorum, ds.dquorumReads) |
||||||
|
|
||||||
|
// Signal that we have the quorum
|
||||||
|
wg.Done() |
||||||
|
|
||||||
|
// Wait for the other responses and immediately release the locks
|
||||||
|
// (do not add them to the locks array because the DRWMutex could
|
||||||
|
// already has been unlocked again by the original calling thread)
|
||||||
|
for ; i < ds.dNodeCount; i++ { |
||||||
|
grantToBeReleased := <-ch |
||||||
|
if grantToBeReleased.isLocked() { |
||||||
|
// release lock
|
||||||
|
sendRelease(ds, ds.restClnts[grantToBeReleased.index], lockName, grantToBeReleased.lockUID, isReadLock) |
||||||
|
} |
||||||
|
} |
||||||
|
}(isReadLock) |
||||||
|
|
||||||
|
wg.Wait() |
||||||
|
|
||||||
|
return quorum |
||||||
|
} |
||||||
|
|
||||||
|
// quorumMet determines whether we have acquired the required quorum of underlying locks or not
|
||||||
|
func quorumMet(locks *[]string, isReadLock bool, quorum, quorumReads int) bool { |
||||||
|
|
||||||
|
count := 0 |
||||||
|
for _, uid := range *locks { |
||||||
|
if isLocked(uid) { |
||||||
|
count++ |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
var metQuorum bool |
||||||
|
if isReadLock { |
||||||
|
metQuorum = count >= quorumReads |
||||||
|
} else { |
||||||
|
metQuorum = count >= quorum |
||||||
|
} |
||||||
|
|
||||||
|
return metQuorum |
||||||
|
} |
||||||
|
|
||||||
|
// releaseAll releases all locks that are marked as locked
|
||||||
|
func releaseAll(ds *Dsync, locks *[]string, lockName string, isReadLock bool) { |
||||||
|
for lock := 0; lock < ds.dNodeCount; lock++ { |
||||||
|
if isLocked((*locks)[lock]) { |
||||||
|
sendRelease(ds, ds.restClnts[lock], lockName, (*locks)[lock], isReadLock) |
||||||
|
(*locks)[lock] = "" |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Unlock unlocks the write lock.
|
||||||
|
//
|
||||||
|
// It is a run-time error if dm is not locked on entry to Unlock.
|
||||||
|
func (dm *DRWMutex) Unlock() { |
||||||
|
|
||||||
|
// create temp array on stack
|
||||||
|
locks := make([]string, dm.clnt.dNodeCount) |
||||||
|
|
||||||
|
{ |
||||||
|
dm.m.Lock() |
||||||
|
defer dm.m.Unlock() |
||||||
|
|
||||||
|
// Check if minimally a single bool is set in the writeLocks array
|
||||||
|
lockFound := false |
||||||
|
for _, uid := range dm.writeLocks { |
||||||
|
if isLocked(uid) { |
||||||
|
lockFound = true |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
if !lockFound { |
||||||
|
panic("Trying to Unlock() while no Lock() is active") |
||||||
|
} |
||||||
|
|
||||||
|
// Copy write locks to stack array
|
||||||
|
copy(locks, dm.writeLocks[:]) |
||||||
|
// Clear write locks array
|
||||||
|
dm.writeLocks = make([]string, dm.clnt.dNodeCount) |
||||||
|
} |
||||||
|
|
||||||
|
isReadLock := false |
||||||
|
unlock(dm.clnt, locks, dm.Name, isReadLock) |
||||||
|
} |
||||||
|
|
||||||
|
// RUnlock releases a read lock held on dm.
|
||||||
|
//
|
||||||
|
// It is a run-time error if dm is not locked on entry to RUnlock.
|
||||||
|
func (dm *DRWMutex) RUnlock() { |
||||||
|
|
||||||
|
// create temp array on stack
|
||||||
|
locks := make([]string, dm.clnt.dNodeCount) |
||||||
|
|
||||||
|
{ |
||||||
|
dm.m.Lock() |
||||||
|
defer dm.m.Unlock() |
||||||
|
if len(dm.readersLocks) == 0 { |
||||||
|
panic("Trying to RUnlock() while no RLock() is active") |
||||||
|
} |
||||||
|
// Copy out first element to release it first (FIFO)
|
||||||
|
copy(locks, dm.readersLocks[0][:]) |
||||||
|
// Drop first element from array
|
||||||
|
dm.readersLocks = dm.readersLocks[1:] |
||||||
|
} |
||||||
|
|
||||||
|
isReadLock := true |
||||||
|
unlock(dm.clnt, locks, dm.Name, isReadLock) |
||||||
|
} |
||||||
|
|
||||||
|
func unlock(ds *Dsync, locks []string, name string, isReadLock bool) { |
||||||
|
|
||||||
|
// We don't need to synchronously wait until we have released all the locks (or the quorum)
|
||||||
|
// (a subsequent lock will retry automatically in case it would fail to get quorum)
|
||||||
|
|
||||||
|
for index, c := range ds.restClnts { |
||||||
|
|
||||||
|
if isLocked(locks[index]) { |
||||||
|
// broadcast lock release to all nodes that granted the lock
|
||||||
|
sendRelease(ds, c, name, locks[index], isReadLock) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// sendRelease sends a release message to a node that previously granted a lock
|
||||||
|
func sendRelease(ds *Dsync, c NetLocker, name, uid string, isReadLock bool) { |
||||||
|
args := LockArgs{ |
||||||
|
UID: uid, |
||||||
|
Resource: name, |
||||||
|
} |
||||||
|
if isReadLock { |
||||||
|
if _, err := c.RUnlock(args); err != nil { |
||||||
|
log("Unable to call RUnlock", err) |
||||||
|
} |
||||||
|
} else { |
||||||
|
if _, err := c.Unlock(args); err != nil { |
||||||
|
log("Unable to call Unlock", err) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// DRLocker returns a sync.Locker interface that implements
|
||||||
|
// the Lock and Unlock methods by calling drw.RLock and drw.RUnlock.
|
||||||
|
func (dm *DRWMutex) DRLocker() sync.Locker { |
||||||
|
return (*drlocker)(dm) |
||||||
|
} |
||||||
|
|
||||||
|
type drlocker DRWMutex |
||||||
|
|
||||||
|
var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") |
||||||
|
|
||||||
|
func randString(n int) string { |
||||||
|
b := make([]rune, n) |
||||||
|
for i := range b { |
||||||
|
b[i] = letterRunes[rand.Intn(len(letterRunes))] |
||||||
|
} |
||||||
|
return string(b) |
||||||
|
} |
||||||
|
|
||||||
|
func getSource() string { |
||||||
|
var funcName string |
||||||
|
pc, filename, lineNum, ok := runtime.Caller(2) |
||||||
|
if ok { |
||||||
|
filename = path.Base(filename) |
||||||
|
funcName = runtime.FuncForPC(pc).Name() |
||||||
|
} else { |
||||||
|
filename = "<unknown>" |
||||||
|
lineNum = 0 |
||||||
|
} |
||||||
|
|
||||||
|
return fmt.Sprintf("[%s:%d:%s()]", filename, lineNum, funcName) |
||||||
|
} |
||||||
|
|
||||||
|
func (dr *drlocker) Lock() { (*DRWMutex)(dr).RLock(randString(16), getSource()) } |
||||||
|
func (dr *drlocker) Unlock() { (*DRWMutex)(dr).RUnlock() } |
||||||
@ -0,0 +1,383 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
// GOMAXPROCS=10 go test
|
||||||
|
|
||||||
|
package dsync_test |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
"runtime" |
||||||
|
"sync" |
||||||
|
"sync/atomic" |
||||||
|
"testing" |
||||||
|
"time" |
||||||
|
|
||||||
|
. "github.com/minio/minio/pkg/dsync" |
||||||
|
) |
||||||
|
|
||||||
|
const ( |
||||||
|
id = "1234-5678" |
||||||
|
source = "main.go" |
||||||
|
) |
||||||
|
|
||||||
|
func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) { |
||||||
|
|
||||||
|
drwm := NewDRWMutex(context.Background(), "simplelock", ds) |
||||||
|
|
||||||
|
if !drwm.GetRLock(id, source, time.Second) { |
||||||
|
panic("Failed to acquire read lock") |
||||||
|
} |
||||||
|
// fmt.Println("1st read lock acquired, waiting...")
|
||||||
|
|
||||||
|
if !drwm.GetRLock(id, source, time.Second) { |
||||||
|
panic("Failed to acquire read lock") |
||||||
|
} |
||||||
|
// fmt.Println("2nd read lock acquired, waiting...")
|
||||||
|
|
||||||
|
go func() { |
||||||
|
time.Sleep(2 * time.Second) |
||||||
|
drwm.RUnlock() |
||||||
|
// fmt.Println("1st read lock released, waiting...")
|
||||||
|
}() |
||||||
|
|
||||||
|
go func() { |
||||||
|
time.Sleep(3 * time.Second) |
||||||
|
drwm.RUnlock() |
||||||
|
// fmt.Println("2nd read lock released, waiting...")
|
||||||
|
}() |
||||||
|
|
||||||
|
// fmt.Println("Trying to acquire write lock, waiting...")
|
||||||
|
locked = drwm.GetLock(id, source, duration) |
||||||
|
if locked { |
||||||
|
// fmt.Println("Write lock acquired, waiting...")
|
||||||
|
time.Sleep(time.Second) |
||||||
|
|
||||||
|
drwm.Unlock() |
||||||
|
} |
||||||
|
// fmt.Println("Write lock failed due to timeout")
|
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
func TestSimpleWriteLockAcquired(t *testing.T) { |
||||||
|
locked := testSimpleWriteLock(t, 5*time.Second) |
||||||
|
|
||||||
|
expected := true |
||||||
|
if locked != expected { |
||||||
|
t.Errorf("TestSimpleWriteLockAcquired(): \nexpected %#v\ngot %#v", expected, locked) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func TestSimpleWriteLockTimedOut(t *testing.T) { |
||||||
|
locked := testSimpleWriteLock(t, time.Second) |
||||||
|
|
||||||
|
expected := false |
||||||
|
if locked != expected { |
||||||
|
t.Errorf("TestSimpleWriteLockTimedOut(): \nexpected %#v\ngot %#v", expected, locked) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) { |
||||||
|
|
||||||
|
drwm := NewDRWMutex(context.Background(), "duallock", ds) |
||||||
|
|
||||||
|
// fmt.Println("Getting initial write lock")
|
||||||
|
if !drwm.GetLock(id, source, time.Second) { |
||||||
|
panic("Failed to acquire initial write lock") |
||||||
|
} |
||||||
|
|
||||||
|
go func() { |
||||||
|
time.Sleep(2 * time.Second) |
||||||
|
drwm.Unlock() |
||||||
|
// fmt.Println("Initial write lock released, waiting...")
|
||||||
|
}() |
||||||
|
|
||||||
|
// fmt.Println("Trying to acquire 2nd write lock, waiting...")
|
||||||
|
locked = drwm.GetLock(id, source, duration) |
||||||
|
if locked { |
||||||
|
// fmt.Println("2nd write lock acquired, waiting...")
|
||||||
|
time.Sleep(time.Second) |
||||||
|
|
||||||
|
drwm.Unlock() |
||||||
|
} |
||||||
|
// fmt.Println("2nd write lock failed due to timeout")
|
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
func TestDualWriteLockAcquired(t *testing.T) { |
||||||
|
locked := testDualWriteLock(t, 5*time.Second) |
||||||
|
|
||||||
|
expected := true |
||||||
|
if locked != expected { |
||||||
|
t.Errorf("TestDualWriteLockAcquired(): \nexpected %#v\ngot %#v", expected, locked) |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
func TestDualWriteLockTimedOut(t *testing.T) { |
||||||
|
locked := testDualWriteLock(t, time.Second) |
||||||
|
|
||||||
|
expected := false |
||||||
|
if locked != expected { |
||||||
|
t.Errorf("TestDualWriteLockTimedOut(): \nexpected %#v\ngot %#v", expected, locked) |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
// Test cases below are copied 1 to 1 from sync/rwmutex_test.go (adapted to use DRWMutex)
|
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func parallelReader(m *DRWMutex, clocked, cunlock, cdone chan bool) { |
||||||
|
if m.GetRLock(id, source, time.Second) { |
||||||
|
clocked <- true |
||||||
|
<-cunlock |
||||||
|
m.RUnlock() |
||||||
|
cdone <- true |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func doTestParallelReaders(numReaders, gomaxprocs int) { |
||||||
|
runtime.GOMAXPROCS(gomaxprocs) |
||||||
|
m := NewDRWMutex(context.Background(), "test-parallel", ds) |
||||||
|
|
||||||
|
clocked := make(chan bool) |
||||||
|
cunlock := make(chan bool) |
||||||
|
cdone := make(chan bool) |
||||||
|
for i := 0; i < numReaders; i++ { |
||||||
|
go parallelReader(m, clocked, cunlock, cdone) |
||||||
|
} |
||||||
|
// Wait for all parallel RLock()s to succeed.
|
||||||
|
for i := 0; i < numReaders; i++ { |
||||||
|
<-clocked |
||||||
|
} |
||||||
|
for i := 0; i < numReaders; i++ { |
||||||
|
cunlock <- true |
||||||
|
} |
||||||
|
// Wait for the goroutines to finish.
|
||||||
|
for i := 0; i < numReaders; i++ { |
||||||
|
<-cdone |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func TestParallelReaders(t *testing.T) { |
||||||
|
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1)) |
||||||
|
doTestParallelReaders(1, 4) |
||||||
|
doTestParallelReaders(3, 4) |
||||||
|
doTestParallelReaders(4, 2) |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func reader(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool) { |
||||||
|
for i := 0; i < numIterations; i++ { |
||||||
|
if rwm.GetRLock(id, source, time.Second) { |
||||||
|
n := atomic.AddInt32(activity, 1) |
||||||
|
if n < 1 || n >= 10000 { |
||||||
|
panic(fmt.Sprintf("wlock(%d)\n", n)) |
||||||
|
} |
||||||
|
for i := 0; i < 100; i++ { |
||||||
|
} |
||||||
|
atomic.AddInt32(activity, -1) |
||||||
|
rwm.RUnlock() |
||||||
|
} |
||||||
|
} |
||||||
|
cdone <- true |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func writer(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool) { |
||||||
|
for i := 0; i < numIterations; i++ { |
||||||
|
if rwm.GetLock(id, source, time.Second) { |
||||||
|
n := atomic.AddInt32(activity, 10000) |
||||||
|
if n != 10000 { |
||||||
|
panic(fmt.Sprintf("wlock(%d)\n", n)) |
||||||
|
} |
||||||
|
for i := 0; i < 100; i++ { |
||||||
|
} |
||||||
|
atomic.AddInt32(activity, -10000) |
||||||
|
rwm.Unlock() |
||||||
|
} |
||||||
|
} |
||||||
|
cdone <- true |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func HammerRWMutex(gomaxprocs, numReaders, numIterations int) { |
||||||
|
runtime.GOMAXPROCS(gomaxprocs) |
||||||
|
// Number of active readers + 10000 * number of active writers.
|
||||||
|
var activity int32 |
||||||
|
rwm := NewDRWMutex(context.Background(), "test", ds) |
||||||
|
cdone := make(chan bool) |
||||||
|
go writer(rwm, numIterations, &activity, cdone) |
||||||
|
var i int |
||||||
|
for i = 0; i < numReaders/2; i++ { |
||||||
|
go reader(rwm, numIterations, &activity, cdone) |
||||||
|
} |
||||||
|
go writer(rwm, numIterations, &activity, cdone) |
||||||
|
for ; i < numReaders; i++ { |
||||||
|
go reader(rwm, numIterations, &activity, cdone) |
||||||
|
} |
||||||
|
// Wait for the 2 writers and all readers to finish.
|
||||||
|
for i := 0; i < 2+numReaders; i++ { |
||||||
|
<-cdone |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func TestRWMutex(t *testing.T) { |
||||||
|
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1)) |
||||||
|
n := 1000 |
||||||
|
if testing.Short() { |
||||||
|
n = 5 |
||||||
|
} |
||||||
|
HammerRWMutex(1, 1, n) |
||||||
|
HammerRWMutex(1, 3, n) |
||||||
|
HammerRWMutex(1, 10, n) |
||||||
|
HammerRWMutex(4, 1, n) |
||||||
|
HammerRWMutex(4, 3, n) |
||||||
|
HammerRWMutex(4, 10, n) |
||||||
|
HammerRWMutex(10, 1, n) |
||||||
|
HammerRWMutex(10, 3, n) |
||||||
|
HammerRWMutex(10, 10, n) |
||||||
|
HammerRWMutex(10, 5, n) |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func TestDRLocker(t *testing.T) { |
||||||
|
wl := NewDRWMutex(context.Background(), "test", ds) |
||||||
|
var rl sync.Locker |
||||||
|
wlocked := make(chan bool, 1) |
||||||
|
rlocked := make(chan bool, 1) |
||||||
|
rl = wl.DRLocker() |
||||||
|
n := 10 |
||||||
|
go func() { |
||||||
|
for i := 0; i < n; i++ { |
||||||
|
rl.Lock() |
||||||
|
rl.Lock() |
||||||
|
rlocked <- true |
||||||
|
wl.Lock(id, source) |
||||||
|
wlocked <- true |
||||||
|
} |
||||||
|
}() |
||||||
|
for i := 0; i < n; i++ { |
||||||
|
<-rlocked |
||||||
|
rl.Unlock() |
||||||
|
select { |
||||||
|
case <-wlocked: |
||||||
|
t.Fatal("RLocker() didn't read-lock it") |
||||||
|
default: |
||||||
|
} |
||||||
|
rl.Unlock() |
||||||
|
<-wlocked |
||||||
|
select { |
||||||
|
case <-rlocked: |
||||||
|
t.Fatal("RLocker() didn't respect the write lock") |
||||||
|
default: |
||||||
|
} |
||||||
|
wl.Unlock() |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func TestUnlockPanic(t *testing.T) { |
||||||
|
defer func() { |
||||||
|
if recover() == nil { |
||||||
|
t.Fatalf("unlock of unlocked RWMutex did not panic") |
||||||
|
} |
||||||
|
}() |
||||||
|
mu := NewDRWMutex(context.Background(), "test", ds) |
||||||
|
mu.Unlock() |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func TestUnlockPanic2(t *testing.T) { |
||||||
|
defer func() { |
||||||
|
if recover() == nil { |
||||||
|
t.Fatalf("unlock of unlocked RWMutex did not panic") |
||||||
|
} |
||||||
|
}() |
||||||
|
mu := NewDRWMutex(context.Background(), "test-unlock-panic-2", ds) |
||||||
|
mu.RLock(id, source) |
||||||
|
mu.Unlock() |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func TestRUnlockPanic(t *testing.T) { |
||||||
|
defer func() { |
||||||
|
if recover() == nil { |
||||||
|
t.Fatalf("read unlock of unlocked RWMutex did not panic") |
||||||
|
} |
||||||
|
}() |
||||||
|
mu := NewDRWMutex(context.Background(), "test", ds) |
||||||
|
mu.RUnlock() |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func TestRUnlockPanic2(t *testing.T) { |
||||||
|
defer func() { |
||||||
|
if recover() == nil { |
||||||
|
t.Fatalf("read unlock of unlocked RWMutex did not panic") |
||||||
|
} |
||||||
|
}() |
||||||
|
mu := NewDRWMutex(context.Background(), "test-runlock-panic-2", ds) |
||||||
|
mu.Lock(id, source) |
||||||
|
mu.RUnlock() |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func benchmarkRWMutex(b *testing.B, localWork, writeRatio int) { |
||||||
|
rwm := NewDRWMutex(context.Background(), "test", ds) |
||||||
|
b.RunParallel(func(pb *testing.PB) { |
||||||
|
foo := 0 |
||||||
|
for pb.Next() { |
||||||
|
foo++ |
||||||
|
if foo%writeRatio == 0 { |
||||||
|
rwm.Lock(id, source) |
||||||
|
rwm.Unlock() |
||||||
|
} else { |
||||||
|
rwm.RLock(id, source) |
||||||
|
for i := 0; i != localWork; i += 1 { |
||||||
|
foo *= 2 |
||||||
|
foo /= 2 |
||||||
|
} |
||||||
|
rwm.RUnlock() |
||||||
|
} |
||||||
|
} |
||||||
|
_ = foo |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func BenchmarkRWMutexWrite100(b *testing.B) { |
||||||
|
benchmarkRWMutex(b, 0, 100) |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func BenchmarkRWMutexWrite10(b *testing.B) { |
||||||
|
benchmarkRWMutex(b, 0, 10) |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func BenchmarkRWMutexWorkWrite100(b *testing.B) { |
||||||
|
benchmarkRWMutex(b, 100, 100) |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from rwmutex_test.go
|
||||||
|
func BenchmarkRWMutexWorkWrite10(b *testing.B) { |
||||||
|
benchmarkRWMutex(b, 100, 10) |
||||||
|
} |
||||||
@ -0,0 +1,103 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package dsync_test |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"sync" |
||||||
|
|
||||||
|
. "github.com/minio/minio/pkg/dsync" |
||||||
|
) |
||||||
|
|
||||||
|
const WriteLock = -1 |
||||||
|
|
||||||
|
type lockServer struct { |
||||||
|
mutex sync.Mutex |
||||||
|
// Map of locks, with negative value indicating (exclusive) write lock
|
||||||
|
// and positive values indicating number of read locks
|
||||||
|
lockMap map[string]int64 |
||||||
|
} |
||||||
|
|
||||||
|
func (l *lockServer) Lock(args *LockArgs, reply *bool) error { |
||||||
|
l.mutex.Lock() |
||||||
|
defer l.mutex.Unlock() |
||||||
|
if _, *reply = l.lockMap[args.Resource]; !*reply { |
||||||
|
l.lockMap[args.Resource] = WriteLock // No locks held on the given name, so claim write lock
|
||||||
|
} |
||||||
|
*reply = !*reply // Negate *reply to return true when lock is granted or false otherwise
|
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
func (l *lockServer) Unlock(args *LockArgs, reply *bool) error { |
||||||
|
l.mutex.Lock() |
||||||
|
defer l.mutex.Unlock() |
||||||
|
var locksHeld int64 |
||||||
|
if locksHeld, *reply = l.lockMap[args.Resource]; !*reply { // No lock is held on the given name
|
||||||
|
return fmt.Errorf("Unlock attempted on an unlocked entity: %s", args.Resource) |
||||||
|
} |
||||||
|
if *reply = locksHeld == WriteLock; !*reply { // Unless it is a write lock
|
||||||
|
return fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.Resource, locksHeld) |
||||||
|
} |
||||||
|
delete(l.lockMap, args.Resource) // Remove the write lock
|
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
const ReadLock = 1 |
||||||
|
|
||||||
|
func (l *lockServer) RLock(args *LockArgs, reply *bool) error { |
||||||
|
l.mutex.Lock() |
||||||
|
defer l.mutex.Unlock() |
||||||
|
var locksHeld int64 |
||||||
|
if locksHeld, *reply = l.lockMap[args.Resource]; !*reply { |
||||||
|
l.lockMap[args.Resource] = ReadLock // No locks held on the given name, so claim (first) read lock
|
||||||
|
*reply = true |
||||||
|
} else { |
||||||
|
if *reply = locksHeld != WriteLock; *reply { // Unless there is a write lock
|
||||||
|
l.lockMap[args.Resource] = locksHeld + ReadLock // Grant another read lock
|
||||||
|
} |
||||||
|
} |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
func (l *lockServer) RUnlock(args *LockArgs, reply *bool) error { |
||||||
|
l.mutex.Lock() |
||||||
|
defer l.mutex.Unlock() |
||||||
|
var locksHeld int64 |
||||||
|
if locksHeld, *reply = l.lockMap[args.Resource]; !*reply { // No lock is held on the given name
|
||||||
|
return fmt.Errorf("RUnlock attempted on an unlocked entity: %s", args.Resource) |
||||||
|
} |
||||||
|
if *reply = locksHeld != WriteLock; !*reply { // A write-lock is held, cannot release a read lock
|
||||||
|
return fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.Resource) |
||||||
|
} |
||||||
|
if locksHeld > ReadLock { |
||||||
|
l.lockMap[args.Resource] = locksHeld - ReadLock // Remove one of the read locks held
|
||||||
|
} else { |
||||||
|
delete(l.lockMap, args.Resource) // Remove the (last) read lock
|
||||||
|
} |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
func (l *lockServer) ForceUnlock(args *LockArgs, reply *bool) error { |
||||||
|
l.mutex.Lock() |
||||||
|
defer l.mutex.Unlock() |
||||||
|
if len(args.UID) != 0 { |
||||||
|
return fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID) |
||||||
|
} |
||||||
|
delete(l.lockMap, args.Resource) // Remove the lock (irrespective of write or read lock)
|
||||||
|
*reply = true |
||||||
|
return nil |
||||||
|
} |
||||||
@ -0,0 +1,60 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package dsync |
||||||
|
|
||||||
|
import ( |
||||||
|
"errors" |
||||||
|
"math" |
||||||
|
) |
||||||
|
|
||||||
|
// Dsync represents dsync client object which is initialized with
|
||||||
|
// authenticated clients, used to initiate lock REST calls.
|
||||||
|
type Dsync struct { |
||||||
|
// Number of nodes participating in the distributed locking.
|
||||||
|
dNodeCount int |
||||||
|
|
||||||
|
// List of rest client objects, one per lock server.
|
||||||
|
restClnts []NetLocker |
||||||
|
|
||||||
|
// Simple majority based quorum, set to dNodeCount/2+1
|
||||||
|
dquorum int |
||||||
|
|
||||||
|
// Simple quorum for read operations, set to dNodeCount/2
|
||||||
|
dquorumReads int |
||||||
|
} |
||||||
|
|
||||||
|
// New - initializes a new dsync object with input restClnts.
|
||||||
|
func New(restClnts []NetLocker) (*Dsync, error) { |
||||||
|
if len(restClnts) < 2 { |
||||||
|
return nil, errors.New("Dsync is not designed for less than 2 nodes") |
||||||
|
} else if len(restClnts) > 32 { |
||||||
|
return nil, errors.New("Dsync is not designed for more than 32 nodes") |
||||||
|
} |
||||||
|
|
||||||
|
ds := &Dsync{} |
||||||
|
ds.dNodeCount = len(restClnts) |
||||||
|
|
||||||
|
// With odd number of nodes, write and read quorum is basically the same
|
||||||
|
ds.dquorum = int(ds.dNodeCount/2) + 1 |
||||||
|
ds.dquorumReads = int(math.Ceil(float64(ds.dNodeCount) / 2.0)) |
||||||
|
|
||||||
|
// Initialize node name and rest path for each NetLocker object.
|
||||||
|
ds.restClnts = make([]NetLocker, ds.dNodeCount) |
||||||
|
copy(ds.restClnts, restClnts) |
||||||
|
|
||||||
|
return ds, nil |
||||||
|
} |
||||||
@ -0,0 +1,58 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2018 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
// GOMAXPROCS=10 go test
|
||||||
|
|
||||||
|
package dsync |
||||||
|
|
||||||
|
import "testing" |
||||||
|
|
||||||
|
// Tests dsync.New
|
||||||
|
func TestNew(t *testing.T) { |
||||||
|
nclnts := make([]NetLocker, 33) |
||||||
|
if _, err := New(nclnts); err == nil { |
||||||
|
t.Fatal("Should have failed") |
||||||
|
} |
||||||
|
|
||||||
|
nclnts = make([]NetLocker, 1) |
||||||
|
if _, err := New(nclnts); err == nil { |
||||||
|
t.Fatal("Should have failed") |
||||||
|
} |
||||||
|
|
||||||
|
nclnts = make([]NetLocker, 2) |
||||||
|
nds, err := New(nclnts) |
||||||
|
if err != nil { |
||||||
|
t.Fatal("Should pass", err) |
||||||
|
} |
||||||
|
|
||||||
|
if nds.dquorumReads != 1 { |
||||||
|
t.Fatalf("Unexpected read quorum values expected 1, got %d", nds.dquorumReads) |
||||||
|
} |
||||||
|
|
||||||
|
if nds.dquorum != 2 { |
||||||
|
t.Fatalf("Unexpected quorum values expected 2, got %d", nds.dquorum) |
||||||
|
} |
||||||
|
|
||||||
|
nclnts = make([]NetLocker, 3) |
||||||
|
nds, err = New(nclnts) |
||||||
|
if err != nil { |
||||||
|
t.Fatal("Should pass", err) |
||||||
|
} |
||||||
|
|
||||||
|
if nds.dquorumReads != nds.dquorum { |
||||||
|
t.Fatalf("Unexpected quorum values for odd nodes we expect read %d and write %d quorum to be same", nds.dquorumReads, nds.dquorum) |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,361 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
// GOMAXPROCS=10 go test
|
||||||
|
|
||||||
|
package dsync_test |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
"log" |
||||||
|
"math/rand" |
||||||
|
"net" |
||||||
|
"net/http" |
||||||
|
"net/rpc" |
||||||
|
"os" |
||||||
|
"strconv" |
||||||
|
"sync" |
||||||
|
"testing" |
||||||
|
"time" |
||||||
|
|
||||||
|
. "github.com/minio/minio/pkg/dsync" |
||||||
|
) |
||||||
|
|
||||||
|
var ds *Dsync |
||||||
|
var rpcPaths []string // list of rpc paths where lock server is serving.
|
||||||
|
|
||||||
|
func startRPCServers(nodes []string) { |
||||||
|
for i := range nodes { |
||||||
|
server := rpc.NewServer() |
||||||
|
server.RegisterName("Dsync", &lockServer{ |
||||||
|
mutex: sync.Mutex{}, |
||||||
|
lockMap: make(map[string]int64), |
||||||
|
}) |
||||||
|
// For some reason the registration paths need to be different (even for different server objs)
|
||||||
|
server.HandleHTTP(rpcPaths[i], fmt.Sprintf("%s-debug", rpcPaths[i])) |
||||||
|
l, e := net.Listen("tcp", ":"+strconv.Itoa(i+12345)) |
||||||
|
if e != nil { |
||||||
|
log.Fatal("listen error:", e) |
||||||
|
} |
||||||
|
go http.Serve(l, nil) |
||||||
|
} |
||||||
|
|
||||||
|
// Let servers start
|
||||||
|
time.Sleep(10 * time.Millisecond) |
||||||
|
} |
||||||
|
|
||||||
|
// TestMain initializes the testing framework
|
||||||
|
func TestMain(m *testing.M) { |
||||||
|
const rpcPath = "/dsync" |
||||||
|
|
||||||
|
rand.Seed(time.Now().UTC().UnixNano()) |
||||||
|
|
||||||
|
nodes := make([]string, 4) // list of node IP addrs or hostname with ports.
|
||||||
|
for i := range nodes { |
||||||
|
nodes[i] = fmt.Sprintf("127.0.0.1:%d", i+12345) |
||||||
|
} |
||||||
|
for i := range nodes { |
||||||
|
rpcPaths = append(rpcPaths, rpcPath+"-"+strconv.Itoa(i)) |
||||||
|
} |
||||||
|
|
||||||
|
// Initialize net/rpc clients for dsync.
|
||||||
|
var clnts []NetLocker |
||||||
|
for i := 0; i < len(nodes); i++ { |
||||||
|
clnts = append(clnts, newClient(nodes[i], rpcPaths[i])) |
||||||
|
} |
||||||
|
|
||||||
|
var err error |
||||||
|
ds, err = New(clnts) |
||||||
|
if err != nil { |
||||||
|
log.Fatalf("set nodes failed with %v", err) |
||||||
|
} |
||||||
|
|
||||||
|
startRPCServers(nodes) |
||||||
|
|
||||||
|
os.Exit(m.Run()) |
||||||
|
} |
||||||
|
|
||||||
|
func TestSimpleLock(t *testing.T) { |
||||||
|
|
||||||
|
dm := NewDRWMutex(context.Background(), "test", ds) |
||||||
|
|
||||||
|
dm.Lock(id, source) |
||||||
|
|
||||||
|
// fmt.Println("Lock acquired, waiting...")
|
||||||
|
time.Sleep(2500 * time.Millisecond) |
||||||
|
|
||||||
|
dm.Unlock() |
||||||
|
} |
||||||
|
|
||||||
|
func TestSimpleLockUnlockMultipleTimes(t *testing.T) { |
||||||
|
|
||||||
|
dm := NewDRWMutex(context.Background(), "test", ds) |
||||||
|
|
||||||
|
dm.Lock(id, source) |
||||||
|
time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) |
||||||
|
dm.Unlock() |
||||||
|
|
||||||
|
dm.Lock(id, source) |
||||||
|
time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) |
||||||
|
dm.Unlock() |
||||||
|
|
||||||
|
dm.Lock(id, source) |
||||||
|
time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) |
||||||
|
dm.Unlock() |
||||||
|
|
||||||
|
dm.Lock(id, source) |
||||||
|
time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) |
||||||
|
dm.Unlock() |
||||||
|
|
||||||
|
dm.Lock(id, source) |
||||||
|
time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) |
||||||
|
dm.Unlock() |
||||||
|
} |
||||||
|
|
||||||
|
// Test two locks for same resource, one succeeds, one fails (after timeout)
|
||||||
|
func TestTwoSimultaneousLocksForSameResource(t *testing.T) { |
||||||
|
|
||||||
|
dm1st := NewDRWMutex(context.Background(), "aap", ds) |
||||||
|
dm2nd := NewDRWMutex(context.Background(), "aap", ds) |
||||||
|
|
||||||
|
dm1st.Lock(id, source) |
||||||
|
|
||||||
|
// Release lock after 10 seconds
|
||||||
|
go func() { |
||||||
|
time.Sleep(10 * time.Second) |
||||||
|
// fmt.Println("Unlocking dm1")
|
||||||
|
|
||||||
|
dm1st.Unlock() |
||||||
|
}() |
||||||
|
|
||||||
|
dm2nd.Lock(id, source) |
||||||
|
|
||||||
|
// fmt.Printf("2nd lock obtained after 1st lock is released\n")
|
||||||
|
time.Sleep(2500 * time.Millisecond) |
||||||
|
|
||||||
|
dm2nd.Unlock() |
||||||
|
} |
||||||
|
|
||||||
|
// Test three locks for same resource, one succeeds, one fails (after timeout)
|
||||||
|
func TestThreeSimultaneousLocksForSameResource(t *testing.T) { |
||||||
|
|
||||||
|
dm1st := NewDRWMutex(context.Background(), "aap", ds) |
||||||
|
dm2nd := NewDRWMutex(context.Background(), "aap", ds) |
||||||
|
dm3rd := NewDRWMutex(context.Background(), "aap", ds) |
||||||
|
|
||||||
|
dm1st.Lock(id, source) |
||||||
|
|
||||||
|
// Release lock after 10 seconds
|
||||||
|
go func() { |
||||||
|
time.Sleep(10 * time.Second) |
||||||
|
// fmt.Println("Unlocking dm1")
|
||||||
|
|
||||||
|
dm1st.Unlock() |
||||||
|
}() |
||||||
|
|
||||||
|
var wg sync.WaitGroup |
||||||
|
wg.Add(2) |
||||||
|
|
||||||
|
go func() { |
||||||
|
defer wg.Done() |
||||||
|
|
||||||
|
dm2nd.Lock(id, source) |
||||||
|
|
||||||
|
// Release lock after 10 seconds
|
||||||
|
go func() { |
||||||
|
time.Sleep(2500 * time.Millisecond) |
||||||
|
// fmt.Println("Unlocking dm2")
|
||||||
|
|
||||||
|
dm2nd.Unlock() |
||||||
|
}() |
||||||
|
|
||||||
|
dm3rd.Lock(id, source) |
||||||
|
|
||||||
|
// fmt.Printf("3rd lock obtained after 1st & 2nd locks are released\n")
|
||||||
|
time.Sleep(2500 * time.Millisecond) |
||||||
|
|
||||||
|
dm3rd.Unlock() |
||||||
|
}() |
||||||
|
|
||||||
|
go func() { |
||||||
|
defer wg.Done() |
||||||
|
|
||||||
|
dm3rd.Lock(id, source) |
||||||
|
|
||||||
|
// Release lock after 10 seconds
|
||||||
|
go func() { |
||||||
|
time.Sleep(2500 * time.Millisecond) |
||||||
|
// fmt.Println("Unlocking dm3")
|
||||||
|
|
||||||
|
dm3rd.Unlock() |
||||||
|
}() |
||||||
|
|
||||||
|
dm2nd.Lock(id, source) |
||||||
|
|
||||||
|
// fmt.Printf("2nd lock obtained after 1st & 3rd locks are released\n")
|
||||||
|
time.Sleep(2500 * time.Millisecond) |
||||||
|
|
||||||
|
dm2nd.Unlock() |
||||||
|
}() |
||||||
|
|
||||||
|
wg.Wait() |
||||||
|
} |
||||||
|
|
||||||
|
// Test two locks for different resources, both succeed
|
||||||
|
func TestTwoSimultaneousLocksForDifferentResources(t *testing.T) { |
||||||
|
|
||||||
|
dm1 := NewDRWMutex(context.Background(), "aap", ds) |
||||||
|
dm2 := NewDRWMutex(context.Background(), "noot", ds) |
||||||
|
|
||||||
|
dm1.Lock(id, source) |
||||||
|
dm2.Lock(id, source) |
||||||
|
|
||||||
|
// fmt.Println("Both locks acquired, waiting...")
|
||||||
|
time.Sleep(2500 * time.Millisecond) |
||||||
|
|
||||||
|
dm1.Unlock() |
||||||
|
dm2.Unlock() |
||||||
|
|
||||||
|
time.Sleep(10 * time.Millisecond) |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from mutex_test.go
|
||||||
|
func HammerMutex(m *DRWMutex, loops int, cdone chan bool) { |
||||||
|
for i := 0; i < loops; i++ { |
||||||
|
m.Lock(id, source) |
||||||
|
m.Unlock() |
||||||
|
} |
||||||
|
cdone <- true |
||||||
|
} |
||||||
|
|
||||||
|
// Borrowed from mutex_test.go
|
||||||
|
func TestMutex(t *testing.T) { |
||||||
|
c := make(chan bool) |
||||||
|
m := NewDRWMutex(context.Background(), "test", ds) |
||||||
|
for i := 0; i < 10; i++ { |
||||||
|
go HammerMutex(m, 1000, c) |
||||||
|
} |
||||||
|
for i := 0; i < 10; i++ { |
||||||
|
<-c |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func BenchmarkMutexUncontended(b *testing.B) { |
||||||
|
type PaddedMutex struct { |
||||||
|
DRWMutex |
||||||
|
pad [128]uint8 |
||||||
|
} |
||||||
|
b.RunParallel(func(pb *testing.PB) { |
||||||
|
var mu PaddedMutex |
||||||
|
for pb.Next() { |
||||||
|
mu.Lock(id, source) |
||||||
|
mu.Unlock() |
||||||
|
} |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
func benchmarkMutex(b *testing.B, slack, work bool) { |
||||||
|
mu := NewDRWMutex(context.Background(), "", ds) |
||||||
|
if slack { |
||||||
|
b.SetParallelism(10) |
||||||
|
} |
||||||
|
b.RunParallel(func(pb *testing.PB) { |
||||||
|
foo := 0 |
||||||
|
for pb.Next() { |
||||||
|
mu.Lock(id, source) |
||||||
|
mu.Unlock() |
||||||
|
if work { |
||||||
|
for i := 0; i < 100; i++ { |
||||||
|
foo *= 2 |
||||||
|
foo /= 2 |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
_ = foo |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
func BenchmarkMutex(b *testing.B) { |
||||||
|
benchmarkMutex(b, false, false) |
||||||
|
} |
||||||
|
|
||||||
|
func BenchmarkMutexSlack(b *testing.B) { |
||||||
|
benchmarkMutex(b, true, false) |
||||||
|
} |
||||||
|
|
||||||
|
func BenchmarkMutexWork(b *testing.B) { |
||||||
|
benchmarkMutex(b, false, true) |
||||||
|
} |
||||||
|
|
||||||
|
func BenchmarkMutexWorkSlack(b *testing.B) { |
||||||
|
benchmarkMutex(b, true, true) |
||||||
|
} |
||||||
|
|
||||||
|
func BenchmarkMutexNoSpin(b *testing.B) { |
||||||
|
// This benchmark models a situation where spinning in the mutex should be
|
||||||
|
// non-profitable and allows to confirm that spinning does not do harm.
|
||||||
|
// To achieve this we create excess of goroutines most of which do local work.
|
||||||
|
// These goroutines yield during local work, so that switching from
|
||||||
|
// a blocked goroutine to other goroutines is profitable.
|
||||||
|
// As a matter of fact, this benchmark still triggers some spinning in the mutex.
|
||||||
|
m := NewDRWMutex(context.Background(), "", ds) |
||||||
|
var acc0, acc1 uint64 |
||||||
|
b.SetParallelism(4) |
||||||
|
b.RunParallel(func(pb *testing.PB) { |
||||||
|
c := make(chan bool) |
||||||
|
var data [4 << 10]uint64 |
||||||
|
for i := 0; pb.Next(); i++ { |
||||||
|
if i%4 == 0 { |
||||||
|
m.Lock(id, source) |
||||||
|
acc0 -= 100 |
||||||
|
acc1 += 100 |
||||||
|
m.Unlock() |
||||||
|
} else { |
||||||
|
for i := 0; i < len(data); i += 4 { |
||||||
|
data[i]++ |
||||||
|
} |
||||||
|
// Elaborate way to say runtime.Gosched
|
||||||
|
// that does not put the goroutine onto global runq.
|
||||||
|
go func() { |
||||||
|
c <- true |
||||||
|
}() |
||||||
|
<-c |
||||||
|
} |
||||||
|
} |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
func BenchmarkMutexSpin(b *testing.B) { |
||||||
|
// This benchmark models a situation where spinning in the mutex should be
|
||||||
|
// profitable. To achieve this we create a goroutine per-proc.
|
||||||
|
// These goroutines access considerable amount of local data so that
|
||||||
|
// unnecessary rescheduling is penalized by cache misses.
|
||||||
|
m := NewDRWMutex(context.Background(), "", ds) |
||||||
|
var acc0, acc1 uint64 |
||||||
|
b.RunParallel(func(pb *testing.PB) { |
||||||
|
var data [16 << 10]uint64 |
||||||
|
for i := 0; pb.Next(); i++ { |
||||||
|
m.Lock(id, source) |
||||||
|
acc0 -= 100 |
||||||
|
acc1 += 100 |
||||||
|
m.Unlock() |
||||||
|
for i := 0; i < len(data); i += 4 { |
||||||
|
data[i]++ |
||||||
|
} |
||||||
|
} |
||||||
|
}) |
||||||
|
} |
||||||
@ -0,0 +1,142 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2017 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package dsync |
||||||
|
|
||||||
|
import ( |
||||||
|
"math/rand" |
||||||
|
"sync" |
||||||
|
"time" |
||||||
|
) |
||||||
|
|
||||||
|
// lockedRandSource provides protected rand source, implements rand.Source interface.
|
||||||
|
type lockedRandSource struct { |
||||||
|
lk sync.Mutex |
||||||
|
src rand.Source |
||||||
|
} |
||||||
|
|
||||||
|
// Int63 returns a non-negative pseudo-random 63-bit integer as an
|
||||||
|
// int64.
|
||||||
|
func (r *lockedRandSource) Int63() (n int64) { |
||||||
|
r.lk.Lock() |
||||||
|
n = r.src.Int63() |
||||||
|
r.lk.Unlock() |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
// Seed uses the provided seed value to initialize the generator to a
|
||||||
|
// deterministic state.
|
||||||
|
func (r *lockedRandSource) Seed(seed int64) { |
||||||
|
r.lk.Lock() |
||||||
|
r.src.Seed(seed) |
||||||
|
r.lk.Unlock() |
||||||
|
} |
||||||
|
|
||||||
|
// MaxJitter will randomize over the full exponential backoff time
|
||||||
|
const MaxJitter = 1.0 |
||||||
|
|
||||||
|
// NoJitter disables the use of jitter for randomizing the
|
||||||
|
// exponential backoff time
|
||||||
|
const NoJitter = 0.0 |
||||||
|
|
||||||
|
// Global random source for fetching random values.
|
||||||
|
var globalRandomSource = rand.New(&lockedRandSource{ |
||||||
|
src: rand.NewSource(time.Now().UTC().UnixNano()), |
||||||
|
}) |
||||||
|
|
||||||
|
// newRetryTimerJitter creates a timer with exponentially increasing delays
|
||||||
|
// until the maximum retry attempts are reached. - this function is a fully
|
||||||
|
// configurable version, meant for only advanced use cases. For the most part
|
||||||
|
// one should use newRetryTimerSimple and newRetryTimer.
|
||||||
|
func newRetryTimerWithJitter(unit time.Duration, cap time.Duration, jitter float64, doneCh <-chan struct{}) <-chan int { |
||||||
|
attemptCh := make(chan int) |
||||||
|
|
||||||
|
// normalize jitter to the range [0, 1.0]
|
||||||
|
if jitter < NoJitter { |
||||||
|
jitter = NoJitter |
||||||
|
} |
||||||
|
if jitter > MaxJitter { |
||||||
|
jitter = MaxJitter |
||||||
|
} |
||||||
|
|
||||||
|
// computes the exponential backoff duration according to
|
||||||
|
// https://www.awsarchitectureblog.com/2015/03/backoff.html
|
||||||
|
exponentialBackoffWait := func(attempt int) time.Duration { |
||||||
|
// 1<<uint(attempt) below could overflow, so limit the value of attempt
|
||||||
|
maxAttempt := 30 |
||||||
|
if attempt > maxAttempt { |
||||||
|
attempt = maxAttempt |
||||||
|
} |
||||||
|
//sleep = random_between(0, min(cap, base * 2 ** attempt))
|
||||||
|
sleep := unit * time.Duration(1<<uint(attempt)) |
||||||
|
if sleep > cap { |
||||||
|
sleep = cap |
||||||
|
} |
||||||
|
if jitter != NoJitter { |
||||||
|
sleep -= time.Duration(globalRandomSource.Float64() * float64(sleep) * jitter) |
||||||
|
} |
||||||
|
return sleep |
||||||
|
} |
||||||
|
|
||||||
|
go func() { |
||||||
|
defer close(attemptCh) |
||||||
|
nextBackoff := 0 |
||||||
|
// Channel used to signal after the expiry of backoff wait seconds.
|
||||||
|
var timer *time.Timer |
||||||
|
for { |
||||||
|
select { // Attempts starts.
|
||||||
|
case attemptCh <- nextBackoff: |
||||||
|
nextBackoff++ |
||||||
|
case <-doneCh: |
||||||
|
// Stop the routine.
|
||||||
|
return |
||||||
|
} |
||||||
|
timer = time.NewTimer(exponentialBackoffWait(nextBackoff)) |
||||||
|
// wait till next backoff time or till doneCh gets a message.
|
||||||
|
select { |
||||||
|
case <-timer.C: |
||||||
|
case <-doneCh: |
||||||
|
// stop the timer and return.
|
||||||
|
timer.Stop() |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
}() |
||||||
|
|
||||||
|
// Start reading..
|
||||||
|
return attemptCh |
||||||
|
} |
||||||
|
|
||||||
|
// Default retry constants.
|
||||||
|
const ( |
||||||
|
defaultRetryUnit = time.Second // 1 second.
|
||||||
|
defaultRetryCap = 1 * time.Second // 1 second.
|
||||||
|
) |
||||||
|
|
||||||
|
// newRetryTimer creates a timer with exponentially increasing delays
|
||||||
|
// until the maximum retry attempts are reached. - this function provides
|
||||||
|
// resulting retry values to be of maximum jitter.
|
||||||
|
func newRetryTimer(unit time.Duration, cap time.Duration, doneCh <-chan struct{}) <-chan int { |
||||||
|
return newRetryTimerWithJitter(unit, cap, MaxJitter, doneCh) |
||||||
|
} |
||||||
|
|
||||||
|
// newRetryTimerSimple creates a timer with exponentially increasing delays
|
||||||
|
// until the maximum retry attempts are reached. - this function is a
|
||||||
|
// simpler version with all default values.
|
||||||
|
func newRetryTimerSimple(doneCh <-chan struct{}) <-chan int { |
||||||
|
return newRetryTimerWithJitter(defaultRetryUnit, defaultRetryCap, MaxJitter, doneCh) |
||||||
|
} |
||||||
@ -0,0 +1,82 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2017 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package dsync |
||||||
|
|
||||||
|
import ( |
||||||
|
"testing" |
||||||
|
"time" |
||||||
|
) |
||||||
|
|
||||||
|
// Tests for retry timer.
|
||||||
|
func TestRetryTimerSimple(t *testing.T) { |
||||||
|
doneCh := make(chan struct{}) |
||||||
|
attemptCh := newRetryTimerSimple(doneCh) |
||||||
|
i := <-attemptCh |
||||||
|
if i != 0 { |
||||||
|
close(doneCh) |
||||||
|
t.Fatalf("Invalid attempt counter returned should be 0, found %d instead", i) |
||||||
|
} |
||||||
|
i = <-attemptCh |
||||||
|
if i <= 0 { |
||||||
|
close(doneCh) |
||||||
|
t.Fatalf("Invalid attempt counter returned should be greater than 0, found %d instead", i) |
||||||
|
} |
||||||
|
close(doneCh) |
||||||
|
_, ok := <-attemptCh |
||||||
|
if ok { |
||||||
|
t.Fatal("Attempt counter should be closed") |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Test retry time with no jitter.
|
||||||
|
func TestRetryTimerWithNoJitter(t *testing.T) { |
||||||
|
doneCh := make(chan struct{}) |
||||||
|
// No jitter
|
||||||
|
attemptCh := newRetryTimerWithJitter(time.Millisecond, 5*time.Millisecond, NoJitter, doneCh) |
||||||
|
i := <-attemptCh |
||||||
|
if i != 0 { |
||||||
|
close(doneCh) |
||||||
|
t.Fatalf("Invalid attempt counter returned should be 0, found %d instead", i) |
||||||
|
} |
||||||
|
// Loop through the maximum possible attempt.
|
||||||
|
for i = range attemptCh { |
||||||
|
if i == 30 { |
||||||
|
close(doneCh) |
||||||
|
} |
||||||
|
} |
||||||
|
_, ok := <-attemptCh |
||||||
|
if ok { |
||||||
|
t.Fatal("Attempt counter should be closed") |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Test retry time with Jitter greater than MaxJitter.
|
||||||
|
func TestRetryTimerWithJitter(t *testing.T) { |
||||||
|
doneCh := make(chan struct{}) |
||||||
|
// Jitter will be set back to 1.0
|
||||||
|
attemptCh := newRetryTimerWithJitter(time.Second, 30*time.Second, 2.0, doneCh) |
||||||
|
i := <-attemptCh |
||||||
|
if i != 0 { |
||||||
|
close(doneCh) |
||||||
|
t.Fatalf("Invalid attempt counter returned should be 0, found %d instead", i) |
||||||
|
} |
||||||
|
close(doneCh) |
||||||
|
_, ok := <-attemptCh |
||||||
|
if ok { |
||||||
|
t.Fatal("Attempt counter should be closed") |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,111 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package dsync_test |
||||||
|
|
||||||
|
import ( |
||||||
|
"net/rpc" |
||||||
|
"sync" |
||||||
|
|
||||||
|
. "github.com/minio/minio/pkg/dsync" |
||||||
|
) |
||||||
|
|
||||||
|
// ReconnectRPCClient is a wrapper type for rpc.Client which provides reconnect on first failure.
|
||||||
|
type ReconnectRPCClient struct { |
||||||
|
mutex sync.Mutex |
||||||
|
rpc *rpc.Client |
||||||
|
addr string |
||||||
|
endpoint string |
||||||
|
} |
||||||
|
|
||||||
|
// newClient constructs a ReconnectRPCClient object with addr and endpoint initialized.
|
||||||
|
// It _doesn't_ connect to the remote endpoint. See Call method to see when the
|
||||||
|
// connect happens.
|
||||||
|
func newClient(addr, endpoint string) NetLocker { |
||||||
|
return &ReconnectRPCClient{ |
||||||
|
addr: addr, |
||||||
|
endpoint: endpoint, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Close closes the underlying socket file descriptor.
|
||||||
|
func (rpcClient *ReconnectRPCClient) Close() error { |
||||||
|
rpcClient.mutex.Lock() |
||||||
|
defer rpcClient.mutex.Unlock() |
||||||
|
// If rpc client has not connected yet there is nothing to close.
|
||||||
|
if rpcClient.rpc == nil { |
||||||
|
return nil |
||||||
|
} |
||||||
|
// Reset rpcClient.rpc to allow for subsequent calls to use a new
|
||||||
|
// (socket) connection.
|
||||||
|
clnt := rpcClient.rpc |
||||||
|
rpcClient.rpc = nil |
||||||
|
return clnt.Close() |
||||||
|
} |
||||||
|
|
||||||
|
// Call makes a RPC call to the remote endpoint using the default codec, namely encoding/gob.
|
||||||
|
func (rpcClient *ReconnectRPCClient) Call(serviceMethod string, args interface{}, reply interface{}) (err error) { |
||||||
|
rpcClient.mutex.Lock() |
||||||
|
defer rpcClient.mutex.Unlock() |
||||||
|
dialCall := func() error { |
||||||
|
// If the rpc.Client is nil, we attempt to (re)connect with the remote endpoint.
|
||||||
|
if rpcClient.rpc == nil { |
||||||
|
clnt, derr := rpc.DialHTTPPath("tcp", rpcClient.addr, rpcClient.endpoint) |
||||||
|
if derr != nil { |
||||||
|
return derr |
||||||
|
} |
||||||
|
rpcClient.rpc = clnt |
||||||
|
} |
||||||
|
// If the RPC fails due to a network-related error, then we reset
|
||||||
|
// rpc.Client for a subsequent reconnect.
|
||||||
|
return rpcClient.rpc.Call(serviceMethod, args, reply) |
||||||
|
} |
||||||
|
if err = dialCall(); err == rpc.ErrShutdown { |
||||||
|
rpcClient.rpc.Close() |
||||||
|
rpcClient.rpc = nil |
||||||
|
err = dialCall() |
||||||
|
} |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
func (rpcClient *ReconnectRPCClient) RLock(args LockArgs) (status bool, err error) { |
||||||
|
err = rpcClient.Call("Dsync.RLock", &args, &status) |
||||||
|
return status, err |
||||||
|
} |
||||||
|
|
||||||
|
func (rpcClient *ReconnectRPCClient) Lock(args LockArgs) (status bool, err error) { |
||||||
|
err = rpcClient.Call("Dsync.Lock", &args, &status) |
||||||
|
return status, err |
||||||
|
} |
||||||
|
|
||||||
|
func (rpcClient *ReconnectRPCClient) RUnlock(args LockArgs) (status bool, err error) { |
||||||
|
err = rpcClient.Call("Dsync.RUnlock", &args, &status) |
||||||
|
return status, err |
||||||
|
} |
||||||
|
|
||||||
|
func (rpcClient *ReconnectRPCClient) Unlock(args LockArgs) (status bool, err error) { |
||||||
|
err = rpcClient.Call("Dsync.Unlock", &args, &status) |
||||||
|
return status, err |
||||||
|
} |
||||||
|
|
||||||
|
func (rpcClient *ReconnectRPCClient) ForceUnlock(args LockArgs) (status bool, err error) { |
||||||
|
err = rpcClient.Call("Dsync.ForceUnlock", &args, &status) |
||||||
|
return status, err |
||||||
|
} |
||||||
|
|
||||||
|
func (rpcClient *ReconnectRPCClient) String() string { |
||||||
|
return "http://" + rpcClient.addr + "/" + rpcClient.endpoint |
||||||
|
} |
||||||
@ -0,0 +1,59 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package dsync |
||||||
|
|
||||||
|
// LockArgs is minimal required values for any dsync compatible lock operation.
|
||||||
|
type LockArgs struct { |
||||||
|
// Unique ID of lock/unlock request.
|
||||||
|
UID string |
||||||
|
|
||||||
|
// Resource contains a entity to be locked/unlocked.
|
||||||
|
Resource string |
||||||
|
|
||||||
|
// Source contains the line number, function and file name of the code
|
||||||
|
// on the client node that requested the lock.
|
||||||
|
Source string |
||||||
|
} |
||||||
|
|
||||||
|
// NetLocker is dsync compatible locker interface.
|
||||||
|
type NetLocker interface { |
||||||
|
// Do read lock for given LockArgs. It should return
|
||||||
|
// * a boolean to indicate success/failure of the operation
|
||||||
|
// * an error on failure of lock request operation.
|
||||||
|
RLock(args LockArgs) (bool, error) |
||||||
|
|
||||||
|
// Do write lock for given LockArgs. It should return
|
||||||
|
// * a boolean to indicate success/failure of the operation
|
||||||
|
// * an error on failure of lock request operation.
|
||||||
|
Lock(args LockArgs) (bool, error) |
||||||
|
|
||||||
|
// Do read unlock for given LockArgs. It should return
|
||||||
|
// * a boolean to indicate success/failure of the operation
|
||||||
|
// * an error on failure of unlock request operation.
|
||||||
|
RUnlock(args LockArgs) (bool, error) |
||||||
|
|
||||||
|
// Do write unlock for given LockArgs. It should return
|
||||||
|
// * a boolean to indicate success/failure of the operation
|
||||||
|
// * an error on failure of unlock request operation.
|
||||||
|
Unlock(args LockArgs) (bool, error) |
||||||
|
|
||||||
|
// Returns underlying endpoint of this lock client instance.
|
||||||
|
String() string |
||||||
|
|
||||||
|
// Close closes any underlying connection to the service endpoint
|
||||||
|
Close() error |
||||||
|
} |
||||||
Loading…
Reference in new issue