Tracing locking errors for better debugging (#3140)

cmd/lock-instrument.go

@@ -88,13 +88,13 @@ func (l LockInfoOriginNotFound) Error() string {
 		l.lockOrigin, l.volume, l.path, l.opsID)
 }
 
-// LockInfoVolPathMssing - Error interface. Returned when the info the
-type LockInfoVolPathMssing struct {
+// LockInfoVolPathMissing - Error interface. Returned when the info the
+type LockInfoVolPathMissing struct {
 	volume string
 	path   string
 }
 
-func (l LockInfoVolPathMssing) Error() string {
+func (l LockInfoVolPathMissing) Error() string {
 	return fmt.Sprintf("No entry in debug Lock Map for Volume: %s, path: %s.", l.volume, l.path)
 }
 
@@ -152,26 +152,26 @@ func (n *nsLockMap) statusBlockedToRunning(param nsParam, lockOrigin, opsID stri
 	debugLockMap, ok := n.debugLockMap[param]
 	if !ok {
 		// The lock state info foe given <volume, path> pair should already exist.
-		// If not return `LockInfoVolPathMssing`.
-		return LockInfoVolPathMssing{param.volume, param.path}
+		// If not return `LockInfoVolPathMissing`.
+		return traceError(LockInfoVolPathMissing{param.volume, param.path})
 	}
 	//  ``debugLockMap`` entry containing lock info for `param <volume, path>` is `nil`.
 	if debugLockMap == nil {
-		return errLockNotInitialized
+		return traceError(errLockNotInitialized)
 	}
 	lockInfo, ok := n.debugLockMap[param].lockInfo[opsID]
 	if !ok {
 		// The lock info entry for given `opsID` should already exist for given <volume, path> pair.
 		// If not return `LockInfoOpsIDNotFound`.
-		return LockInfoOpsIDNotFound{param.volume, param.path, opsID}
+		return traceError(LockInfoOpsIDNotFound{param.volume, param.path, opsID})
 	}
 	// The entry for the lock origined at `lockOrigin` should already exist. If not return `LockInfoOriginNotFound`.
 	if lockInfo.lockOrigin != lockOrigin {
-		return LockInfoOriginNotFound{param.volume, param.path, opsID, lockOrigin}
+		return traceError(LockInfoOriginNotFound{param.volume, param.path, opsID, lockOrigin})
 	}
 	// Status of the lock should already be set to "Blocked". If not return `LockInfoStateNotBlocked`.
 	if lockInfo.status != blockedStatus {
-		return LockInfoStateNotBlocked{param.volume, param.path, opsID}
+		return traceError(LockInfoStateNotBlocked{param.volume, param.path, opsID})
 	}
 	// All checks finished. Changing the status of the operation from blocked to running and updating the time.
 	n.debugLockMap[param].lockInfo[opsID] = newLockInfo
@@ -230,7 +230,7 @@ func (n *nsLockMap) statusNoneToBlocked(param nsParam, lockOrigin, opsID string,
 func (n *nsLockMap) deleteLockInfoEntryForVolumePath(param nsParam) error {
 	// delete the lock info for the given operation.
 	if _, found := n.debugLockMap[param]; !found {
-		return LockInfoVolPathMssing{param.volume, param.path}
+		return traceError(LockInfoVolPathMissing{param.volume, param.path})
 	}
 	// Remove from the map if there are no more references for the given (volume,path) pair.
 	delete(n.debugLockMap, param)
@@ -244,14 +244,14 @@ func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, opsID string) error
 	// delete the lock info for the given operation.
 	infoMap, found := n.debugLockMap[param]
 	if !found {
-		return LockInfoVolPathMssing{param.volume, param.path}
+		return traceError(LockInfoVolPathMissing{param.volume, param.path})
 	}
 	// The opertion finished holding the lock on the resource, remove
 	// the entry for the given operation with the operation ID.
 	_, foundInfo := infoMap.lockInfo[opsID]
 	if !foundInfo {
 		// Unlock request with invalid opertion ID not accepted.
-		return LockInfoOpsIDNotFound{param.volume, param.path, opsID}
+		return traceError(LockInfoOpsIDNotFound{param.volume, param.path, opsID})
 	}
 	// Decrease the global running and lock reference counter.
 	n.runningLockCounter--

cmd/lock-instrument_test.go

@@ -278,7 +278,7 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
 			readLock:   false,
 			setBlocked: false,
 			// expected metrics.
-			expectedErr: LockInfoVolPathMssing{"my-bucket", "my-object-2"},
+			expectedErr: LockInfoVolPathMissing{"my-bucket", "my-object-2"},
 		},
 		// Test case - 3.
 		// Entry for the given operationID doesn't exist in the lock state info.
@@ -325,8 +325,8 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
 	actualErr := nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin,
 		testCases[0].opsID, testCases[0].readLock)
 
-	expectedErr := LockInfoVolPathMssing{testCases[0].volume, testCases[0].path}
-	if actualErr != expectedErr {
+	expectedErr := LockInfoVolPathMissing{testCases[0].volume, testCases[0].path}
+	if errorCause(actualErr) != expectedErr {
 		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedErr, actualErr)
 	}
 
@@ -340,7 +340,7 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
 	actualErr = nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin,
 		testCases[0].opsID, testCases[0].readLock)
 
-	if actualErr != errLockNotInitialized {
+	if errorCause(actualErr) != errLockNotInitialized {
 		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", errLockNotInitialized, actualErr)
 	}
 
@@ -356,7 +356,7 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
 		testCases[0].opsID, testCases[0].readLock)
 
 	expectedOpsErr := LockInfoOpsIDNotFound{testCases[0].volume, testCases[0].path, testCases[0].opsID}
-	if actualErr != expectedOpsErr {
+	if errorCause(actualErr) != expectedOpsErr {
 		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedOpsErr, actualErr)
 	}
 
@@ -381,7 +381,7 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
 		testCases[0].opsID, testCases[0].readLock)
 
 	expectedBlockErr := LockInfoStateNotBlocked{testCases[0].volume, testCases[0].path, testCases[0].opsID}
-	if actualErr != expectedBlockErr {
+	if errorCause(actualErr) != expectedBlockErr {
 		t.Fatalf("Errors mismatch: Expected: \"%s\", got: \"%s\"", expectedBlockErr, actualErr)
 	}
 
@@ -402,7 +402,7 @@ func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
 		}
 		// invoking the method under test.
 		actualErr = nsMutex.statusBlockedToRunning(param, testCase.lockOrigin, testCase.opsID, testCase.readLock)
-		if actualErr != testCase.expectedErr {
+		if errorCause(actualErr) != testCase.expectedErr {
 			t.Fatalf("Test %d: Errors mismatch: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, actualErr)
 		}
 		// In case of no error proceed with validating the lock state information.
@@ -520,8 +520,8 @@ func TestNsLockMapStatusNoneToBlocked(t *testing.T) {
 	actualErr := nsMutex.statusBlockedToRunning(param, testCases[0].lockOrigin,
 		testCases[0].opsID, testCases[0].readLock)
 
-	expectedErr := LockInfoVolPathMssing{testCases[0].volume, testCases[0].path}
-	if actualErr != expectedErr {
+	expectedErr := LockInfoVolPathMissing{testCases[0].volume, testCases[0].path}
+	if errorCause(actualErr) != expectedErr {
 		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedErr, actualErr)
 	}
 
@@ -564,8 +564,8 @@ func TestNsLockMapDeleteLockInfoEntryForOps(t *testing.T) {
 
 	actualErr := nsMutex.deleteLockInfoEntryForOps(param, testCases[0].opsID)
 
-	expectedErr := LockInfoVolPathMssing{testCases[0].volume, testCases[0].path}
-	if actualErr != expectedErr {
+	expectedErr := LockInfoVolPathMissing{testCases[0].volume, testCases[0].path}
+	if errorCause(actualErr) != expectedErr {
 		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedErr, actualErr)
 	}
 
@@ -584,7 +584,7 @@ func TestNsLockMapDeleteLockInfoEntryForOps(t *testing.T) {
 	actualErr = nsMutex.deleteLockInfoEntryForOps(param, "non-existent-OpsID")
 
 	expectedOpsIDErr := LockInfoOpsIDNotFound{param.volume, param.path, "non-existent-OpsID"}
-	if actualErr != expectedOpsIDErr {
+	if errorCause(actualErr) != expectedOpsIDErr {
 		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedOpsIDErr, actualErr)
 	}
 	// case - 4.
@@ -647,8 +647,8 @@ func TestNsLockMapDeleteLockInfoEntryForVolumePath(t *testing.T) {
 	// Set the status of the lock to blocked and then to running.
 	param := nsParam{testCases[0].volume, testCases[0].path}
 	actualErr := nsMutex.deleteLockInfoEntryForVolumePath(param)
-	expectedNilErr := LockInfoVolPathMssing{param.volume, param.path}
-	if actualErr != expectedNilErr {
+	expectedNilErr := LockInfoVolPathMissing{param.volume, param.path}
+	if errorCause(actualErr) != expectedNilErr {
 		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedNilErr, actualErr)
 	}