Implement list, clear locks REST API w/ pkg/madmin support (#3491)
* Filter lock info based on bucket, prefix and time since lock was held * Implement list and clear locks REST API * madmin: Add list and clear locks API * locks: Clear locks matching bucket, prefix, relTime. * Gather lock information across nodes for both list and clear locks admin REST API. * docs: Add lock API to management APIsmaster
parent
cae62ce543
commit
c8f57133a4
@ -0,0 +1,81 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package cmd |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"testing" |
||||||
|
"time" |
||||||
|
) |
||||||
|
|
||||||
|
// TestListLocksInfo - Test for listLocksInfo.
|
||||||
|
func TestListLocksInfo(t *testing.T) { |
||||||
|
// Initialize globalNSMutex to validate listing of lock
|
||||||
|
// instrumentation information.
|
||||||
|
isDistXL := false |
||||||
|
initNSLock(isDistXL) |
||||||
|
|
||||||
|
// Acquire a few locks to populate lock instrumentation.
|
||||||
|
// Take 10 read locks on bucket1/prefix1/obj1
|
||||||
|
for i := 0; i < 10; i++ { |
||||||
|
readLk := globalNSMutex.NewNSLock("bucket1", "prefix1/obj1") |
||||||
|
readLk.RLock() |
||||||
|
} |
||||||
|
|
||||||
|
// Take write locks on bucket1/prefix/obj{11..19}
|
||||||
|
for i := 0; i < 10; i++ { |
||||||
|
wrLk := globalNSMutex.NewNSLock("bucket1", fmt.Sprintf("prefix1/obj%d", 10+i)) |
||||||
|
wrLk.Lock() |
||||||
|
} |
||||||
|
|
||||||
|
testCases := []struct { |
||||||
|
bucket string |
||||||
|
prefix string |
||||||
|
relTime time.Duration |
||||||
|
numLocks int |
||||||
|
}{ |
||||||
|
// Test 1 - Matches all the locks acquired above.
|
||||||
|
{ |
||||||
|
bucket: "bucket1", |
||||||
|
prefix: "prefix1", |
||||||
|
relTime: time.Duration(0 * time.Second), |
||||||
|
numLocks: 20, |
||||||
|
}, |
||||||
|
// Test 2 - Bucket doesn't match.
|
||||||
|
{ |
||||||
|
bucket: "bucket", |
||||||
|
prefix: "prefix1", |
||||||
|
relTime: time.Duration(0 * time.Second), |
||||||
|
numLocks: 0, |
||||||
|
}, |
||||||
|
// Test 3 - Prefix doesn't match.
|
||||||
|
{ |
||||||
|
bucket: "bucket1", |
||||||
|
prefix: "prefix11", |
||||||
|
relTime: time.Duration(0 * time.Second), |
||||||
|
numLocks: 0, |
||||||
|
}, |
||||||
|
} |
||||||
|
|
||||||
|
for i, test := range testCases { |
||||||
|
actual := listLocksInfo(test.bucket, test.prefix, test.relTime) |
||||||
|
if len(actual) != test.numLocks { |
||||||
|
t.Errorf("Test %d - Expected %d locks but observed %d locks", |
||||||
|
i+1, test.numLocks, len(actual)) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,82 @@ |
|||||||
|
# Management REST API |
||||||
|
|
||||||
|
## Authentication |
||||||
|
- AWS signatureV4 |
||||||
|
- We use "minio" as region. Here region is set only for signature calculation. |
||||||
|
|
||||||
|
## List of management APIs |
||||||
|
- Service |
||||||
|
- Stop |
||||||
|
- Restart |
||||||
|
- Status |
||||||
|
|
||||||
|
- Locks |
||||||
|
- List |
||||||
|
- Clear |
||||||
|
|
||||||
|
- Healing |
||||||
|
|
||||||
|
### Service Management APIs |
||||||
|
* Stop |
||||||
|
- POST /?service |
||||||
|
- x-minio-operation: stop |
||||||
|
- Response: On success 200 |
||||||
|
|
||||||
|
* Restart |
||||||
|
- POST /?service |
||||||
|
- x-minio-operation: restart |
||||||
|
- Response: On success 200 |
||||||
|
|
||||||
|
* Status |
||||||
|
- GET /?service |
||||||
|
- x-minio-operation: status |
||||||
|
- Response: On success 200, return json formatted StorageInfo object. |
||||||
|
|
||||||
|
### Lock Management APIs |
||||||
|
* ListLocks |
||||||
|
- GET /?lock&bucket=mybucket&prefix=myprefix&older-than=rel_time |
||||||
|
- x-minio-operation: list |
||||||
|
- Response: On success 200, json encoded response containing all locks held, older than rel_time. e.g, older than 3 hours. |
||||||
|
- Possible error responses |
||||||
|
- ErrInvalidBucketName |
||||||
|
<Error> |
||||||
|
<Code>InvalidBucketName</Code> |
||||||
|
<Message>The specified bucket is not valid.</Message> |
||||||
|
<Key></Key> |
||||||
|
<BucketName></BucketName> |
||||||
|
<Resource>/</Resource> |
||||||
|
<RequestId>3L137</RequestId> |
||||||
|
<HostId>3L137</HostId> |
||||||
|
</Error> |
||||||
|
|
||||||
|
- ErrInvalidObjectName |
||||||
|
<Error> |
||||||
|
<Code>XMinioInvalidObjectName</Code> |
||||||
|
<Message>Object name contains unsupported characters. Unsupported characters are `^*|\"</Message> |
||||||
|
<Key></Key> |
||||||
|
<BucketName></BucketName> |
||||||
|
<Resource>/</Resource> |
||||||
|
<RequestId>3L137</RequestId> |
||||||
|
<HostId>3L137</HostId> |
||||||
|
</Error> |
||||||
|
|
||||||
|
- ErrInvalidDuration |
||||||
|
<Error> |
||||||
|
<Code>InvalidDuration</Code> |
||||||
|
<Message>Relative duration provided in the request is invalid.</Message> |
||||||
|
<Key></Key> |
||||||
|
<BucketName></BucketName> |
||||||
|
<Resource>/</Resource> |
||||||
|
<RequestId>3L137</RequestId> |
||||||
|
<HostId>3L137</HostId> |
||||||
|
</Error> |
||||||
|
|
||||||
|
|
||||||
|
* ClearLocks |
||||||
|
- POST /?lock&bucket=mybucket&prefix=myprefix&older-than=rel_time |
||||||
|
- x-minio-operation: clear |
||||||
|
- Response: On success 200, json encoded response containing all locks cleared, older than rel_time. e.g, older than 3 hours. |
||||||
|
- Possible error responses, similar to errors listed in ListLocks. |
||||||
|
- ErrInvalidBucketName |
||||||
|
- ErrInvalidObjectName |
||||||
|
- ErrInvalidDuration |
@ -1,33 +0,0 @@ |
|||||||
# Service REST API |
|
||||||
|
|
||||||
## Authentication |
|
||||||
- AWS signatureV4 |
|
||||||
- We use "minio" as region. Here region is set only for signature calculation. |
|
||||||
|
|
||||||
## List of management APIs |
|
||||||
- Service |
|
||||||
- Stop |
|
||||||
- Restart |
|
||||||
- Status |
|
||||||
|
|
||||||
- Locks |
|
||||||
- List |
|
||||||
- Clear |
|
||||||
|
|
||||||
- Healing |
|
||||||
|
|
||||||
### Service Management APIs |
|
||||||
* Stop |
|
||||||
- POST /?service |
|
||||||
- x-minio-operation: stop |
|
||||||
- Response: On success 200 |
|
||||||
|
|
||||||
* Restart |
|
||||||
- POST /?service |
|
||||||
- x-minio-operation: restart |
|
||||||
- Response: On success 200 |
|
||||||
|
|
||||||
* Status |
|
||||||
- GET /?service |
|
||||||
- x-minio-operation: status |
|
||||||
- Response: On success 200, return json formatted StorageInfo object. |
|
@ -0,0 +1,47 @@ |
|||||||
|
// +build ignore
|
||||||
|
|
||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
* |
||||||
|
*/ |
||||||
|
|
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"log" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/minio/minio/pkg/madmin" |
||||||
|
) |
||||||
|
|
||||||
|
func main() { |
||||||
|
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
|
||||||
|
// dummy values, please replace them with original values.
|
||||||
|
|
||||||
|
// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
|
||||||
|
// New returns an Minio Admin client object.
|
||||||
|
madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) |
||||||
|
if err != nil { |
||||||
|
log.Fatalln(err) |
||||||
|
} |
||||||
|
|
||||||
|
// Clear locks held on mybucket/myprefix older than olderThan seconds.
|
||||||
|
olderThan := time.Duration(30 * time.Second) |
||||||
|
locksCleared, err := madmClnt.ClearLocks("mybucket", "myprefix", olderThan) |
||||||
|
if err != nil { |
||||||
|
log.Fatalln(err) |
||||||
|
} |
||||||
|
log.Println(locksCleared) |
||||||
|
} |
@ -0,0 +1,46 @@ |
|||||||
|
// +build ignore
|
||||||
|
|
||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
* |
||||||
|
*/ |
||||||
|
|
||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"log" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/minio/minio/pkg/madmin" |
||||||
|
) |
||||||
|
|
||||||
|
func main() { |
||||||
|
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
|
||||||
|
// dummy values, please replace them with original values.
|
||||||
|
|
||||||
|
// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
|
||||||
|
// New returns an Minio Admin client object.
|
||||||
|
madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) |
||||||
|
if err != nil { |
||||||
|
log.Fatalln(err) |
||||||
|
} |
||||||
|
|
||||||
|
// List locks held on mybucket/myprefix older than 30s.
|
||||||
|
locksHeld, err := madmClnt.ListLocks("mybucket", "myprefix", time.Duration(30*time.Second)) |
||||||
|
if err != nil { |
||||||
|
log.Fatalln(err) |
||||||
|
} |
||||||
|
log.Println(locksHeld) |
||||||
|
} |
@ -0,0 +1,151 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
* |
||||||
|
*/ |
||||||
|
|
||||||
|
package madmin |
||||||
|
|
||||||
|
import ( |
||||||
|
"encoding/json" |
||||||
|
"errors" |
||||||
|
"io" |
||||||
|
"io/ioutil" |
||||||
|
"net/http" |
||||||
|
"net/url" |
||||||
|
"time" |
||||||
|
) |
||||||
|
|
||||||
|
type statusType string |
||||||
|
|
||||||
|
const ( |
||||||
|
runningStatus statusType = "Running" |
||||||
|
blockedStatus statusType = "Blocked" |
||||||
|
) |
||||||
|
|
||||||
|
type lockType string |
||||||
|
|
||||||
|
const ( |
||||||
|
debugRLockStr lockType = "RLock" |
||||||
|
debugWLockStr lockType = "WLock" |
||||||
|
) |
||||||
|
|
||||||
|
// OpsLockState - represents lock specific details.
|
||||||
|
type OpsLockState struct { |
||||||
|
OperationID string `json:"opsID"` // String containing operation ID.
|
||||||
|
LockSource string `json:"lockSource"` // Operation type (GetObject, PutObject...)
|
||||||
|
LockType lockType `json:"lockType"` // Lock type (RLock, WLock)
|
||||||
|
Status statusType `json:"status"` // Status can be Running/Ready/Blocked.
|
||||||
|
Since time.Time `json:"statusSince"` // Time when the lock was initially held.
|
||||||
|
Duration time.Duration `json:"statusDuration"` // Duration since the lock was held.
|
||||||
|
} |
||||||
|
|
||||||
|
// VolumeLockInfo - represents summary and individual lock details of all
|
||||||
|
// locks held on a given bucket, object.
|
||||||
|
type VolumeLockInfo struct { |
||||||
|
Bucket string `json:"bucket"` |
||||||
|
Object string `json:"object"` |
||||||
|
// All locks blocked + running for given <volume,path> pair.
|
||||||
|
LocksOnObject int64 `json:"locksOnObject"` |
||||||
|
// Count of operations which has successfully acquired the lock
|
||||||
|
// but hasn't unlocked yet( operation in progress).
|
||||||
|
LocksAcquiredOnObject int64 `json:"locksAcquiredOnObject"` |
||||||
|
// Count of operations which are blocked waiting for the lock
|
||||||
|
// to be released.
|
||||||
|
TotalBlockedLocks int64 `json:"locksBlockedOnObject"` |
||||||
|
// State information containing state of the locks for all operations
|
||||||
|
// on given <volume,path> pair.
|
||||||
|
LockDetailsOnObject []OpsLockState `json:"lockDetailsOnObject"` |
||||||
|
} |
||||||
|
|
||||||
|
// getLockInfos - unmarshal []VolumeLockInfo from a reader.
|
||||||
|
func getLockInfos(body io.Reader) ([]VolumeLockInfo, error) { |
||||||
|
respBytes, err := ioutil.ReadAll(body) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
var lockInfos []VolumeLockInfo |
||||||
|
|
||||||
|
err = json.Unmarshal(respBytes, &lockInfos) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
return lockInfos, nil |
||||||
|
} |
||||||
|
|
||||||
|
// ListLocks - Calls List Locks Management API to fetch locks matching
|
||||||
|
// bucket, prefix and held before the duration supplied.
|
||||||
|
func (adm *AdminClient) ListLocks(bucket, prefix string, olderThan time.Duration) ([]VolumeLockInfo, error) { |
||||||
|
queryVal := make(url.Values) |
||||||
|
queryVal.Set("lock", "") |
||||||
|
queryVal.Set("bucket", bucket) |
||||||
|
queryVal.Set("prefix", prefix) |
||||||
|
queryVal.Set("older-than", olderThan.String()) |
||||||
|
|
||||||
|
hdrs := make(http.Header) |
||||||
|
hdrs.Set(minioAdminOpHeader, "list") |
||||||
|
|
||||||
|
reqData := requestData{ |
||||||
|
queryValues: queryVal, |
||||||
|
customHeaders: hdrs, |
||||||
|
} |
||||||
|
|
||||||
|
// Execute GET on /?lock to list locks.
|
||||||
|
resp, err := adm.executeMethod("GET", reqData) |
||||||
|
|
||||||
|
defer closeResponse(resp) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK { |
||||||
|
return nil, errors.New("Got HTTP Status: " + resp.Status) |
||||||
|
} |
||||||
|
|
||||||
|
return getLockInfos(resp.Body) |
||||||
|
} |
||||||
|
|
||||||
|
// ClearLocks - Calls Clear Locks Management API to clear locks held
|
||||||
|
// on bucket, matching prefix older than duration supplied.
|
||||||
|
func (adm *AdminClient) ClearLocks(bucket, prefix string, olderThan time.Duration) ([]VolumeLockInfo, error) { |
||||||
|
queryVal := make(url.Values) |
||||||
|
queryVal.Set("lock", "") |
||||||
|
queryVal.Set("bucket", bucket) |
||||||
|
queryVal.Set("prefix", prefix) |
||||||
|
queryVal.Set("older-than", olderThan.String()) |
||||||
|
|
||||||
|
hdrs := make(http.Header) |
||||||
|
hdrs.Set(minioAdminOpHeader, "clear") |
||||||
|
|
||||||
|
reqData := requestData{ |
||||||
|
queryValues: queryVal, |
||||||
|
customHeaders: hdrs, |
||||||
|
} |
||||||
|
|
||||||
|
// Execute POST on /?lock to clear locks.
|
||||||
|
resp, err := adm.executeMethod("POST", reqData) |
||||||
|
|
||||||
|
defer closeResponse(resp) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK { |
||||||
|
return nil, errors.New("Got HTTP Status: " + resp.Status) |
||||||
|
} |
||||||
|
|
||||||
|
return getLockInfos(resp.Body) |
||||||
|
} |
@ -0,0 +1,61 @@ |
|||||||
|
/* |
||||||
|
* Minio Cloud Storage, (C) 2016 Minio, Inc. |
||||||
|
* |
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
* you may not use this file except in compliance with the License. |
||||||
|
* You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
* |
||||||
|
*/ |
||||||
|
|
||||||
|
package madmin |
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"encoding/json" |
||||||
|
"reflect" |
||||||
|
"testing" |
||||||
|
) |
||||||
|
|
||||||
|
// Test for getLockInfos helper function.
|
||||||
|
func TestGetLockInfos(t *testing.T) { |
||||||
|
testCases := []struct { |
||||||
|
// Used to construct a io.Reader holding xml serialized lock information
|
||||||
|
inputLocks []VolumeLockInfo |
||||||
|
}{ |
||||||
|
// To build a reader with _no_ lock information.
|
||||||
|
{ |
||||||
|
inputLocks: []VolumeLockInfo{}, |
||||||
|
}, |
||||||
|
// To build a reader with _one_ lock information.
|
||||||
|
{ |
||||||
|
inputLocks: []VolumeLockInfo{{Bucket: "bucket", Object: "object"}}, |
||||||
|
}, |
||||||
|
} |
||||||
|
for i, test := range testCases { |
||||||
|
jsonBytes, err := json.Marshal(test.inputLocks) |
||||||
|
if err != nil { |
||||||
|
t.Fatalf("Test %d - Failed to marshal input lockInfos - %v", i+1, err) |
||||||
|
} |
||||||
|
actualLocks, err := getLockInfos(bytes.NewReader(jsonBytes)) |
||||||
|
if err != nil { |
||||||
|
t.Fatalf("Test %d - Failed to get lock information - %v", i+1, err) |
||||||
|
} |
||||||
|
if !reflect.DeepEqual(actualLocks, test.inputLocks) { |
||||||
|
t.Errorf("Test %d - Expected %v but received %v", i+1, test.inputLocks, actualLocks) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Invalid json representation of []VolumeLockInfo
|
||||||
|
_, err := getLockInfos(bytes.NewReader([]byte("invalidBytes"))) |
||||||
|
if err == nil { |
||||||
|
t.Errorf("Test expected to fail, but passed") |
||||||
|
} |
||||||
|
} |
Loading…
Reference in new issue