From 5730d404785edb077dcedf99f38ccda823d448c8 Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Thu, 21 Jul 2016 13:15:54 -0700 Subject: [PATCH] tests: Added GetObject, DeleteObject and PutObject unit-tests (#2222) --- faulty-disk_test.go | 115 ++++++++++++++++++++++ object-api-putobject_test.go | 161 +++++++++++++++++++----------- xl-v1-object_test.go | 184 +++++++++++++++++++++++++++++++++++ 3 files changed, 404 insertions(+), 56 deletions(-) create mode 100644 faulty-disk_test.go diff --git a/faulty-disk_test.go b/faulty-disk_test.go new file mode 100644 index 000000000..cb1632cfc --- /dev/null +++ b/faulty-disk_test.go @@ -0,0 +1,115 @@ +/* + * Minio Cloud Storage, (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +// Simulates disk returning errFaultyDisk on all methods of StorageAPI +// interface after successCount number of successes. +type faultyDisk struct { + disk *posix + successCount int +} + +// instantiates a faulty +func newFaultyDisk(disk *posix, n int) *faultyDisk { + return &faultyDisk{disk: disk, successCount: n} +} + +func (f *faultyDisk) MakeVol(volume string) (err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.MakeVol(volume) + } + return errFaultyDisk +} +func (f *faultyDisk) ListVols() (vols []VolInfo, err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.ListVols() + } + return nil, errFaultyDisk +} + +func (f *faultyDisk) StatVol(volume string) (volInfo VolInfo, err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.StatVol(volume) + } + return VolInfo{}, errFaultyDisk +} +func (f *faultyDisk) DeleteVol(volume string) (err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.DeleteVol(volume) + } + return errFaultyDisk +} + +func (f *faultyDisk) ListDir(volume, path string) (entries []string, err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.ListDir(volume, path) + } + return []string{}, errFaultyDisk +} + +func (f *faultyDisk) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.ReadFile(volume, path, offset, buf) + } + return 0, errFaultyDisk +} + +func (f *faultyDisk) AppendFile(volume, path string, buf []byte) error { + if f.successCount > 0 { + f.successCount-- + return f.disk.AppendFile(volume, path, buf) + } + return errFaultyDisk +} + +func (f *faultyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error { + if f.successCount > 0 { + f.successCount-- + return f.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath) + } + return errFaultyDisk +} + +func (f *faultyDisk) StatFile(volume string, path string) (file FileInfo, err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.StatFile(volume, path) + } + return FileInfo{}, errFaultyDisk +} + +func (f *faultyDisk) DeleteFile(volume string, path string) (err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.DeleteFile(volume, path) + } + return errFaultyDisk +} + +func (f *faultyDisk) ReadAll(volume string, path string) (buf []byte, err error) { + if f.successCount > 0 { + f.successCount-- + return f.disk.ReadAll(volume, path) + } + return nil, errFaultyDisk +} diff --git a/object-api-putobject_test.go b/object-api-putobject_test.go index 86332e991..e56fb6c45 100644 --- a/object-api-putobject_test.go +++ b/object-api-putobject_test.go @@ -20,13 +20,24 @@ import ( "bytes" "crypto/md5" "encoding/hex" - "fmt" "io/ioutil" "os" "path" "testing" ) +// md5Hex ignores error from Write method since it never returns one. Check +// crypto/md5 doc for more details. +func md5Hex(b []byte) string { + md5Writer := md5.New() + md5Writer.Write(b) + return hex.EncodeToString(md5Writer.Sum(nil)) +} + +func md5Header(data []byte) map[string]string { + return map[string]string{"md5Sum": md5Hex([]byte(data))} +} + // Wrapper for calling PutObject tests for both XL multiple disks and single node setup. func TestObjectAPIPutObject(t *testing.T) { ExecObjectLayerTest(t, testObjectAPIPutObject) @@ -52,71 +63,109 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl t.Fatalf("%s : %s", instanceType, err.Error()) } - failCases := []struct { - bucketName string - objName string - inputReaderData string - inputMeta map[string]string - intputDataSize int64 - // flag indicating whether the test should pass. - shouldPass bool + var ( + nilBytes []byte + data = []byte("hello") + fiveMBBytes = bytes.Repeat([]byte("a"), 5*1024*124) + ) + invalidMD5 := md5Hex([]byte("meh")) + invalidMD5Header := md5Header([]byte("meh")) + + testCases := []struct { + bucketName string + objName string + inputData []byte + inputMeta map[string]string + intputDataSize int64 // expected error output. expectedMd5 string expectedError error }{ // Test case 1-4. // Cases with invalid bucket name. - {".test", "obj", "", nil, 0, false, "", fmt.Errorf("%s", "Bucket name invalid: .test")}, - {"------", "obj", "", nil, 0, false, "", fmt.Errorf("%s", "Bucket name invalid: ------")}, - {"$this-is-not-valid-too", "obj", "", nil, 0, false, "", - fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")}, - {"a", "obj", "", nil, 0, false, "", fmt.Errorf("%s", "Bucket name invalid: a")}, + {".test", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: ".test"}}, + {"------", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: "------"}}, + {"$this-is-not-valid-too", "obj", []byte(""), nil, 0, "", + BucketNameInvalid{Bucket: "$this-is-not-valid-too"}}, + {"a", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: "a"}}, + // Test case - 5. // Case with invalid object names. - {bucket, "", "", nil, 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket#")}, + {bucket, "", []byte(""), nil, 0, "", ObjectNameInvalid{Bucket: bucket, Object: ""}}, + // Test case - 6. // Valid object and bucket names but non-existent bucket. - {"abc", "def", "", nil, 0, false, "", fmt.Errorf("%s", "Bucket not found: abc")}, + {"abc", "def", []byte(""), nil, 0, "", BucketNotFound{Bucket: "abc"}}, + // Test case - 7. // Input to replicate Md5 mismatch. - {bucket, object, "", map[string]string{"md5Sum": "a35"}, 0, false, "", - fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated "+"d41d8cd98f00b204e9800998ecf8427e")}, + {bucket, object, []byte(""), map[string]string{"md5Sum": "a35"}, 0, "", + BadDigest{ExpectedMD5: "a35", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}}, + // Test case - 8. // Input with size more than the size of actual data inside the reader. - {bucket, object, "abcd", map[string]string{"md5Sum": "a35"}, int64(len("abcd") + 1), false, "", + {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, int64(len("abcd") + 1), "", IncompleteBody{}}, + // Test case - 9. // Input with size less than the size of actual data inside the reader. - {bucket, object, "abcd", map[string]string{"md5Sum": "a35"}, int64(len("abcd") - 1), false, "", - fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated 900150983cd24fb0d6963f7d28e17f72")}, + {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, int64(len("abcd") - 1), "", + BadDigest{ExpectedMD5: "a35", CalculatedMD5: "900150983cd24fb0d6963f7d28e17f72"}}, + // Test case - 10-13. // Validating for success cases. - {bucket, object, "abcd", map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), true, "", nil}, - {bucket, object, "efgh", map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), true, "", nil}, - {bucket, object, "ijkl", map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), true, "", nil}, - {bucket, object, "mnop", map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil}, + {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), "", nil}, + {bucket, object, []byte("efgh"), map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), "", nil}, + {bucket, object, []byte("ijkl"), map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), "", nil}, + {bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), "", nil}, + + // Test case 14-16. + // With no metadata + {bucket, object, data, nil, int64(len(data)), md5Hex(data), nil}, + {bucket, object, nilBytes, nil, int64(len(nilBytes)), md5Hex(nilBytes), nil}, + {bucket, object, fiveMBBytes, nil, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, + + // Test case 17-19. + // With arbitrary metadata + {bucket, object, data, map[string]string{"answer": "42"}, int64(len(data)), md5Hex(data), nil}, + {bucket, object, nilBytes, map[string]string{"answer": "42"}, int64(len(nilBytes)), md5Hex(nilBytes), nil}, + {bucket, object, fiveMBBytes, map[string]string{"answer": "42"}, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, + + // Test case 20-22. + // With valid md5sum in header + {bucket, object, data, md5Header(data), int64(len(data)), md5Hex(data), nil}, + {bucket, object, nilBytes, md5Header(nilBytes), int64(len(nilBytes)), md5Hex(nilBytes), nil}, + {bucket, object, fiveMBBytes, md5Header(fiveMBBytes), int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil}, + + // Test case 23-25. + // data with invalid md5sum in header + {bucket, object, data, invalidMD5Header, int64(len(data)), md5Hex(data), BadDigest{invalidMD5, md5Hex(data)}}, + {bucket, object, nilBytes, invalidMD5Header, int64(len(nilBytes)), md5Hex(nilBytes), BadDigest{invalidMD5, md5Hex(nilBytes)}}, + {bucket, object, fiveMBBytes, invalidMD5Header, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), BadDigest{invalidMD5, md5Hex(fiveMBBytes)}}, + + // Test case 26-28. + // data with size different from the actual number of bytes available in the reader + {bucket, object, data, nil, int64(len(data) - 1), md5Hex(data[:len(data)-1]), nil}, + {bucket, object, nilBytes, nil, int64(len(nilBytes) + 1), md5Hex(nilBytes), IncompleteBody{}}, + {bucket, object, fiveMBBytes, nil, int64(0), md5Hex(fiveMBBytes), nil}, } - for i, testCase := range failCases { - actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMeta) - // All are test cases above are expected to fail. - - if actualErr != nil && testCase.shouldPass { - t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) + for i, testCase := range testCases { + actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) + if actualErr != nil && testCase.expectedError == nil { + t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error()) } - if actualErr == nil && !testCase.shouldPass { - t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error()) + if actualErr == nil && testCase.expectedError != nil { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error()) } // Failed as expected, but does it fail for the expected reason. - if actualErr != nil && !testCase.shouldPass { - if testCase.expectedError.Error() != actualErr.Error() { - t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error()) - } + if actualErr != nil && testCase.expectedError != actualErr { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error()) } // Test passes as expected, but the output values are verified for correctness here. - if actualErr == nil && testCase.shouldPass { + if actualErr == nil { // Asserting whether the md5 output is correct. - if testCase.inputMeta["md5Sum"] != actualMd5Hex { + if expectedMD5, ok := testCase.inputMeta["md5Sum"]; ok && expectedMD5 != actualMd5Hex { t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex) } } @@ -155,11 +204,11 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di } testCases := []struct { - bucketName string - objName string - inputReaderData string - inputMeta map[string]string - intputDataSize int64 + bucketName string + objName string + inputData []byte + inputMeta map[string]string + intputDataSize int64 // flag indicating whether the test should pass. shouldPass bool // expected error output. @@ -167,14 +216,14 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di expectedError error }{ // Validating for success cases. - {bucket, object, "abcd", map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), true, "", nil}, - {bucket, object, "efgh", map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), true, "", nil}, - {bucket, object, "ijkl", map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), true, "", nil}, - {bucket, object, "mnop", map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil}, + {bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), true, "", nil}, + {bucket, object, []byte("efgh"), map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), true, "", nil}, + {bucket, object, []byte("ijkl"), map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), true, "", nil}, + {bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil}, } for i, testCase := range testCases { - actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMeta) + actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) } @@ -202,11 +251,11 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di // Validate the last test. testCase := struct { - bucketName string - objName string - inputReaderData string - inputMeta map[string]string - intputDataSize int64 + bucketName string + objName string + inputData []byte + inputMeta map[string]string + intputDataSize int64 // flag indicating whether the test should pass. shouldPass bool // expected error output. @@ -215,14 +264,14 @@ func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, di }{ bucket, object, - "mnop", + []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), false, "", InsufficientWriteQuorum{}, } - _, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMeta) + _, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta) if actualErr != nil && testCase.shouldPass { t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", len(testCases)+1, instanceType, actualErr.Error()) } diff --git a/xl-v1-object_test.go b/xl-v1-object_test.go index 90a008ba5..3a9646107 100644 --- a/xl-v1-object_test.go +++ b/xl-v1-object_test.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/md5" "encoding/hex" + "io/ioutil" "testing" ) @@ -60,3 +61,186 @@ func TestRepeatPutObjectPart(t *testing.T) { } } + +func TestXLDeleteObjectBasic(t *testing.T) { + testCases := []struct { + bucket string + object string + expectedErr error + }{ + {".test", "obj", BucketNameInvalid{Bucket: ".test"}}, + {"----", "obj", BucketNameInvalid{Bucket: "----"}}, + {"bucket", "", ObjectNameInvalid{Bucket: "bucket", Object: ""}}, + {"bucket", "obj/", ObjectNameInvalid{Bucket: "bucket", Object: "obj/"}}, + {"bucket", "/obj", ObjectNameInvalid{Bucket: "bucket", Object: "/obj"}}, + {"bucket", "doesnotexist", ObjectNotFound{Bucket: "bucket", Object: "doesnotexist"}}, + {"bucket", "obj", nil}, + } + + // Create an instance of xl backend + xl, fsDirs, err := getXLObjectLayer() + if err != nil { + t.Fatal(err) + } + + // Make bucket for Test 7 to pass + err = xl.MakeBucket("bucket") + if err != nil { + t.Fatal(err) + } + + // Create object "obj" under bucket "bucket" for Test 7 to pass + _, err = xl.PutObject("bucket", "obj", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + + for i, test := range testCases { + actualErr := xl.DeleteObject(test.bucket, test.object) + if test.expectedErr != nil && actualErr != test.expectedErr { + t.Errorf("Test %d: Expected to fail with %s, but failed with %s", i+1, test.expectedErr, actualErr) + } + if test.expectedErr == nil && actualErr != nil { + t.Errorf("Test %d: Expected to pass, but failed with %s", i+1, actualErr) + } + } + // Cleanup backend directories + removeRoots(fsDirs) +} + +func TestXLDeleteObjectDiskNotFound(t *testing.T) { + // Create an instance of xl backend. + obj, fsDirs, err := getXLObjectLayer() + if err != nil { + t.Fatal(err) + } + + xl := obj.(xlObjects) + + // Create "bucket" + err = obj.MakeBucket("bucket") + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + // Create object "obj" under bucket "bucket". + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + + // for a 16 disk setup, quorum is 9. To simulate disks not found yet + // quorum is available, we remove disks leaving quorum disks behind. + for i := range xl.storageDisks[:7] { + xl.storageDisks[i] = newFaultyDisk(xl.storageDisks[i].(*posix), 0) + } + err = obj.DeleteObject(bucket, object) + if err != nil { + t.Fatal(err) + } + + // Create "obj" under "bucket". + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + if err != nil { + t.Fatal(err) + } + + // Remove one more disk to 'lose' quorum, by setting it to nil. + xl.storageDisks[7] = &faultyDisk{} + xl.storageDisks[8] = &faultyDisk{} + err = obj.DeleteObject(bucket, object) + if err != toObjectErr(errXLWriteQuorum, bucket, object) { + t.Errorf("Expected deleteObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err) + } + // Cleanup backend directories + removeRoots(fsDirs) +} + +func TestGetObjectNoQuorum(t *testing.T) { + // Create an instance of xl backend. + obj, fsDirs, err := getXLObjectLayer() + if err != nil { + t.Fatal(err) + } + + xl := obj.(xlObjects) + + // Create "bucket" + err = obj.MakeBucket("bucket") + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + // Create "object" under "bucket". + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + if err != nil { + t.Fatal(err) + } + // Disable caching to avoid returning early and not covering other code-paths + xl.objCacheEnabled = false + // Make 9 disks offline, which leaves less than quorum number of disks + // in a 16 disk XL setup. The original disks are 'replaced' with + // faultyDisks that fail after 'f' successful StorageAPI method + // invocations, where f - [0,2) + for f := 0; f < 2; f++ { + for i := range xl.storageDisks[:9] { + switch diskType := xl.storageDisks[i].(type) { + case *posix: + xl.storageDisks[i] = newFaultyDisk(diskType, f) + case *faultyDisk: + xl.storageDisks[i] = newFaultyDisk(diskType.disk, f) + } + } + // Fetch object from store. + err = xl.GetObject(bucket, object, 0, int64(len("abcd")), ioutil.Discard) + if err != toObjectErr(errXLReadQuorum, bucket, object) { + t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err) + } + } + // Cleanup backend directories. + removeRoots(fsDirs) +} + +func TestPutObjectNoQuorum(t *testing.T) { + // Create an instance of xl backend. + obj, fsDirs, err := getXLObjectLayer() + if err != nil { + t.Fatal(err) + } + + xl := obj.(xlObjects) + + // Create "bucket" + err = obj.MakeBucket("bucket") + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + // Create "object" under "bucket". + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + if err != nil { + t.Fatal(err) + } + + // Make 9 disks offline, which leaves less than quorum number of disks + // in a 16 disk XL setup. The original disks are 'replaced' with + // faultyDisks that fail after 'f' successful StorageAPI method + // invocations, where f - [0,3) + for f := 0; f < 3; f++ { + for i := range xl.storageDisks[:9] { + switch diskType := xl.storageDisks[i].(type) { + case *posix: + xl.storageDisks[i] = newFaultyDisk(diskType, f) + case *faultyDisk: + xl.storageDisks[i] = newFaultyDisk(diskType.disk, f) + } + } + // Upload new content to same object "object" + _, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil) + if err != toObjectErr(errXLWriteQuorum, bucket, object) { + t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err) + } + } + // Cleanup backend directories. + removeRoots(fsDirs) +}