From ad511b0eb8a928fd79d0d995e227b8cdfeb4f736 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Tue, 5 Jan 2021 10:45:26 -0800 Subject: [PATCH] tests: Fix occasional data race (#11223) CI tests could trigger a data race. Servers are generally not expected to reinitialize, so tests could trigger data races when reinitializing and async operations are running. We add the option to safely reset global vars instead of overwriting. Fixes races like: ``` WARNING: DATA RACE Read at 0x00000477ab18 by goroutine 1159: github.com/minio/minio/cmd.FileInfo.ToObjectInfo() /home/runner/work/minio/minio/cmd/erasure-metadata.go:105 +0x16d github.com/minio/minio/cmd.erasureObjects.putObject() /home/runner/work/minio/minio/cmd/erasure-object.go:748 +0x13f8 github.com/minio/minio/cmd.(*erasureObjects).listPath.func3.2() /home/runner/work/minio/minio/cmd/metacache-set.go:682 +0x7d3 github.com/minio/minio/cmd.newMetacacheBlockWriter.func1.2() /home/runner/work/minio/minio/cmd/metacache-stream.go:777 +0x1c4 github.com/minio/minio/cmd.newMetacacheBlockWriter.func1() /home/runner/work/minio/minio/cmd/metacache-stream.go:806 +0x614 Previous write at 0x00000477ab18 by goroutine 1269: [failed to restore the stack] Goroutine 1159 (running) created at: github.com/minio/minio/cmd.newMetacacheBlockWriter() /home/runner/work/minio/minio/cmd/metacache-stream.go:760 +0x112 github.com/minio/minio/cmd.(*erasureObjects).listPath.func3() /home/runner/work/minio/minio/cmd/metacache-set.go:672 +0xe22 Goroutine 1269 (running) created at: testing.(*T).Run() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1095 +0x537 testing.runTests.func1() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1339 +0xa6 testing.tRunner() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1050 +0x1eb testing.runTests() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1337 +0x594 testing.(*M).Run() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1252 +0x2ff github.com/minio/minio/cmd.TestMain() /home/runner/work/minio/minio/cmd/test-utils_test.go:120 +0x44e main.main() _testmain.go:1408 +0x223 ================== ================== WARNING: DATA RACE Read at 0x00000477aae8 by goroutine 1159: github.com/minio/minio/cmd.(*BucketVersioningSys).Enabled() /home/runner/work/minio/minio/cmd/bucket-versioning.go:26 +0x52 github.com/minio/minio/cmd.FileInfo.ToObjectInfo() /home/runner/work/minio/minio/cmd/erasure-metadata.go:105 +0x197 github.com/minio/minio/cmd.erasureObjects.putObject() /home/runner/work/minio/minio/cmd/erasure-object.go:748 +0x13f8 github.com/minio/minio/cmd.(*erasureObjects).listPath.func3.2() /home/runner/work/minio/minio/cmd/metacache-set.go:682 +0x7d3 github.com/minio/minio/cmd.newMetacacheBlockWriter.func1.2() /home/runner/work/minio/minio/cmd/metacache-stream.go:777 +0x1c4 github.com/minio/minio/cmd.newMetacacheBlockWriter.func1() /home/runner/work/minio/minio/cmd/metacache-stream.go:806 +0x614 Previous write at 0x00000477aae8 by goroutine 1269: [failed to restore the stack] Goroutine 1159 (running) created at: github.com/minio/minio/cmd.newMetacacheBlockWriter() /home/runner/work/minio/minio/cmd/metacache-stream.go:760 +0x112 github.com/minio/minio/cmd.(*erasureObjects).listPath.func3() /home/runner/work/minio/minio/cmd/metacache-set.go:672 +0xe22 Goroutine 1269 (running) created at: testing.(*T).Run() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1095 +0x537 testing.runTests.func1() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1339 +0xa6 testing.tRunner() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1050 +0x1eb testing.runTests() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1337 +0x594 testing.(*M).Run() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1252 +0x2ff github.com/minio/minio/cmd.TestMain() /home/runner/work/minio/minio/cmd/test-utils_test.go:120 +0x44e main.main() _testmain.go:1408 +0x223 ================== ``` --- cmd/bucket-metadata-sys.go | 9 +++++++++ cmd/bucket-versioning.go | 5 +++++ cmd/server-main.go | 13 +++++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/bucket-metadata-sys.go b/cmd/bucket-metadata-sys.go index d2db2c4fa..05b1cea50 100644 --- a/cmd/bucket-metadata-sys.go +++ b/cmd/bucket-metadata-sys.go @@ -477,6 +477,15 @@ func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, ob } } +// Reset the state of the BucketMetadataSys. +func (sys *BucketMetadataSys) Reset() { + sys.Lock() + for k := range sys.metadataMap { + delete(sys.metadataMap, k) + } + sys.Unlock() +} + // NewBucketMetadataSys - creates new policy system. func NewBucketMetadataSys() *BucketMetadataSys { return &BucketMetadataSys{ diff --git a/cmd/bucket-versioning.go b/cmd/bucket-versioning.go index 55c2e50b7..fca5af0b7 100644 --- a/cmd/bucket-versioning.go +++ b/cmd/bucket-versioning.go @@ -51,6 +51,11 @@ func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, erro return globalBucketMetadataSys.GetVersioningConfig(bucket) } +// Reset BucketVersioningSys to initial state. +func (sys *BucketVersioningSys) Reset() { + // There is currently no internal state. +} + // NewBucketVersioningSys - creates new versioning system. func NewBucketVersioningSys() *BucketVersioningSys { return &BucketVersioningSys{} diff --git a/cmd/server-main.go b/cmd/server-main.go index cf2d0cfb6..732d36d02 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -183,7 +183,12 @@ func newAllSubsystems() { globalNotificationSys = NewNotificationSys(globalEndpoints) // Create new bucket metadata system. - globalBucketMetadataSys = NewBucketMetadataSys() + if globalBucketMetadataSys == nil { + globalBucketMetadataSys = NewBucketMetadataSys() + } else { + // Reinitialize safely when testing. + globalBucketMetadataSys.Reset() + } // Create the bucket bandwidth monitor globalBucketMonitor = bandwidth.NewMonitor(GlobalServiceDoneCh) @@ -210,7 +215,11 @@ func newAllSubsystems() { globalBucketQuotaSys = NewBucketQuotaSys() // Create new bucket versioning subsystem - globalBucketVersioningSys = NewBucketVersioningSys() + if globalBucketVersioningSys == nil { + globalBucketVersioningSys = NewBucketVersioningSys() + } else { + globalBucketVersioningSys.Reset() + } // Create new bucket replication subsytem globalBucketTargetSys = NewBucketTargetSys()