From 5aa5dcdc6dd94f186addcb5afd9c443ff09e3be5 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 10 Jan 2020 02:35:06 -0800 Subject: [PATCH] lock: improve locker initialization at init (#8776) Use reference format to initialize lockers during startup, also handle `nil` for NetLocker in dsync and remove *errorLocker* implementation Add further tuning parameters such as - DialTimeout is now 15 seconds from 30 seconds - KeepAliveTimeout is not 20 seconds, 5 seconds more than default 15 seconds - ResponseHeaderTimeout to 10 seconds - ExpectContinueTimeout is reduced to 3 seconds - DualStack is enabled by default remove setting it to `true` - Reduce IdleConnTimeout to 30 seconds from 1 minute to avoid idleConn build up Fixes #8773 --- .travis.yml | 3 +- Dockerfile.simpleci | 6 +- Makefile | 12 +- buildscripts/go-coverage.sh | 5 - buildscripts/race.sh | 7 + buildscripts/verify-healing.sh | 197 ++++++++++++++++++++++++++++ cmd/admin-heal-ops.go | 55 ++++---- cmd/background-heal-ops.go | 1 + cmd/background-newdisks-heal-ops.go | 8 +- cmd/config-encrypted.go | 25 +++- cmd/endpoint.go | 12 +- cmd/local-locker.go | 35 ----- cmd/lock-rest-client.go | 4 +- cmd/object-api-multipart_test.go | 6 - cmd/peer-rest-server.go | 3 +- cmd/posix.go | 4 +- cmd/server-main.go | 19 +-- cmd/utils.go | 16 ++- cmd/xl-sets.go | 54 +++----- cmd/xl-v1.go | 32 +---- cmd/xl-v1_test.go | 67 ---------- pkg/dsync/drwmutex.go | 13 +- 22 files changed, 331 insertions(+), 253 deletions(-) delete mode 100755 buildscripts/go-coverage.sh create mode 100755 buildscripts/race.sh create mode 100755 buildscripts/verify-healing.sh delete mode 100644 cmd/xl-v1_test.go diff --git a/.travis.yml b/.travis.yml index 7f66ce3bc..15c3ccd7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,8 +32,7 @@ matrix: - make - diff -au <(gofmt -s -d cmd) <(printf "") - diff -au <(gofmt -s -d pkg) <(printf "") - - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done - - make verifiers + - make test-race - make crosscompile - make verify - cd browser && npm install && npm run test && cd .. diff --git a/Dockerfile.simpleci b/Dockerfile.simpleci index 7fd5b4015..bff2f369b 100644 --- a/Dockerfile.simpleci +++ b/Dockerfile.simpleci @@ -34,11 +34,13 @@ USER ci RUN make RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")' RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")' -RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done -RUN make verifiers +RUN make test-race RUN make crosscompile RUN make verify +## -- add healing tests +RUN make verify-healing + #------------------------------------------------------------- # Stage 2: Test Frontend #------------------------------------------------------------- diff --git a/Makefile b/Makefile index f267816bb..9694bbe5d 100644 --- a/Makefile +++ b/Makefile @@ -64,15 +64,21 @@ test: verifiers build @echo "Running unit tests" @GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null +test-race: verifiers build + @echo "Running unit tests under -race" + @(env bash $(PWD)/buildscripts/race.sh) + # Verify minio binary verify: @echo "Verifying build with race" @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null @(env bash $(PWD)/buildscripts/verify-build.sh) -coverage: build - @echo "Running all coverage for minio" - @(env bash $(PWD)/buildscripts/go-coverage.sh) +# Verify healing of disks with minio binary +verify-healing: + @echo "Verify healing build with race" + @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null + @(env bash $(PWD)/buildscripts/verify-healing.sh) # Builds minio locally. build: checks diff --git a/buildscripts/go-coverage.sh b/buildscripts/go-coverage.sh deleted file mode 100755 index 5efcfbb03..000000000 --- a/buildscripts/go-coverage.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -set -e - -GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./... diff --git a/buildscripts/race.sh b/buildscripts/race.sh new file mode 100755 index 000000000..12f03c3b2 --- /dev/null +++ b/buildscripts/race.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -e + +for d in $(go list ./... | grep -v browser); do + CGO_ENABLED=1 go test -v -race --timeout 20m "$d" +done diff --git a/buildscripts/verify-healing.sh b/buildscripts/verify-healing.sh new file mode 100755 index 000000000..51ebe63d6 --- /dev/null +++ b/buildscripts/verify-healing.sh @@ -0,0 +1,197 @@ +#!/bin/bash +# +# MinIO Cloud Storage, (C) 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +set -E +set -o pipefail + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +WORK_DIR="$PWD/.verify-$RANDOM" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" ) + +function start_minio_3_node() { + declare -a minio_pids + export MINIO_ACCESS_KEY=minio + export MINIO_SECRET_KEY=minio123 + + for i in $(seq 1 3); do + ARGS+=("http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/6/") + done + + "${MINIO[@]}" server --address ":9001" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9001.log" 2>&1 & + minio_pids[0]=$! + + "${MINIO[@]}" server --address ":9002" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9002.log" 2>&1 & + minio_pids[1]=$! + + "${MINIO[@]}" server --address ":9003" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9003.log" 2>&1 & + minio_pids[2]=$! + + sleep "$1" + echo "${minio_pids[@]}" +} + + +function check_online() { + for i in $(seq 1 3); do + if grep -q 'Server switching to safe mode' ${WORK_DIR}/dist-minio-$[9000+$i].log; then + echo "1" + fi + done +} + +function purge() +{ + rm -rf "$1" +} + +function __init__() +{ + echo "Initializing environment" + mkdir -p "$WORK_DIR" + mkdir -p "$MINIO_CONFIG_DIR" + + ## version is purposefully set to '3' for minio to migrate configuration file + echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json" +} + +function perform_test_1() { + minio_pids=( $(start_minio_3_node 20) ) + for pid in "${minio_pids[@]}"; do + kill "$pid" + done + + echo "Testing in Distributed Erasure setup healing test case 1" + echo "Remove the contents of the disks belonging to '2' erasure set" + + rm -rf ${WORK_DIR}/2/* + + minio_pids=( $(start_minio_3_node 60) ) + for pid in "${minio_pids[@]}"; do + if ! kill "$pid"; then + cat "${WORK_DIR}/*.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + done + + rv=$(check_online) + if [ "$rv" == "1" ]; then + for pid in "${minio_pids[@]}"; do + kill -9 "$pid" + done + for i in $(seq 1 3); do + echo "server$i log:" + cat "${WORK_DIR}/dist-minio-$[9000+$i].log" + done + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi +} + +function perform_test_2() { + minio_pids=( $(start_minio_3_node 20) ) + for pid in "${minio_pids[@]}"; do + kill "$pid" + done + + echo "Testing in Distributed Erasure setup healing test case 2" + echo "Remove the contents of the disks belonging to '1' erasure set" + + rm -rf ${WORK_DIR}/1/*/ + + minio_pids=( $(start_minio_3_node 60) ) + for pid in "${minio_pids[@]}"; do + if ! kill "$pid"; then + cat "${WORK_DIR}/*.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + done + + rv=$(check_online) + + if [ "$rv" == "1" ]; then + for pid in "${minio_pids[@]}"; do + kill -9 "$pid" + done + for i in $(seq 1 3); do + echo "server$i log:" + cat "${WORK_DIR}/dist-minio-$[9000+$i].log" + done + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi +} + +function perform_test_3() { + minio_pids=( $(start_minio_3_node 20) ) + for pid in "${minio_pids[@]}"; do + kill "$pid" + done + + echo "Testing in Distributed Erasure setup healing test case 3" + echo "Remove the contents of the disks belonging to '3' erasure set" + + rm -rf ${WORK_DIR}/3/*/ + + minio_pids=( $(start_minio_3_node 60) ) + for pid in "${minio_pids[@]}"; do + if ! kill "$pid"; then + cat "${WORK_DIR}/*.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + done + + rv=$(check_online) + + if [ "$rv" == "1" ]; then + for pid in "${minio_pids[@]}"; do + kill -9 "$pid" + done + for i in $(seq 1 3); do + echo "server$i log:" + cat "${WORK_DIR}/dist-minio-$[9000+$i].log" + done + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi +} + +function main() +{ + perform_test_1 + perform_test_2 + perform_test_3 +} + +( __init__ "$@" && main "$@" ) +rv=$? +purge "$WORK_DIR" +exit "$rv" diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go index 0e1d7dc36..62897f9c2 100644 --- a/cmd/admin-heal-ops.go +++ b/cmd/admin-heal-ops.go @@ -577,44 +577,37 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType) func (h *healSequence) healItemsFromSourceCh() error { h.lastHealActivity = UTCNow() - // Start healing the config prefix. - if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil { - logger.LogIf(h.ctx, err) - } - - // Start healing the bucket config prefix. - if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil { - logger.LogIf(h.ctx, err) - } - - // Start healing the background ops prefix. - if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil { + if err := h.healItems(); err != nil { logger.LogIf(h.ctx, err) } - for path := range h.sourceCh { + for { + select { + case path := <-h.sourceCh: + var itemType madmin.HealItemType + switch { + case path == nopHeal: + continue + case path == SlashSeparator: + itemType = madmin.HealItemMetadata + case !strings.Contains(path, SlashSeparator): + itemType = madmin.HealItemBucket + default: + itemType = madmin.HealItemObject + } - var itemType madmin.HealItemType - switch { - case path == nopHeal: - continue - case path == SlashSeparator: - itemType = madmin.HealItemMetadata - case !strings.Contains(path, SlashSeparator): - itemType = madmin.HealItemBucket - default: - itemType = madmin.HealItemObject - } + if err := h.queueHealTask(path, itemType); err != nil { + logger.LogIf(h.ctx, err) + } - if err := h.queueHealTask(path, itemType); err != nil { - logger.LogIf(h.ctx, err) + h.scannedItemsCount++ + h.lastHealActivity = UTCNow() + case <-h.traverseAndHealDoneCh: + return nil + case <-GlobalServiceDoneCh: + return nil } - - h.scannedItemsCount++ - h.lastHealActivity = UTCNow() } - - return nil } func (h *healSequence) healFromSourceCh() { diff --git a/cmd/background-heal-ops.go b/cmd/background-heal-ops.go index 47f71f944..04ff997bc 100644 --- a/cmd/background-heal-ops.go +++ b/cmd/background-heal-ops.go @@ -61,6 +61,7 @@ func (h *healRoutine) run() { if !ok { break } + if httpServer := newHTTPServerFn(); httpServer != nil { // Wait at max 10 minute for an inprogress request before proceeding to heal waitCount := 600 diff --git a/cmd/background-newdisks-heal-ops.go b/cmd/background-newdisks-heal-ops.go index b81fd8d12..f7038a331 100644 --- a/cmd/background-newdisks-heal-ops.go +++ b/cmd/background-newdisks-heal-ops.go @@ -62,10 +62,9 @@ func monitorLocalDisksAndHeal() { time.Sleep(time.Second) } - // Perform automatic disk healing when a new one is inserted + // Perform automatic disk healing when a disk is replaced locally. for { - time.Sleep(defaultMonitorNewDiskInterval) - + // Attempt a heal as the server starts-up first. localDisksInZoneHeal := make([]Endpoints, len(z.zones)) for i, ep := range globalEndpoints { localDisksToHeal := Endpoints{} @@ -88,9 +87,12 @@ func monitorLocalDisksAndHeal() { // Reformat disks bgSeq.sourceCh <- SlashSeparator + // Ensure that reformatting disks is finished bgSeq.sourceCh <- nopHeal + time.Sleep(defaultMonitorNewDiskInterval) + var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal)) // Compute the list of erasure set to heal for i, localDisksToHeal := range localDisksInZoneHeal { diff --git a/cmd/config-encrypted.go b/cmd/config-encrypted.go index 7b6708838..5de7f5d3f 100644 --- a/cmd/config-encrypted.go +++ b/cmd/config-encrypted.go @@ -84,7 +84,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error { return nil } if !globalActiveCred.IsValid() { - return errInvalidArgument + return config.ErrMissingCredentialsBackendEncrypted(nil) } } @@ -267,7 +267,12 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error { } if !utf8.Valid(data) { - return errors.New("config data not in plain-text form") + _, err = decryptData(data, globalActiveCred) + if err == nil { + // Config is already encrypted with right keys + continue + } + return errors.New("config data not in plain-text form or encrypted") } cencdata, err = madmin.EncryptData(globalActiveCred.String(), data) @@ -279,9 +284,11 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error { return err } } - if encrypted && globalActiveCred.IsValid() { + + if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() { logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs") } + return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete) } @@ -309,7 +316,8 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede var marker string for { - res, err := objAPI.ListObjects(context.Background(), minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList) + res, err := objAPI.ListObjects(context.Background(), minioMetaBucket, + minioConfigPrefix, marker, "", maxObjectList) if err != nil { return err } @@ -339,7 +347,12 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede } if !utf8.Valid(data) { - return errors.New("config data not in plain-text form") + _, err = decryptData(data, globalActiveCred) + if err == nil { + // Config is already encrypted with right keys + continue + } + return errors.New("config data not in plain-text form or encrypted") } cencdata, err = madmin.EncryptData(globalActiveCred.String(), data) @@ -359,7 +372,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede marker = res.NextMarker } - if encrypted && globalActiveCred.IsValid() { + if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() { logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs") } diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 28198751a..3516597b7 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -53,8 +53,7 @@ const ( // Endpoint - any type of endpoint. type Endpoint struct { *url.URL - IsLocal bool - SetIndex int + IsLocal bool } func (endpoint Endpoint) String() string { @@ -535,9 +534,8 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp return endpoints, setupType, nil } - for i, iargs := range args { + for _, iargs := range args { // Convert args to endpoints - var newEndpoints Endpoints eps, err := NewEndpoints(iargs...) if err != nil { return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) @@ -548,11 +546,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) } - for _, ep := range eps { - ep.SetIndex = i - newEndpoints = append(newEndpoints, ep) - } - endpoints = append(endpoints, newEndpoints...) + endpoints = append(endpoints, eps...) } if len(endpoints) == 0 { diff --git a/cmd/local-locker.go b/cmd/local-locker.go index 2f9a2e6ee..fbdbb67fd 100644 --- a/cmd/local-locker.go +++ b/cmd/local-locker.go @@ -17,7 +17,6 @@ package cmd import ( - "errors" "fmt" "sync" "time" @@ -39,40 +38,6 @@ func isWriteLock(lri []lockRequesterInfo) bool { return len(lri) == 1 && lri[0].Writer } -type errorLocker struct{} - -func (d *errorLocker) String() string { - return "" -} - -func (d *errorLocker) Lock(args dsync.LockArgs) (reply bool, err error) { - return false, errors.New("unable to lock") -} - -func (d *errorLocker) Unlock(args dsync.LockArgs) (reply bool, err error) { - return false, errors.New("unable to unlock") -} - -func (d *errorLocker) RLock(args dsync.LockArgs) (reply bool, err error) { - return false, errors.New("unable to rlock") -} - -func (d *errorLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) { - return false, errors.New("unable to runlock") -} - -func (d *errorLocker) Close() error { - return nil -} - -func (d *errorLocker) IsOnline() bool { - return false -} - -func (d *errorLocker) Expired(args dsync.LockArgs) (reply bool, err error) { - return false, errors.New("unable to check for lock expiration") -} - // localLocker implements Dsync.NetLocker type localLocker struct { mutex sync.Mutex diff --git a/cmd/lock-rest-client.go b/cmd/lock-rest-client.go index 87ab6f5af..66e46f761 100644 --- a/cmd/lock-rest-client.go +++ b/cmd/lock-rest-client.go @@ -76,8 +76,8 @@ func (client *lockRESTClient) call(method string, values url.Values, body io.Rea } if isNetworkError(err) { - time.AfterFunc(defaultRetryUnit*3, func() { - // After 3 seconds, take this lock client online for a retry. + time.AfterFunc(defaultRetryUnit, func() { + // After 1 seconds, take this lock client online for a retry. atomic.StoreInt32(&client.connected, 1) }) diff --git a/cmd/object-api-multipart_test.go b/cmd/object-api-multipart_test.go index 3eb189362..d5b951c62 100644 --- a/cmd/object-api-multipart_test.go +++ b/cmd/object-api-multipart_test.go @@ -238,12 +238,6 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [ t.Fatalf("Test %s: expected to fail but passed instead", instanceType) } - // as majority of xl.json are not available, we expect uploadID to be not available. - expectedErr1 := BucketNotFound{Bucket: testCase.bucketName} - if err.Error() != expectedErr1.Error() { - t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr1, err) - } - // This causes invalid upload id. for _, disk := range disks { os.RemoveAll(disk) diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index 64b7a0fff..1a725cff9 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -563,7 +563,7 @@ func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Requ vars := mux.Vars(r) dryRunString := vars[peerRESTDryRun] if dryRunString == "" { - s.writeErrorResponse(w, errors.New("dry run parameter is missing")) + s.writeErrorResponse(w, errors.New("dry-run parameter is missing")) return } @@ -583,6 +583,7 @@ func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Requ s.writeErrorResponse(w, errServerNotInitialized) return } + err := objAPI.ReloadFormat(context.Background(), dryRun) if err != nil { s.writeErrorResponse(w, err) diff --git a/cmd/posix.go b/cmd/posix.go index dd02a4c2f..77596ff80 100644 --- a/cmd/posix.go +++ b/cmd/posix.go @@ -554,7 +554,9 @@ func (s *posix) SetDiskID(id string) { func (s *posix) MakeVolBulk(volumes ...string) (err error) { for _, volume := range volumes { if err = s.MakeVol(volume); err != nil { - return err + if err != errVolumeExists { + return err + } } } return nil diff --git a/cmd/server-main.go b/cmd/server-main.go index 434d467ef..539534483 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -200,9 +200,6 @@ func initSafeMode(buckets []BucketInfo) (err error) { } }(objLock) - // Calls New() and initializes all sub-systems. - newAllSubsystems() - // Migrate all backend configs to encrypted backend configs, optionally // handles rotating keys for encryption. if err = handleEncryptedConfigBackend(newObject, true); err != nil { @@ -365,6 +362,16 @@ func serverMain(ctx *cli.Context) { globalObjectAPI = newObject globalObjLayerMutex.Unlock() + // Calls New() and initializes all sub-systems. + newAllSubsystems() + + // Enable healing to heal drives if possible + if globalIsXL { + initBackgroundHealing() + initLocalDisksAutoHeal() + initGlobalHeal() + } + buckets, err := newObject.ListBuckets(context.Background()) if err != nil { logger.Fatal(err, "Unable to list buckets") @@ -391,12 +398,6 @@ func serverMain(ctx *cli.Context) { initDataUsageStats() initDailyLifecycle() - if globalIsXL { - initBackgroundHealing() - initLocalDisksAutoHeal() - initGlobalHeal() - } - // Disable safe mode operation, after all initialization is over. globalObjLayerMutex.Lock() globalSafeMode = false diff --git a/cmd/utils.go b/cmd/utils.go index 8d2053167..6fae2e912 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -145,9 +145,10 @@ const ( // (Acceptable values range from 1 to 10000 inclusive) globalMaxPartID = 10000 - // Default values used while communicating with the cloud backends - defaultDialTimeout = 30 * time.Second - defaultDialKeepAlive = 30 * time.Second + // Default values used while communicating for + // internode communication. + defaultDialTimeout = 15 * time.Second + defaultDialKeepAlive = 20 * time.Second ) // isMaxObjectSize - verify if max object size @@ -350,7 +351,6 @@ func newCustomDialContext(dialTimeout, dialKeepAlive time.Duration) dialContext dialer := &net.Dialer{ Timeout: dialTimeout, KeepAlive: dialKeepAlive, - DualStack: true, } return dialer.DialContext(ctx, network, addr) @@ -363,10 +363,12 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout, dialKeepAlive ti tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: newCustomDialContext(dialTimeout, dialKeepAlive), + MaxIdleConns: 256, MaxIdleConnsPerHost: 256, - IdleConnTimeout: 60 * time.Second, - TLSHandshakeTimeout: 30 * time.Second, - ExpectContinueTimeout: 10 * time.Second, + IdleConnTimeout: 30 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + ExpectContinueTimeout: 3 * time.Second, TLSClientConfig: tlsConfig, // Go net/http automatically unzip if content-type is // gzip disable this feature, as we are always interested diff --git a/cmd/xl-sets.go b/cmd/xl-sets.go index 33858c00b..055d5558b 100644 --- a/cmd/xl-sets.go +++ b/cmd/xl-sets.go @@ -74,9 +74,6 @@ type xlSets struct { // Distributed locker clients. xlLockers setsDsyncLockers - // Lockers map holds dsync lockers for each endpoint - lockersMap map[Endpoint]dsync.NetLocker - // List of endpoints provided on the command line. endpoints Endpoints @@ -112,9 +109,6 @@ func (s *xlSets) isConnected(endpoint Endpoint) bool { if s.xlDisks[i][j].String() != endpointStr { continue } - if !s.xlLockers[i][j].IsOnline() { - continue - } return s.xlDisks[i][j].IsOnline() } } @@ -161,9 +155,9 @@ func findDiskIndex(refFormat, format *formatXLV3) (int, int, error) { return -1, -1, fmt.Errorf("diskID: %s not found", format.XL.This) } -// connectDisksAndLockersWithQuorum is same as connectDisksAndLockers but waits +// connectDisksWithQuorum is same as connectDisks but waits // for quorum number of formatted disks to be online in any given sets. -func (s *xlSets) connectDisksAndLockersWithQuorum() { +func (s *xlSets) connectDisksWithQuorum() { var onlineDisks int for onlineDisks < len(s.endpoints)/2 { for _, endpoint := range s.endpoints { @@ -184,7 +178,6 @@ func (s *xlSets) connectDisksAndLockersWithQuorum() { } disk.SetDiskID(format.XL.This) s.xlDisks[i][j] = disk - s.xlLockers[i][j] = s.lockersMap[endpoint] onlineDisks++ } // Sleep for a while - so that we don't go into @@ -193,9 +186,9 @@ func (s *xlSets) connectDisksAndLockersWithQuorum() { } } -// connectDisksAndLockers - attempt to connect all the endpoints, loads format +// connectDisks - attempt to connect all the endpoints, loads format // and re-arranges the disks in proper position. -func (s *xlSets) connectDisksAndLockers() { +func (s *xlSets) connectDisks() { for _, endpoint := range s.endpoints { if s.isConnected(endpoint) { continue @@ -215,7 +208,6 @@ func (s *xlSets) connectDisksAndLockers() { disk.SetDiskID(format.XL.This) s.xlDisksMu.Lock() s.xlDisks[i][j] = disk - s.xlLockers[i][j] = s.lockersMap[endpoint] s.xlDisksMu.Unlock() } } @@ -235,7 +227,7 @@ func (s *xlSets) monitorAndConnectEndpoints(monitorInterval time.Duration) { case <-s.disksConnectDoneCh: return case <-ticker.C: - s.connectDisksAndLockers() + s.connectDisks() } } } @@ -246,12 +238,6 @@ func (s *xlSets) GetLockers(setIndex int) func() []dsync.NetLocker { defer s.xlDisksMu.Unlock() lockers := make([]dsync.NetLocker, s.drivesPerSet) copy(lockers, s.xlLockers[setIndex]) - for i, lk := range lockers { - // Add error lockers for unavailable locker. - if lk == nil { - lockers[i] = &errorLocker{} - } - } return lockers } } @@ -271,17 +257,11 @@ const defaultMonitorConnectEndpointInterval = time.Second * 10 // Set to 10 secs // Initialize new set of erasure coded sets. func newXLSets(endpoints Endpoints, format *formatXLV3, setCount int, drivesPerSet int) (*xlSets, error) { - lockersMap := make(map[Endpoint]dsync.NetLocker) - for _, endpoint := range endpoints { - lockersMap[endpoint] = newLockAPI(endpoint) - } - // Initialize the XL sets instance. s := &xlSets{ sets: make([]*xlObjects, setCount), xlDisks: make([][]StorageAPI, setCount), xlLockers: make([][]dsync.NetLocker, setCount), - lockersMap: lockersMap, endpoints: endpoints, setCount: setCount, drivesPerSet: drivesPerSet, @@ -308,12 +288,20 @@ func newXLSets(endpoints Endpoints, format *formatXLV3, setCount int, drivesPerS nsMutex: mutex, bp: bp, } + go s.sets[i].cleanupStaleMultipartUploads(context.Background(), GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh) } + // Rely on endpoints list to initialize, init lockers. + for i := 0; i < s.setCount; i++ { + for j := 0; j < s.drivesPerSet; j++ { + s.xlLockers[i][j] = newLockAPI(s.endpoints[i*s.drivesPerSet+j]) + } + } + // Connect disks right away, but wait until we have `format.json` quorum. - s.connectDisksAndLockersWithQuorum() + s.connectDisksWithQuorum() // Start the disk monitoring and connect routine. go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval) @@ -1312,9 +1300,9 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) { // Replace the new format. s.format = refFormat - // Close all existing disks, lockers and reconnect all the disks/lockers. + // Close all existing disks and reconnect all the disks. s.xlDisks.Close() - s.connectDisksAndLockers() + s.connectDisks() // Restart monitoring loop to monitor reformatted disks again. go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval) @@ -1435,11 +1423,6 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe return res, errNoHealRequired } - // All disks are unformatted, return quorum error. - if shouldInitXLDisks(sErrs) { - return res, errXLReadQuorum - } - refFormat, err := getFormatXLInQuorum(formats) if err != nil { return res, err @@ -1506,7 +1489,7 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe // Disconnect/relinquish all existing disks, lockers and reconnect the disks, lockers. s.xlDisks.Close() - s.connectDisksAndLockers() + s.connectDisks() // Restart our monitoring loop to start monitoring newly formatted disks. go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval) @@ -1668,9 +1651,6 @@ func (s *xlSets) IsReady(_ context.Context) bool { if s.xlDisks[i][j] == nil { continue } - if !s.xlLockers[i][j].IsOnline() { - continue - } if s.xlDisks[i][j].IsOnline() { activeDisks++ } diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index cd1cdd601..fd97ff86c 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -123,7 +123,6 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin // Wait for the routines. for i, err := range g.Wait() { peerAddr, pErr := getPeerAddress(disksInfo[i].RelativePath) - if pErr != nil { continue } @@ -144,39 +143,20 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin return disksInfo, onlineDisks, offlineDisks } -// returns sorted disksInfo slice which has only valid entries. -// i.e the entries where the total size of the disk is not stated -// as 0Bytes, this means that the disk is not online or ignored. -func sortValidDisksInfo(disksInfo []DiskInfo) []DiskInfo { - var validDisksInfo []DiskInfo - for _, diskInfo := range disksInfo { - if diskInfo.Total == 0 { - continue - } - validDisksInfo = append(validDisksInfo, diskInfo) - } - sort.Sort(byDiskTotal(validDisksInfo)) - return validDisksInfo -} - // Get an aggregated storage info across all disks. func getStorageInfo(disks []StorageAPI) StorageInfo { disksInfo, onlineDisks, offlineDisks := getDisksInfo(disks) // Sort so that the first element is the smallest. - validDisksInfo := sortValidDisksInfo(disksInfo) - // If there are no valid disks, set total and free disks to 0 - if len(validDisksInfo) == 0 { - return StorageInfo{} - } + sort.Sort(byDiskTotal(disksInfo)) // Combine all disks to get total usage - usedList := make([]uint64, len(validDisksInfo)) - totalList := make([]uint64, len(validDisksInfo)) - availableList := make([]uint64, len(validDisksInfo)) - mountPaths := make([]string, len(validDisksInfo)) + usedList := make([]uint64, len(disksInfo)) + totalList := make([]uint64, len(disksInfo)) + availableList := make([]uint64, len(disksInfo)) + mountPaths := make([]string, len(disksInfo)) - for i, di := range validDisksInfo { + for i, di := range disksInfo { usedList[i] = di.Used totalList[i] = di.Total availableList[i] = di.Free diff --git a/cmd/xl-v1_test.go b/cmd/xl-v1_test.go deleted file mode 100644 index 41b139408..000000000 --- a/cmd/xl-v1_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* - * MinIO Cloud Storage, (C) 2016 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "reflect" - "testing" -) - -// Sort valid disks info. -func TestSortingValidDisks(t *testing.T) { - testCases := []struct { - disksInfo []DiskInfo - validDisksInfo []DiskInfo - }{ - // One of the disks is offline. - { - disksInfo: []DiskInfo{ - {Total: 150, Free: 10}, - {Total: 0, Free: 0}, - {Total: 200, Free: 10}, - {Total: 100, Free: 10}, - }, - validDisksInfo: []DiskInfo{ - {Total: 100, Free: 10}, - {Total: 150, Free: 10}, - {Total: 200, Free: 10}, - }, - }, - // All disks are online. - { - disksInfo: []DiskInfo{ - {Total: 150, Free: 10}, - {Total: 200, Free: 10}, - {Total: 100, Free: 10}, - {Total: 115, Free: 10}, - }, - validDisksInfo: []DiskInfo{ - {Total: 100, Free: 10}, - {Total: 115, Free: 10}, - {Total: 150, Free: 10}, - {Total: 200, Free: 10}, - }, - }, - } - - for i, testCase := range testCases { - validDisksInfo := sortValidDisksInfo(testCase.disksInfo) - if !reflect.DeepEqual(validDisksInfo, testCase.validDisksInfo) { - t.Errorf("Test %d: Expected %#v, Got %#v", i+1, testCase.validDisksInfo, validDisksInfo) - } - } -} diff --git a/pkg/dsync/drwmutex.go b/pkg/dsync/drwmutex.go index 553efd138..7e1cb6a64 100644 --- a/pkg/dsync/drwmutex.go +++ b/pkg/dsync/drwmutex.go @@ -18,6 +18,7 @@ package dsync import ( "context" + "errors" "fmt" golog "log" "math" @@ -191,6 +192,12 @@ func lock(ds *Dsync, locks *[]string, lockName, id, source string, isReadLock bo go func(index int, isReadLock bool, c NetLocker) { defer wg.Done() + g := Granted{index: index} + if c == nil { + ch <- g + return + } + args := LockArgs{ UID: id, Resource: lockName, @@ -209,7 +216,6 @@ func lock(ds *Dsync, locks *[]string, lockName, id, source string, isReadLock bo } } - g := Granted{index: index} if locked { g.lockUID = args.UID } @@ -400,6 +406,11 @@ func unlock(ds *Dsync, locks []string, name string, isReadLock bool, restClnts [ // sendRelease sends a release message to a node that previously granted a lock func sendRelease(ds *Dsync, c NetLocker, name, uid string, isReadLock bool) { + if c == nil { + log("Unable to call RUnlock", errors.New("netLocker is offline")) + return + } + args := LockArgs{ UID: uid, Resource: name,