Add self-healing feature (#7604)
- Background Heal routine receives heal requests from a channel, either to heal format, buckets or objects - Daily sweeper lists all objects in all buckets, these objects don't necessarly have read quorum so they can be removed if these objects are unhealable - Heal daily ops receives objects from the daily sweeper and send them to the heal routine.master
Anis Elleuch
5 years ago
committed by
kannappanr
parent
97090aa16c
commit
7abadfccc2
@ -0,0 +1,160 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
|
||||
"github.com/minio/minio/cmd/logger" |
||||
"github.com/minio/minio/pkg/madmin" |
||||
) |
||||
|
||||
// healTask represents what to heal along with options
|
||||
// path: '/' => Heal disk formats along with metadata
|
||||
// path: 'bucket/' or '/bucket/' => Heal bucket
|
||||
// path: 'bucket/object' => Heal object
|
||||
type healTask struct { |
||||
path string |
||||
opts madmin.HealOpts |
||||
// Healing response will be sent here
|
||||
responseCh chan healResult |
||||
} |
||||
|
||||
// healResult represents a healing result with a possible error
|
||||
type healResult struct { |
||||
result madmin.HealResultItem |
||||
err error |
||||
} |
||||
|
||||
// healRoutine receives heal tasks, to heal buckets, objects and format.json
|
||||
type healRoutine struct { |
||||
tasks chan healTask |
||||
doneCh chan struct{} |
||||
} |
||||
|
||||
// Add a new task in the tasks queue
|
||||
func (h *healRoutine) queueHealTask(task healTask) { |
||||
h.tasks <- task |
||||
} |
||||
|
||||
// Wait for heal requests and process them
|
||||
func (h *healRoutine) run() { |
||||
ctx := context.Background() |
||||
for { |
||||
select { |
||||
case task, ok := <-h.tasks: |
||||
if !ok { |
||||
break |
||||
} |
||||
if globalHTTPServer != nil { |
||||
// Wait at max 1 minute for an inprogress request
|
||||
// before proceeding to heal
|
||||
waitCount := 60 |
||||
// Any requests in progress, delay the heal.
|
||||
for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 { |
||||
waitCount-- |
||||
time.Sleep(1 * time.Second) |
||||
} |
||||
} |
||||
|
||||
var res madmin.HealResultItem |
||||
var err error |
||||
bucket, object := urlPath2BucketObjectName(task.path) |
||||
switch { |
||||
case bucket == "" && object == "": |
||||
res, err = bgHealDiskFormat(ctx, task.opts) |
||||
case bucket != "" && object == "": |
||||
res, err = bgHealBucket(ctx, bucket, task.opts) |
||||
case bucket != "" && object != "": |
||||
res, err = bgHealObject(ctx, bucket, object, task.opts) |
||||
} |
||||
task.responseCh <- healResult{result: res, err: err} |
||||
case <-h.doneCh: |
||||
return |
||||
case <-GlobalServiceDoneCh: |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
func initHealRoutine() *healRoutine { |
||||
return &healRoutine{ |
||||
tasks: make(chan healTask), |
||||
doneCh: make(chan struct{}), |
||||
} |
||||
|
||||
} |
||||
|
||||
func initBackgroundHealing() { |
||||
healBg := initHealRoutine() |
||||
go healBg.run() |
||||
|
||||
globalBackgroundHealing = healBg |
||||
} |
||||
|
||||
// bgHealDiskFormat - heals format.json, return value indicates if a
|
||||
// failure error occurred.
|
||||
func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealResultItem, error) { |
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn() |
||||
if objectAPI == nil { |
||||
return madmin.HealResultItem{}, errServerNotInitialized |
||||
} |
||||
|
||||
res, err := objectAPI.HealFormat(ctx, opts.DryRun) |
||||
|
||||
// return any error, ignore error returned when disks have
|
||||
// already healed.
|
||||
if err != nil && err != errNoHealRequired { |
||||
return madmin.HealResultItem{}, err |
||||
} |
||||
|
||||
// Healing succeeded notify the peers to reload format and re-initialize disks.
|
||||
// We will not notify peers if healing is not required.
|
||||
if err == nil { |
||||
for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) { |
||||
if nerr.Err != nil { |
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) |
||||
logger.LogIf(ctx, nerr.Err) |
||||
} |
||||
} |
||||
} |
||||
|
||||
return res, nil |
||||
} |
||||
|
||||
// bghealBucket - traverses and heals given bucket
|
||||
func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) { |
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn() |
||||
if objectAPI == nil { |
||||
return madmin.HealResultItem{}, errServerNotInitialized |
||||
} |
||||
|
||||
return objectAPI.HealBucket(ctx, bucket, opts.DryRun, opts.Remove) |
||||
} |
||||
|
||||
// bgHealObject - heal the given object and record result
|
||||
func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) { |
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn() |
||||
if objectAPI == nil { |
||||
return madmin.HealResultItem{}, errServerNotInitialized |
||||
} |
||||
return objectAPI.HealObject(ctx, bucket, object, opts.DryRun, opts.Remove, opts.ScanMode) |
||||
} |
||||
@ -0,0 +1,89 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"context" |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/minio/minio/cmd/logger" |
||||
"github.com/minio/minio/pkg/madmin" |
||||
) |
||||
|
||||
const ( |
||||
bgHealingUUID = "0000-0000-0000-0000" |
||||
) |
||||
|
||||
// NewBgHealSequence creates a background healing sequence
|
||||
// operation which crawls all objects and heal them.
|
||||
func newBgHealSequence(numDisks int) *healSequence { |
||||
|
||||
reqInfo := &logger.ReqInfo{API: "BackgroundHeal"} |
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo) |
||||
|
||||
hs := madmin.HealOpts{ |
||||
// Remove objects that do not have read-quorum
|
||||
Remove: true, |
||||
ScanMode: madmin.HealDeepScan, |
||||
} |
||||
|
||||
return &healSequence{ |
||||
sourceCh: make(chan string), |
||||
startTime: UTCNow(), |
||||
clientToken: bgHealingUUID, |
||||
settings: hs, |
||||
currentStatus: healSequenceStatus{ |
||||
Summary: healNotStartedStatus, |
||||
HealSettings: hs, |
||||
NumDisks: numDisks, |
||||
updateLock: &sync.RWMutex{}, |
||||
}, |
||||
traverseAndHealDoneCh: make(chan error), |
||||
stopSignalCh: make(chan struct{}), |
||||
ctx: ctx, |
||||
reportProgress: false, |
||||
} |
||||
} |
||||
|
||||
func initDailyHeal() { |
||||
go startDailyHeal() |
||||
} |
||||
|
||||
func startDailyHeal() { |
||||
var objAPI ObjectLayer |
||||
var ctx = context.Background() |
||||
|
||||
// Wait until the object API is ready
|
||||
for { |
||||
objAPI = newObjectLayerFn() |
||||
if objAPI == nil { |
||||
time.Sleep(time.Second) |
||||
continue |
||||
} |
||||
break |
||||
} |
||||
|
||||
// Find number of disks in the setup
|
||||
info := objAPI.StorageInfo(ctx) |
||||
numDisks := info.Backend.OnlineDisks + info.Backend.OfflineDisks |
||||
|
||||
nh := newBgHealSequence(numDisks) |
||||
globalSweepHealState.LaunchNewHealSequence(nh) |
||||
|
||||
registerDailySweepListener(nh.sourceCh) |
||||
} |
||||
@ -0,0 +1,145 @@ |
||||
/* |
||||
* MinIO Cloud Storage, (C) 2019 MinIO, Inc. |
||||
* |
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
* you may not use this file except in compliance with the License. |
||||
* You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"context" |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/minio/minio/cmd/logger" |
||||
) |
||||
|
||||
// The list of modules listening for the daily listing of all objects
|
||||
// such as the daily heal ops, disk usage and bucket lifecycle management.
|
||||
var globalDailySweepListeners = make([]chan string, 0) |
||||
var globalDailySweepListenersMu = sync.Mutex{} |
||||
|
||||
// Add a new listener to the daily objects listing
|
||||
func registerDailySweepListener(ch chan string) { |
||||
globalDailySweepListenersMu.Lock() |
||||
defer globalDailySweepListenersMu.Unlock() |
||||
|
||||
globalDailySweepListeners = append(globalDailySweepListeners, ch) |
||||
} |
||||
|
||||
// Safe copy of globalDailySweepListeners content
|
||||
func copyDailySweepListeners() []chan string { |
||||
globalDailySweepListenersMu.Lock() |
||||
defer globalDailySweepListenersMu.Unlock() |
||||
|
||||
var listenersCopy = make([]chan string, len(globalDailySweepListeners)) |
||||
copy(listenersCopy, globalDailySweepListeners) |
||||
|
||||
return listenersCopy |
||||
} |
||||
|
||||
// sweepRound will list all objects, having read quorum or not and
|
||||
// feeds to all listeners, such as the background healing
|
||||
func sweepRound(ctx context.Context, objAPI ObjectLayer) error { |
||||
zeroDuration := time.Millisecond |
||||
zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration) |
||||
|
||||
// General lock so we avoid parallel daily sweep by different instances.
|
||||
sweepLock := globalNSMutex.NewNSLock("system", "daily-sweep") |
||||
if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil { |
||||
return err |
||||
} |
||||
defer sweepLock.Unlock() |
||||
|
||||
buckets, err := objAPI.ListBuckets(ctx) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// List all objects, having read quorum or not in all buckets
|
||||
// and send them to all the registered sweep listeners
|
||||
for _, bucket := range buckets { |
||||
// Send bucket names to all listeners
|
||||
for _, l := range copyDailySweepListeners() { |
||||
l <- bucket.Name |
||||
} |
||||
|
||||
marker := "" |
||||
for { |
||||
res, err := objAPI.ListObjectsHeal(ctx, bucket.Name, "", marker, "", 1000) |
||||
if err != nil { |
||||
continue |
||||
} |
||||
for _, obj := range res.Objects { |
||||
for _, l := range copyDailySweepListeners() { |
||||
l <- pathJoin(bucket.Name, obj.Name) |
||||
} |
||||
} |
||||
if !res.IsTruncated { |
||||
break |
||||
} else { |
||||
marker = res.NextMarker |
||||
} |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// initDailySweeper creates a go-routine which will list all
|
||||
// objects in all buckets in a daily basis
|
||||
func initDailySweeper() { |
||||
go dailySweeper() |
||||
} |
||||
|
||||
// List all objects in all buckets in a daily basis
|
||||
func dailySweeper() { |
||||
var lastSweepTime time.Time |
||||
var objAPI ObjectLayer |
||||
|
||||
var ctx = context.Background() |
||||
|
||||
// Wait until the object layer is ready
|
||||
for { |
||||
objAPI = newObjectLayerFn() |
||||
if objAPI == nil { |
||||
time.Sleep(time.Second) |
||||
continue |
||||
} |
||||
break |
||||
} |
||||
|
||||
// Perform a sweep round each 24 hours
|
||||
for { |
||||
if time.Since(lastSweepTime) < 24*time.Hour { |
||||
time.Sleep(time.Hour) |
||||
continue |
||||
} |
||||
|
||||
err := sweepRound(ctx, objAPI) |
||||
if err != nil { |
||||
switch err.(type) { |
||||
// Unable to hold a lock means there is another
|
||||
// instance doing the sweep round
|
||||
case OperationTimedOut: |
||||
lastSweepTime = time.Now() |
||||
default: |
||||
logger.LogIf(ctx, err) |
||||
time.Sleep(time.Minute) |
||||
continue |
||||
} |
||||
} else { |
||||
lastSweepTime = time.Now() |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue