oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix healing on multiple zones (#8555)

Browse Source 
It is expected in zone healing underlying
callers should return appropriate errors
master
Harshavardhana 5 years ago committed by GitHub
parent fd0fa4e5c5
commit fb43d64dc3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 120 additions and 63 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 10
      cmd/bucket-handlers.go
  2. 45
      cmd/notification.go
  3. 5
      cmd/object-lock.go
  4. 12
      cmd/peer-rest-client.go
  5. 71
      cmd/peer-rest-common.go
  6. 25
      cmd/peer-rest-server.go
  7. 9
      cmd/xl-v1-healing.go
  8. 6
      cmd/xl-v1-healing_test.go

10
cmd/bucket-handlers.go
Unescape View File

@ -564,6 +564,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
return return
} }
globalBucketObjectLockConfig.Set(bucket, Retention{}) globalBucketObjectLockConfig.Set(bucket, Retention{})
globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, Retention{})
} }
// Make sure to add Location information here only for bucket // Make sure to add Location information here only for bucket
@ -896,12 +897,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
} }
} }
globalBucketObjectLockConfig.Delete(bucket)
globalNotificationSys.RemoveNotification(bucket)
globalPolicySys.Remove(bucket)
globalNotificationSys.DeleteBucket(ctx, bucket) globalNotificationSys.DeleteBucket(ctx, bucket)
globalLifecycleSys.Remove(bucket)
globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
// Write success response. // Write success response.
writeSuccessNoContent(w) writeSuccessNoContent(w)
@ -1026,8 +1022,8 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
globalBucketObjectLockConfig.Set(bucket, retention) globalBucketObjectLockConfig.Set(bucket, retention)
globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, retention) globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, retention)
} else { } else {
globalBucketObjectLockConfig.Set(bucket, Retention{}) globalBucketObjectLockConfig.Remove(bucket)
globalBucketObjectLockConfig.Delete(bucket) globalNotificationSys.RemoveBucketObjectLockConfig(ctx, bucket)
} }
// Write success response. // Write success response.

45
cmd/notification.go
Unescape View File

@ -525,6 +525,11 @@ func (sys *NotificationSys) SetBucketPolicy(ctx context.Context, bucketName stri
// DeleteBucket - calls DeleteBucket RPC call on all peers. // DeleteBucket - calls DeleteBucket RPC call on all peers.
func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string) { func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string) {
globalNotificationSys.RemoveNotification(bucketName)
globalBucketObjectLockConfig.Remove(bucketName)
globalPolicySys.Remove(bucketName)
globalLifecycleSys.Remove(bucketName)
go func() { go func() {
ng := WithNPeers(len(sys.peerClients)) ng := WithNPeers(len(sys.peerClients))
for idx, client := range sys.peerClients { for idx, client := range sys.peerClients {
@ -557,6 +562,23 @@ func (sys *NotificationSys) RemoveBucketPolicy(ctx context.Context, bucketName s
}() }()
} }
// RemoveBucketObjectLockConfig - calls RemoveBucketObjectLockConfig RPC call on all peers.
func (sys *NotificationSys) RemoveBucketObjectLockConfig(ctx context.Context, bucketName string) {
go func() {
ng := WithNPeers(len(sys.peerClients))
for idx, client := range sys.peerClients {
if client == nil {
continue
}
client := client
ng.Go(ctx, func() error {
return client.RemoveBucketObjectLockConfig(bucketName)
}, idx, *client.host)
}
ng.Wait()
}()
}
// SetBucketLifecycle - calls SetBucketLifecycle on all peers. // SetBucketLifecycle - calls SetBucketLifecycle on all peers.
func (sys *NotificationSys) SetBucketLifecycle(ctx context.Context, bucketName string, func (sys *NotificationSys) SetBucketLifecycle(ctx context.Context, bucketName string,
bucketLifecycle *lifecycle.Lifecycle) { bucketLifecycle *lifecycle.Lifecycle) {
@ -984,21 +1006,22 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
// PutBucketObjectLockConfig - put bucket object lock configuration to all peers. // PutBucketObjectLockConfig - put bucket object lock configuration to all peers.
func (sys *NotificationSys) PutBucketObjectLockConfig(ctx context.Context, bucketName string, retention Retention) { func (sys *NotificationSys) PutBucketObjectLockConfig(ctx context.Context, bucketName string, retention Retention) {
var wg sync.WaitGroup g := errgroup.WithNErrs(len(sys.peerClients))
for _, client := range sys.peerClients { for index, client := range sys.peerClients {
if client == nil { if client == nil {
continue continue
} }
wg.Add(1) index := index
go func(client *peerRESTClient) { g.Go(func() error {
defer wg.Done() return sys.peerClients[index].PutBucketObjectLockConfig(bucketName, retention)
if err := client.PutBucketObjectLockConfig(bucketName, retention); err != nil { }, index)
logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name) }
logger.LogIf(ctx, err) for i, err := range g.Wait() {
} if err != nil {
}(client) logger.GetReqInfo(ctx).AppendTags("remotePeer", sys.peerClients[i].host.String())
logger.LogIf(ctx, err)
}
} }
wg.Wait()
} }
// NetReadPerfInfo - Network read performance information. // NetReadPerfInfo - Network read performance information.

5
cmd/object-lock.go
Unescape View File

@ -118,8 +118,8 @@ func (config *BucketObjectLockConfig) Get(bucketName string) (r Retention, ok bo
return r, ok return r, ok
} }
// Delete - delete retention configuration. // Remove - removes retention configuration.
func (config *BucketObjectLockConfig) Delete(bucketName string) { func (config *BucketObjectLockConfig) Remove(bucketName string) {
config.Lock() config.Lock()
delete(config.retentionMap, bucketName) delete(config.retentionMap, bucketName)
config.Unlock() config.Unlock()
@ -414,6 +414,7 @@ func checkGovernanceBypassAllowed(ctx context.Context, r *http.Request, bucket,
if err != nil { if err != nil {
// ignore case where object no longer exists // ignore case where object no longer exists
if toAPIError(ctx, err).Code == "NoSuchKey" { if toAPIError(ctx, err).Code == "NoSuchKey" {
oi.UserDefined = map[string]string{}
return oi, ErrNone return oi, ErrNone
} }
return oi, toAPIErrorCode(ctx, err) return oi, toAPIErrorCode(ctx, err)

12
cmd/peer-rest-client.go
Unescape View File

@ -361,6 +361,18 @@ func (client *peerRESTClient) RemoveBucketPolicy(bucket string) error {
return nil return nil
} }
// RemoveBucketObjectLockConfig - Remove bucket object lock config on the peer node.
func (client *peerRESTClient) RemoveBucketObjectLockConfig(bucket string) error {
values := make(url.Values)
values.Set(peerRESTBucket, bucket)
respBody, err := client.call(peerRESTMethodBucketObjectLockConfigRemove, values, nil, -1)
if err != nil {
return err
}
defer http.DrainBody(respBody)
return nil
}
// SetBucketPolicy - Set bucket policy on the peer node. // SetBucketPolicy - Set bucket policy on the peer node.
func (client *peerRESTClient) SetBucketPolicy(bucket string, bucketPolicy *policy.Policy) error { func (client *peerRESTClient) SetBucketPolicy(bucket string, bucketPolicy *policy.Policy) error {
values := make(url.Values) values := make(url.Values)

71
cmd/peer-rest-common.go
Unescape View File

@ -24,41 +24,42 @@ const (
) )
const ( const (
peerRESTMethodNetReadPerfInfo = "/netreadperfinfo" peerRESTMethodNetReadPerfInfo = "/netreadperfinfo"
peerRESTMethodCollectNetPerfInfo = "/collectnetperfinfo" peerRESTMethodCollectNetPerfInfo = "/collectnetperfinfo"
peerRESTMethodServerInfo = "/serverinfo" peerRESTMethodServerInfo = "/serverinfo"
peerRESTMethodCPULoadInfo = "/cpuloadinfo" peerRESTMethodCPULoadInfo = "/cpuloadinfo"
peerRESTMethodMemUsageInfo = "/memusageinfo" peerRESTMethodMemUsageInfo = "/memusageinfo"
peerRESTMethodDrivePerfInfo = "/driveperfinfo" peerRESTMethodDrivePerfInfo = "/driveperfinfo"
peerRESTMethodDeleteBucket = "/deletebucket" peerRESTMethodDeleteBucket = "/deletebucket"
peerRESTMethodServerUpdate = "/serverupdate" peerRESTMethodServerUpdate = "/serverupdate"
peerRESTMethodSignalService = "/signalservice" peerRESTMethodSignalService = "/signalservice"
peerRESTMethodBackgroundHealStatus = "/backgroundhealstatus" peerRESTMethodBackgroundHealStatus = "/backgroundhealstatus"
peerRESTMethodBackgroundOpsStatus = "/backgroundopsstatus" peerRESTMethodBackgroundOpsStatus = "/backgroundopsstatus"
peerRESTMethodGetLocks = "/getlocks" peerRESTMethodGetLocks = "/getlocks"
peerRESTMethodBucketPolicyRemove = "/removebucketpolicy" peerRESTMethodBucketPolicyRemove = "/removebucketpolicy"
peerRESTMethodLoadUser = "/loaduser" peerRESTMethodLoadUser = "/loaduser"
peerRESTMethodDeleteUser = "/deleteuser" peerRESTMethodDeleteUser = "/deleteuser"
peerRESTMethodLoadPolicy = "/loadpolicy" peerRESTMethodLoadPolicy = "/loadpolicy"
peerRESTMethodLoadPolicyMapping = "/loadpolicymapping" peerRESTMethodLoadPolicyMapping = "/loadpolicymapping"
peerRESTMethodDeletePolicy = "/deletepolicy" peerRESTMethodDeletePolicy = "/deletepolicy"
peerRESTMethodLoadUsers = "/loadusers" peerRESTMethodLoadUsers = "/loadusers"
peerRESTMethodLoadGroup = "/loadgroup" peerRESTMethodLoadGroup = "/loadgroup"
peerRESTMethodStartProfiling = "/startprofiling" peerRESTMethodStartProfiling = "/startprofiling"
peerRESTMethodDownloadProfilingData = "/downloadprofilingdata" peerRESTMethodDownloadProfilingData = "/downloadprofilingdata"
peerRESTMethodBucketPolicySet = "/setbucketpolicy" peerRESTMethodBucketPolicySet = "/setbucketpolicy"
peerRESTMethodBucketNotificationPut = "/putbucketnotification" peerRESTMethodBucketNotificationPut = "/putbucketnotification"
peerRESTMethodBucketNotificationListen = "/listenbucketnotification" peerRESTMethodBucketNotificationListen = "/listenbucketnotification"
peerRESTMethodReloadFormat = "/reloadformat" peerRESTMethodReloadFormat = "/reloadformat"
peerRESTMethodTargetExists = "/targetexists" peerRESTMethodTargetExists = "/targetexists"
peerRESTMethodSendEvent = "/sendevent" peerRESTMethodSendEvent = "/sendevent"
peerRESTMethodTrace = "/trace" peerRESTMethodTrace = "/trace"
peerRESTMethodBucketLifecycleSet = "/setbucketlifecycle" peerRESTMethodBucketLifecycleSet = "/setbucketlifecycle"
peerRESTMethodBucketLifecycleRemove = "/removebucketlifecycle" peerRESTMethodBucketLifecycleRemove = "/removebucketlifecycle"
peerRESTMethodLog = "/log" peerRESTMethodLog = "/log"
peerRESTMethodHardwareCPUInfo = "/cpuhardwareinfo" peerRESTMethodHardwareCPUInfo = "/cpuhardwareinfo"
peerRESTMethodHardwareNetworkInfo = "/networkhardwareinfo" peerRESTMethodHardwareNetworkInfo = "/networkhardwareinfo"
peerRESTMethodPutBucketObjectLockConfig = "/putbucketobjectlockconfig" peerRESTMethodPutBucketObjectLockConfig = "/putbucketobjectlockconfig"
peerRESTMethodBucketObjectLockConfigRemove = "/removebucketobjectlockconfig"
) )
const ( const (

25
cmd/peer-rest-server.go
Unescape View File

@ -510,7 +510,8 @@ func (s *peerRESTServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Requ
globalNotificationSys.RemoveNotification(bucketName) globalNotificationSys.RemoveNotification(bucketName)
globalPolicySys.Remove(bucketName) globalPolicySys.Remove(bucketName)
globalBucketObjectLockConfig.Delete(bucketName) globalBucketObjectLockConfig.Remove(bucketName)
globalLifecycleSys.Remove(bucketName)
w.(http.Flusher).Flush() w.(http.Flusher).Flush()
} }
@ -761,6 +762,24 @@ func (s *peerRESTServer) PutBucketNotificationHandler(w http.ResponseWriter, r *
w.(http.Flusher).Flush() w.(http.Flusher).Flush()
} }
// RemoveBucketObjectLockConfigHandler - handles DELETE bucket object lock configuration.
func (s *peerRESTServer) RemoveBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
s.writeErrorResponse(w, errors.New("Invalid request"))
return
}
vars := mux.Vars(r)
bucketName := vars[peerRESTBucket]
if bucketName == "" {
s.writeErrorResponse(w, errors.New("Bucket name is missing"))
return
}
globalBucketObjectLockConfig.Remove(bucketName)
w.(http.Flusher).Flush()
}
// PutBucketObjectLockConfigHandler - handles PUT bucket object lock configuration. // PutBucketObjectLockConfigHandler - handles PUT bucket object lock configuration.
func (s *peerRESTServer) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) { func (s *peerRESTServer) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) { if !s.IsValid(w, r) {
@ -1027,7 +1046,6 @@ func (s *peerRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
func registerPeerRESTHandlers(router *mux.Router) { func registerPeerRESTHandlers(router *mux.Router) {
server := &peerRESTServer{} server := &peerRESTServer{}
subrouter := router.PathPrefix(peerRESTPrefix).Subrouter() subrouter := router.PathPrefix(peerRESTPrefix).Subrouter()
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodPutBucketObjectLockConfig).HandlerFunc(httpTraceHdrs(server.PutBucketObjectLockConfigHandler)).Queries(restQueries(peerRESTBucket)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetReadPerfInfo).HandlerFunc(httpTraceHdrs(server.NetReadPerfInfoHandler)).Queries(restQueries(peerRESTNetPerfSize)...) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetReadPerfInfo).HandlerFunc(httpTraceHdrs(server.NetReadPerfInfoHandler)).Queries(restQueries(peerRESTNetPerfSize)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCollectNetPerfInfo).HandlerFunc(httpTraceHdrs(server.CollectNetPerfInfoHandler)).Queries(restQueries(peerRESTNetPerfSize)...) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCollectNetPerfInfo).HandlerFunc(httpTraceHdrs(server.CollectNetPerfInfoHandler)).Queries(restQueries(peerRESTNetPerfSize)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler)) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler))
@ -1069,6 +1087,9 @@ func registerPeerRESTHandlers(router *mux.Router) {
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLog).HandlerFunc(server.ConsoleLogHandler) subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLog).HandlerFunc(server.ConsoleLogHandler)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodPutBucketObjectLockConfig).HandlerFunc(httpTraceHdrs(server.PutBucketObjectLockConfigHandler)).Queries(restQueries(peerRESTBucket)...)
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBucketObjectLockConfigRemove).HandlerFunc(httpTraceHdrs(server.RemoveBucketObjectLockConfigHandler)).Queries(restQueries(peerRESTBucket)...)
// If none of the routes match add default error handler routes // If none of the routes match add default error handler routes
router.NotFoundHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler)) router.NotFoundHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
router.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler)) router.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))

9
cmd/xl-v1-healing.go
Unescape View File

@ -697,9 +697,10 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
writeQuorum = len(xl.getDisks())/2 + 1 writeQuorum = len(xl.getDisks())/2 + 1
} }
if !dryRun && remove { if !dryRun && remove {
err = xl.deleteObject(healCtx, bucket, object, writeQuorum, false) xl.deleteObject(healCtx, bucket, object, writeQuorum, false)
} }
return defaultHealResult(xlMetaV1{}, storageDisks, errs, bucket, object), err err = reduceReadQuorumErrs(ctx, errs, nil, writeQuorum-1)
return defaultHealResult(xlMetaV1{}, storageDisks, errs, bucket, object), toObjectErr(err, bucket, object)
} }
latestXLMeta, err := getLatestXLMeta(healCtx, partsMetadata, errs) latestXLMeta, err := getLatestXLMeta(healCtx, partsMetadata, errs)
@ -718,7 +719,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
// Only if we get errors from all the disks we return error. Else we need to // Only if we get errors from all the disks we return error. Else we need to
// continue to return filled madmin.HealResultItem struct which includes info // continue to return filled madmin.HealResultItem struct which includes info
// on what disks the file is available etc. // on what disks the file is available etc.
if reducedErr := reduceReadQuorumErrs(ctx, errs, nil, latestXLMeta.Erasure.DataBlocks); reducedErr != nil { if err = reduceReadQuorumErrs(ctx, errs, nil, latestXLMeta.Erasure.DataBlocks); err != nil {
if m, ok := isObjectDangling(partsMetadata, errs, []error{}); ok { if m, ok := isObjectDangling(partsMetadata, errs, []error{}); ok {
writeQuorum := m.Erasure.DataBlocks + 1 writeQuorum := m.Erasure.DataBlocks + 1
if m.Erasure.DataBlocks == 0 { if m.Erasure.DataBlocks == 0 {
@ -728,7 +729,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
xl.deleteObject(ctx, bucket, object, writeQuorum, false) xl.deleteObject(ctx, bucket, object, writeQuorum, false)
} }
} }
return defaultHealResult(latestXLMeta, storageDisks, errs, bucket, object), toObjectErr(reducedErr, bucket, object) return defaultHealResult(latestXLMeta, storageDisks, errs, bucket, object), toObjectErr(err, bucket, object)
} }
} }

6
cmd/xl-v1-healing_test.go
Unescape View File

@ -185,10 +185,12 @@ func TestHealObjectCorrupted(t *testing.T) {
xl.getDisks()[i].DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile)) xl.getDisks()[i].DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile))
} }
// Try healing now, expect to receive errDiskNotFound. // Try healing now, expect to receive errFileNotFound.
_, err = objLayer.HealObject(context.Background(), bucket, object, false, true, madmin.HealDeepScan) _, err = objLayer.HealObject(context.Background(), bucket, object, false, true, madmin.HealDeepScan)
if err != nil { if err != nil {
t.Errorf("Expected nil but received %v", err) if _, ok := err.(ObjectNotFound); !ok {
t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
}
} }
// since majority of xl.jsons are not available, object should be successfully deleted. // since majority of xl.jsons are not available, object should be successfully deleted.

Write Preview
Loading…
Cancel
Save