diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index 4bb0090ad..9552e08f1 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -295,6 +295,20 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ // ignores any errors here. storageInfo, _ := objectAPI.StorageInfo(ctx, false) + // Collect any disk healing. + healing, _ := getAggregatedBackgroundHealState(ctx) + healDisks := make(map[string]struct{}, len(healing.HealDisks)) + for _, disk := range healing.HealDisks { + healDisks[disk] = struct{}{} + } + + // find all disks which belong to each respective endpoints + for i, disk := range storageInfo.Disks { + if _, ok := healDisks[disk.Endpoint]; ok { + storageInfo.Disks[i].Healing = true + } + } + // Marshal API response jsonBytes, err := json.Marshal(storageInfo) if err != nil { diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go index a54fae9a8..ce7a3665d 100644 --- a/cmd/admin-heal-ops.go +++ b/cmd/admin-heal-ops.go @@ -85,7 +85,7 @@ type healSequenceStatus struct { // structure to hold state of all heal sequences in server memory type allHealState struct { - sync.Mutex + sync.RWMutex // map of heal path to heal sequence healSeqMap map[string]*healSequence @@ -105,21 +105,21 @@ func newHealState() *allHealState { } func (ahs *allHealState) healDriveCount() int { - ahs.Lock() - defer ahs.Unlock() + ahs.RLock() + defer ahs.RUnlock() return len(ahs.healLocalDisks) } func (ahs *allHealState) getHealLocalDisks() Endpoints { - ahs.Lock() - defer ahs.Unlock() + ahs.RLock() + defer ahs.RUnlock() - var healLocalDisks Endpoints + var endpoints Endpoints for ep := range ahs.healLocalDisks { - healLocalDisks = append(healLocalDisks, ep) + endpoints = append(endpoints, ep) } - return healLocalDisks + return endpoints } func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) { diff --git a/cmd/background-newdisks-heal-ops.go b/cmd/background-newdisks-heal-ops.go index d56508b23..e84cc4285 100644 --- a/cmd/background-newdisks-heal-ops.go +++ b/cmd/background-newdisks-heal-ops.go @@ -26,7 +26,17 @@ import ( "github.com/minio/minio/cmd/logger" ) -const defaultMonitorNewDiskInterval = time.Second * 10 +const ( + defaultMonitorNewDiskInterval = time.Second * 10 + healingTrackerFilename = ".healing.bin" +) + +//go:generate msgp -file $GOFILE -unexported +type healingTracker struct { + ID string + + // future add more tracking capabilities +} func initAutoHeal(ctx context.Context, objAPI ObjectLayer) { z, ok := objAPI.(*erasureZones) @@ -47,9 +57,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) { time.Sleep(time.Second) } - for _, ep := range getLocalDisksToHeal() { - globalBackgroundHealState.pushHealLocalDisks(ep) - } + globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...) if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 { logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...", @@ -76,9 +84,11 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) { } // Try to connect to the current endpoint // and reformat if the current disk is not formatted - _, _, err := connectEndpoint(endpoint) + disk, _, err := connectEndpoint(endpoint) if errors.Is(err, errUnformattedDisk) { disksToHeal = append(disksToHeal, endpoint) + } else if err == nil && disk != nil && disk.Healing() { + disksToHeal = append(disksToHeal, disk.Endpoint()) } } } @@ -106,7 +116,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS case <-time.After(defaultMonitorNewDiskInterval): waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second) - var erasureSetInZoneEndpointToHeal []map[int]Endpoints + var erasureSetInZoneDisksToHeal []map[int][]StorageAPI + healDisks := globalBackgroundHealState.getHealLocalDisks() if len(healDisks) > 0 { // Reformat disks @@ -118,22 +129,21 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...", len(healDisks))) - erasureSetInZoneEndpointToHeal = make([]map[int]Endpoints, len(z.zones)) + erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.zones)) for i := range z.zones { - erasureSetInZoneEndpointToHeal[i] = map[int]Endpoints{} + erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{} } } // heal only if new disks found. for _, endpoint := range healDisks { - // Load the new format of this passed endpoint - _, format, err := connectEndpoint(endpoint) + disk, format, err := connectEndpoint(endpoint) if err != nil { printEndpointError(endpoint, err, true) continue } - zoneIdx := globalEndpoints.GetLocalZoneIdx(endpoint) + zoneIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint()) if zoneIdx < 0 { continue } @@ -145,32 +155,31 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS continue } - erasureSetInZoneEndpointToHeal[zoneIdx][setIndex] = append(erasureSetInZoneEndpointToHeal[zoneIdx][setIndex], endpoint) + erasureSetInZoneDisksToHeal[zoneIdx][setIndex] = append(erasureSetInZoneDisksToHeal[zoneIdx][setIndex], disk) } - for i, setMap := range erasureSetInZoneEndpointToHeal { - for setIndex, endpoints := range setMap { - for _, ep := range endpoints { - logger.Info("Healing disk '%s' on %s zone", ep, humanize.Ordinal(i+1)) + buckets, _ := z.ListBucketsHeal(ctx) + for i, setMap := range erasureSetInZoneDisksToHeal { + for setIndex, disks := range setMap { + for _, disk := range disks { + logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1)) - buckets, err := z.ListBucketsHeal(ctx) - if err != nil { + lbDisks := z.zones[i].sets[setIndex].getLoadBalancedDisks() + if err := healErasureSet(ctx, setIndex, buckets, lbDisks, z.zones[i].setDriveCount); err != nil { logger.LogIf(ctx, err) continue } - if len(buckets) > 0 { - disks := z.zones[i].sets[setIndex].getLoadBalancedDisks() - if err := healErasureSet(ctx, setIndex, buckets, disks, z.zones[i].setDriveCount); err != nil { - logger.LogIf(ctx, err) - continue - } - } + logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1)) - logger.Info("Healing disk '%s' on %s zone complete", ep, humanize.Ordinal(i+1)) + if err := disk.DeleteFile(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix), + healingTrackerFilename); err != nil { + logger.LogIf(ctx, err) + continue + } // Only upon success pop the healed disk. - globalBackgroundHealState.popHealLocalDisks(ep) + globalBackgroundHealState.popHealLocalDisks(disk.Endpoint()) } } } diff --git a/cmd/background-newdisks-heal-ops_gen.go b/cmd/background-newdisks-heal-ops_gen.go new file mode 100644 index 000000000..fae339a2c --- /dev/null +++ b/cmd/background-newdisks-heal-ops_gen.go @@ -0,0 +1,110 @@ +package cmd + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + z.ID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z healingTracker) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "ID" + err = en.Append(0x81, 0xa2, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.ID) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z healingTracker) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "ID" + o = append(o, 0x81, 0xa2, 0x49, 0x44) + o = msgp.AppendString(o, z.ID) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + z.ID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z healingTracker) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + return +} diff --git a/cmd/background-newdisks-heal-ops_gen_test.go b/cmd/background-newdisks-heal-ops_gen_test.go new file mode 100644 index 000000000..177aa91ab --- /dev/null +++ b/cmd/background-newdisks-heal-ops_gen_test.go @@ -0,0 +1,123 @@ +package cmd + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalhealingTracker(t *testing.T) { + v := healingTracker{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsghealingTracker(b *testing.B) { + v := healingTracker{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsghealingTracker(b *testing.B) { + v := healingTracker{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalhealingTracker(b *testing.B) { + v := healingTracker{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodehealingTracker(t *testing.T) { + v := healingTracker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodehealingTracker Msgsize() is inaccurate") + } + + vn := healingTracker{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodehealingTracker(b *testing.B) { + v := healingTracker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodehealingTracker(b *testing.B) { + v := healingTracker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/erasure-bucket.go b/cmd/erasure-bucket.go index d483b772d..f632c4ed4 100644 --- a/cmd/erasure-bucket.go +++ b/cmd/erasure-bucket.go @@ -85,31 +85,40 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) { // getBucketInfo - returns the BucketInfo from one of the load balanced disks. func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (bucketInfo BucketInfo, err error) { - var bucketErrs []error - for _, disk := range er.getLoadBalancedDisks() { - if disk == nil { - bucketErrs = append(bucketErrs, errDiskNotFound) - continue - } - volInfo, serr := disk.StatVol(ctx, bucketName) - if serr == nil { - return BucketInfo(volInfo), nil - } - err = serr - // For any reason disk went offline continue and pick the next one. - if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) { - bucketErrs = append(bucketErrs, err) - continue + storageDisks := er.getDisks() + + g := errgroup.WithNErrs(len(storageDisks)) + var bucketsInfo = make([]BucketInfo, len(storageDisks)) + // Undo previous make bucket entry on all underlying storage disks. + for index := range storageDisks { + index := index + g.Go(func() error { + if storageDisks[index] == nil { + return errDiskNotFound + } + volInfo, err := storageDisks[index].StatVol(ctx, bucketName) + if err != nil { + return err + } + bucketsInfo[index] = BucketInfo(volInfo) + return nil + }, index) + } + + errs := g.Wait() + + for i, err := range errs { + if err == nil { + return bucketsInfo[i], nil } - // Any error which cannot be ignored, we return quickly. - return BucketInfo{}, err } + // If all our errors were ignored, then we try to // reduce to one error based on read quorum. // `nil` is deliberately passed for ignoredErrs // because these errors were already ignored. - readQuorum := getReadQuorum(len(er.getDisks())) - return BucketInfo{}, reduceReadQuorumErrs(ctx, bucketErrs, nil, readQuorum) + readQuorum := getReadQuorum(len(storageDisks)) + return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, readQuorum) } // GetBucketInfo - returns BucketInfo for a bucket. diff --git a/cmd/erasure-common.go b/cmd/erasure-common.go index eda68feb4..69fe57eee 100644 --- a/cmd/erasure-common.go +++ b/cmd/erasure-common.go @@ -28,7 +28,9 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) { // Based on the random shuffling return back randomized disks. for _, i := range hashOrder(UTCNow().String(), len(disks)) { if disks[i-1] != nil && disks[i-1].IsLocal() { - newDisks = append(newDisks, disks[i-1]) + if !disks[i-1].Healing() && disks[i-1].IsOnline() { + newDisks = append(newDisks, disks[i-1]) + } } } return newDisks @@ -40,9 +42,6 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) { func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAPI) { disks := er.getLoadBalancedDisks() for _, disk := range disks { - if disk == nil { - continue - } newDisks = append(newDisks, disk) ndisks-- if ndisks == 0 { @@ -53,11 +52,16 @@ func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAP } // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice. +// ensures to skip disks if they are not healing and online. func (er erasureObjects) getLoadBalancedDisks() (newDisks []StorageAPI) { disks := er.getDisks() + // Based on the random shuffling return back randomized disks. for _, i := range hashOrder(UTCNow().String(), len(disks)) { - newDisks = append(newDisks, disks[i-1]) + // Do not consume disks which are being healed. + if disks[i-1] != nil && !disks[i-1].Healing() && disks[i-1].IsOnline() { + newDisks = append(newDisks, disks[i-1]) + } } return newDisks } diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index f352857e1..81ec7685e 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -148,10 +148,8 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec result.Delimiter = delimiter var uploadIDs []string - for _, disk := range er.getLoadBalancedDisks() { - if disk == nil { - continue - } + var disk StorageAPI + for _, disk = range er.getLoadBalancedDisks() { uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1) if err != nil { if err == errDiskNotFound { @@ -176,30 +174,20 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec populatedUploadIds := set.NewStringSet() -retry: - for _, disk := range er.getLoadBalancedDisks() { - if disk == nil { + for _, uploadID := range uploadIDs { + if populatedUploadIds.Contains(uploadID) { continue } - for _, uploadID := range uploadIDs { - if populatedUploadIds.Contains(uploadID) { - continue - } - fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "") - if err != nil { - if err == errDiskNotFound || err == errFileNotFound { - goto retry - } - return result, toObjectErr(err, bucket, object) - } - populatedUploadIds.Add(uploadID) - uploads = append(uploads, MultipartInfo{ - Object: object, - UploadID: uploadID, - Initiated: fi.ModTime, - }) + fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "") + if err != nil { + return result, toObjectErr(err, bucket, object) } - break + populatedUploadIds.Add(uploadID) + uploads = append(uploads, MultipartInfo{ + Object: object, + UploadID: uploadID, + Initiated: fi.ModTime, + }) } sort.Slice(uploads, func(i int, j int) bool { diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index 858b893cf..5d78bf0b7 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -231,6 +231,11 @@ func (s *erasureSets) connectDisks() { return } disk.SetDiskID(format.Erasure.This) + if endpoint.IsLocal && disk.Healing() { + globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint()) + logger.Info(fmt.Sprintf("Found the drive %s which needs healing, attempting to heal...", disk)) + } + s.erasureDisksMu.Lock() if s.erasureDisks[setIndex][diskIndex] != nil { s.erasureDisks[setIndex][diskIndex].Close() @@ -316,7 +321,7 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto endpointStrings: endpointStrings, setCount: setCount, setDriveCount: setDriveCount, - listTolerancePerSet: setDriveCount / 2, + listTolerancePerSet: 3, // Expect 3 good entries across disks. format: format, disksConnectEvent: make(chan diskConnectInfo), distributionAlgo: format.Erasure.DistributionAlgo, @@ -1385,7 +1390,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H } // Save formats `format.json` across all disks. - if err = saveFormatErasureAll(ctx, storageDisks, tmpNewFormats); err != nil { + if err = saveFormatErasureAllWithErrs(ctx, storageDisks, sErrs, tmpNewFormats); err != nil { return madmin.HealResultItem{}, err } diff --git a/cmd/erasure-zones.go b/cmd/erasure-zones.go index 0d7e89a0d..bdaca636b 100644 --- a/cmd/erasure-zones.go +++ b/cmd/erasure-zones.go @@ -661,7 +661,7 @@ func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix, for _, zone := range z.zones { zonesEntryChs = append(zonesEntryChs, - zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.setDriveCount, false)) + zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.listTolerancePerSet, false)) zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet) } @@ -780,7 +780,7 @@ func (z *erasureZones) listObjectsSplunk(ctx context.Context, bucket, prefix, ma entryChs, endWalkCh := zone.poolSplunk.Release(listParams{bucket, recursive, marker, prefix}) if entryChs == nil { endWalkCh = make(chan struct{}) - entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount, true) + entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet, true) } zonesEntryChs = append(zonesEntryChs, entryChs) zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh) @@ -872,7 +872,7 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker, entryChs, endWalkCh := zone.pool.Release(listParams{bucket, recursive, marker, prefix}) if entryChs == nil { endWalkCh = make(chan struct{}) - entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount, false) + entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet, false) } zonesEntryChs = append(zonesEntryChs, entryChs) zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh) @@ -1274,7 +1274,7 @@ func (z *erasureZones) listObjectVersions(ctx context.Context, bucket, prefix, m entryChs, endWalkCh := zone.poolVersions.Release(listParams{bucket, recursive, marker, prefix}) if entryChs == nil { endWalkCh = make(chan struct{}) - entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount) + entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet) } zonesEntryChs = append(zonesEntryChs, entryChs) zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh) diff --git a/cmd/erasure.go b/cmd/erasure.go index cee84a965..ce9579997 100644 --- a/cmd/erasure.go +++ b/cmd/erasure.go @@ -167,6 +167,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di AvailableSpace: info.Free, UUID: info.ID, RootDisk: info.RootDisk, + Healing: info.Healing, State: diskErrToDriveState(err), } if info.Total > 0 { @@ -256,46 +257,20 @@ func (er erasureObjects) StorageInfo(ctx context.Context, local bool) (StorageIn // Updates are sent on a regular basis and the caller *must* consume them. func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error { if len(buckets) == 0 { + logger.Info(color.Green("data-crawl:") + " No buckets found, skipping crawl") return nil } - // Collect any disk healing. - healing, err := getAggregatedBackgroundHealState(ctx) - if err != nil { - return err - } - - healDisks := make(map[string]struct{}, len(healing.HealDisks)) - for _, disk := range healing.HealDisks { - healDisks[disk] = struct{}{} - } - // Collect disks we can use. - var disks []StorageAPI - for _, d := range er.getLoadBalancedDisks() { - if d == nil || !d.IsOnline() { - continue - } - di, err := d.DiskInfo(ctx) - if err != nil { - logger.LogIf(ctx, err) - continue - } - if _, ok := healDisks[di.Endpoint]; ok { - logger.Info(color.Green("data-crawl:")+" Disk %q is Healing, skipping disk.", di.Endpoint) - continue - } - disks = append(disks, d) - } + disks := er.getLoadBalancedDisks() if len(disks) == 0 { - logger.Info(color.Green("data-crawl:") + " No disks found, skipping crawl") + logger.Info(color.Green("data-crawl:") + " all disks are offline or being healed, skipping crawl") return nil } // Load bucket totals oldCache := dataUsageCache{} - err = oldCache.load(ctx, er, dataUsageCacheName) - if err != nil { + if err := oldCache.load(ctx, er, dataUsageCacheName); err != nil { return err } @@ -403,6 +378,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc // Calc usage before := cache.Info.LastUpdate + var err error cache, err = disk.CrawlAndGetDataUsage(ctx, cache) cache.Info.BloomFilter = nil if err != nil { diff --git a/cmd/format-erasure.go b/cmd/format-erasure.go index 924f6ae43..6d7352195 100644 --- a/cmd/format-erasure.go +++ b/cmd/format-erasure.go @@ -21,6 +21,7 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "io/ioutil" "reflect" @@ -335,7 +336,7 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur return formats, g.Wait() } -func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error { +func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) error { if disk == nil || format == nil { return errDiskNotFound } @@ -368,6 +369,18 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error { } disk.SetDiskID(diskID) + if heal { + htracker := healingTracker{ + ID: diskID, + } + htrackerBytes, err := htracker.MarshalMsg(nil) + if err != nil { + return err + } + return disk.WriteAll(context.TODO(), minioMetaBucket, + pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename), + bytes.NewReader(htrackerBytes)) + } return nil } @@ -551,7 +564,8 @@ func formatErasureFixLocalDeploymentID(endpoints Endpoints, storageDisks []Stora return nil } format.ID = refFormat.ID - if err := saveFormatErasure(storageDisks[index], format); err != nil { + // Heal the drive if we fixed its deployment ID. + if err := saveFormatErasure(storageDisks[index], format, true); err != nil { logger.LogIf(GlobalContext, err) return fmt.Errorf("Unable to save format.json, %w", err) } @@ -686,6 +700,27 @@ func initErasureMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*fo return nil } +// saveFormatErasureAllWithErrs - populates `format.json` on disks in its order. +// also adds `.healing.bin` on the disks which are being actively healed. +func saveFormatErasureAllWithErrs(ctx context.Context, storageDisks []StorageAPI, fErrs []error, formats []*formatErasureV3) error { + g := errgroup.WithNErrs(len(storageDisks)) + + // Write `format.json` to all disks. + for index := range storageDisks { + index := index + g.Go(func() error { + if formats[index] == nil { + return errDiskNotFound + } + return saveFormatErasure(storageDisks[index], formats[index], errors.Is(fErrs[index], errUnformattedDisk)) + }, index) + } + + writeQuorum := getWriteQuorum(len(storageDisks)) + // Wait for the routines to finish. + return reduceWriteQuorumErrs(ctx, g.Wait(), nil, writeQuorum) +} + // saveFormatErasureAll - populates `format.json` on disks in its order. func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatErasureV3) error { g := errgroup.WithNErrs(len(storageDisks)) @@ -697,7 +732,7 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format if formats[index] == nil { return errDiskNotFound } - return saveFormatErasure(storageDisks[index], formats[index]) + return saveFormatErasure(storageDisks[index], formats[index], false) }, index) } @@ -771,7 +806,8 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats } if formats[i].Erasure.This == "" { formats[i].Erasure.This = formats[i].Erasure.Sets[0][i] - if err := saveFormatErasure(storageDisks[i], formats[i]); err != nil { + // Heal the drive if drive has .This empty. + if err := saveFormatErasure(storageDisks[i], formats[i], true); err != nil { return err } } diff --git a/cmd/http/server.go b/cmd/http/server.go index f8f594b40..fa1740751 100644 --- a/cmd/http/server.go +++ b/cmd/http/server.go @@ -130,6 +130,9 @@ func (srv *Server) Shutdown() error { srv.listenerMutex.Lock() err := srv.listener.Close() srv.listenerMutex.Unlock() + if err != nil { + return err + } // Wait for opened connection to be closed up to Shutdown timeout. shutdownTimeout := srv.ShutdownTimeout @@ -144,12 +147,12 @@ func (srv *Server) Shutdown() error { if err == nil { _ = pprof.Lookup("goroutine").WriteTo(tmp, 1) tmp.Close() - return errors.New("timed out. some connections are still active. doing abnormal shutdown. goroutines written to " + tmp.Name()) + return errors.New("timed out. some connections are still active. goroutines written to " + tmp.Name()) } - return errors.New("timed out. some connections are still active. doing abnormal shutdown") + return errors.New("timed out. some connections are still active") case <-ticker.C: if atomic.LoadInt32(&srv.requestCount) <= 0 { - return err + return nil } } } diff --git a/cmd/naughty-disk_test.go b/cmd/naughty-disk_test.go index 8c85b4a1e..4146a5b4e 100644 --- a/cmd/naughty-disk_test.go +++ b/cmd/naughty-disk_test.go @@ -58,10 +58,18 @@ func (d *naughtyDisk) IsLocal() bool { return d.disk.IsLocal() } +func (d *naughtyDisk) Endpoint() Endpoint { + return d.disk.Endpoint() +} + func (d *naughtyDisk) Hostname() string { return d.disk.Hostname() } +func (d *naughtyDisk) Healing() bool { + return d.disk.Healing() +} + func (d *naughtyDisk) Close() (err error) { if err = d.calcError(); err != nil { return err diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index efedc0666..f2e481e00 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -141,7 +141,7 @@ func formatErasureCleanupTmpLocalEndpoints(endpoints Endpoints) error { return fmt.Errorf("unable to rename (%s -> %s) %w", pathJoin(epPath, minioMetaTmpBucket), tmpOld, - err) + osErrToFileErr(err)) } // Removal of tmp-old folder is backgrounded completely. diff --git a/cmd/server-main.go b/cmd/server-main.go index 3f3c74769..9f41d579d 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -464,12 +464,12 @@ func serverMain(ctx *cli.Context) { } newObject, err := newObjectLayer(GlobalContext, globalEndpoints) - logger.SetDeploymentID(globalDeploymentID) if err != nil { - globalHTTPServer.Shutdown() - logger.Fatal(err, "Unable to initialize backend") + logFatalErrs(err, Endpoint{}, true) } + logger.SetDeploymentID(globalDeploymentID) + // Once endpoints are finalized, initialize the new object api in safe mode. globalObjLayerMutex.Lock() globalSafeMode = true diff --git a/cmd/server-rlimit.go b/cmd/server-rlimit.go index 7002d4001..79c0a87f1 100644 --- a/cmd/server-rlimit.go +++ b/cmd/server-rlimit.go @@ -24,7 +24,6 @@ import ( func setMaxResources() (err error) { // Set the Go runtime max threads threshold to 90% of kernel setting. - // Do not return when an error when encountered since it is not a crucial task. sysMaxThreads, mErr := sys.GetMaxThreads() if mErr == nil { minioMaxThreads := (sysMaxThreads * 90) / 100 diff --git a/cmd/storage-errors.go b/cmd/storage-errors.go index efb0e104f..9e0cf5995 100644 --- a/cmd/storage-errors.go +++ b/cmd/storage-errors.go @@ -55,7 +55,7 @@ var errFileNotFound = StorageErr("file not found") var errFileVersionNotFound = StorageErr("file version not found") // errTooManyOpenFiles - too many open files. -var errTooManyOpenFiles = StorageErr("too many open files") +var errTooManyOpenFiles = StorageErr("too many open files, please increase 'ulimit -n'") // errFileNameTooLong - given file name is too long than supported length. var errFileNameTooLong = StorageErr("file name too long") diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go index b47a927ce..e6b056612 100644 --- a/cmd/storage-interface.go +++ b/cmd/storage-interface.go @@ -30,10 +30,13 @@ type StorageAPI interface { IsOnline() bool // Returns true if disk is online. IsLocal() bool - Hostname() string // Returns host name if remote host. + Hostname() string // Returns host name if remote host. + Endpoint() Endpoint // Returns endpoint. + Close() error GetDiskID() (string, error) SetDiskID(id string) + Healing() bool // Returns if disk is healing. DiskInfo(ctx context.Context) (info DiskInfo, err error) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 6ce79251d..035477261 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -156,6 +156,14 @@ func (client *storageRESTClient) Hostname() string { return client.endpoint.Host } +func (client *storageRESTClient) Endpoint() Endpoint { + return client.endpoint +} + +func (client *storageRESTClient) Healing() bool { + return false +} + func (client *storageRESTClient) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) { b := cache.serialize() respBody, err := client.call(ctx, storageRESTMethodCrawlAndGetDataUsage, url.Values{}, bytes.NewBuffer(b), int64(len(b))) diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index 1d4191593..a4e7ac1c7 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -826,6 +826,69 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req w.(http.Flusher).Flush() } +// A single function to write certain errors to be fatal +// or informative based on the `exit` flag, please look +// at each implementation of error for added hints. +// +// FIXME: This is an unusual function but serves its purpose for +// now, need to revist the overall erroring structure here. +// Do not like it :-( +func logFatalErrs(err error, endpoint Endpoint, exit bool) { + if errors.Is(err, errMinDiskSize) { + logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint(err.Error()), "Unable to initialize backend") + } else if errors.Is(err, errUnsupportedDisk) { + var hint string + if endpoint.URL != nil { + hint = fmt.Sprintf("Disk '%s' does not support O_DIRECT flags, MinIO erasure coding requires filesystems with O_DIRECT support", endpoint.Path) + } else { + hint = "Disks do not support O_DIRECT flags, MinIO erasure coding requires filesystems with O_DIRECT support" + } + logger.Fatal(config.ErrUnsupportedBackend(err).Hint(hint), "Unable to initialize backend") + } else if errors.Is(err, errDiskNotDir) { + var hint string + if endpoint.URL != nil { + hint = fmt.Sprintf("Disk '%s' is not a directory, MinIO erasure coding needs a directory", endpoint.Path) + } else { + hint = "Disks are not directories, MinIO erasure coding needs directories" + } + logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint(hint), "Unable to initialize backend") + } else if errors.Is(err, errFileAccessDenied) { + // Show a descriptive error with a hint about how to fix it. + var username string + if u, err := user.Current(); err == nil { + username = u.Username + } else { + username = "" + } + var hint string + if endpoint.URL != nil { + hint = fmt.Sprintf("Run the following command to add write permissions: `sudo chown -R %s %s && sudo chmod u+rxw %s`", + username, endpoint.Path, endpoint.Path) + } else { + hint = fmt.Sprintf("Run the following command to add write permissions: `sudo chown -R %s. && sudo chmod u+rxw `", username) + } + logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint(hint), "Unable to initialize backend") + } else if errors.Is(err, errFaultyDisk) { + if !exit { + logger.LogIf(GlobalContext, fmt.Errorf("disk is faulty at %s, please replace the drive - disk will be offline", endpoint)) + } else { + logger.Fatal(err, "Unable to initialize backend") + } + } else if errors.Is(err, errDiskFull) { + if !exit { + logger.LogIf(GlobalContext, fmt.Errorf("disk is already full at %s, incoming I/O will fail - disk will be offline", endpoint)) + } else { + logger.Fatal(err, "Unable to initialize backend") + } + } else { + if !exit { + logger.LogIf(GlobalContext, fmt.Errorf("disk returned an unexpected error at %s, please investigate - disk will be offline", endpoint)) + } else { + logger.Fatal(err, "Unable to initialize backend") + } + } +} + // registerStorageRPCRouter - register storage rpc router. func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones) { for _, ep := range endpointZones { @@ -835,32 +898,9 @@ func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones } storage, err := newXLStorage(endpoint) if err != nil { - switch err { - case errMinDiskSize: - logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint(err.Error()), "Unable to initialize backend") - case errUnsupportedDisk: - hint := fmt.Sprintf("'%s' does not support O_DIRECT flags, MinIO erasure coding requires filesystems with O_DIRECT support", endpoint.Path) - logger.Fatal(config.ErrUnsupportedBackend(err).Hint(hint), "Unable to initialize backend") - case errDiskNotDir: - hint := fmt.Sprintf("'%s' MinIO erasure coding needs a directory", endpoint.Path) - logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint(hint), "Unable to initialize backend") - case errFileAccessDenied: - // Show a descriptive error with a hint about how to fix it. - var username string - if u, err := user.Current(); err == nil { - username = u.Username - } else { - username = "" - } - hint := fmt.Sprintf("Run the following command to add write permissions: `sudo chown -R %s %s && sudo chmod u+rxw %s`", username, endpoint.Path, endpoint.Path) - logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint(hint), "Unable to initialize posix backend") - case errFaultyDisk: - logger.LogIf(GlobalContext, fmt.Errorf("disk is faulty at %s, please replace the drive", endpoint)) - case errDiskFull: - logger.LogIf(GlobalContext, fmt.Errorf("disk is already full at %s, incoming I/O will fail", endpoint)) - default: - logger.LogIf(GlobalContext, fmt.Errorf("disk returned an unexpected error at %s, please investigate", endpoint)) - } + // if supported errors don't fail, we proceed to + // printing message and moving forward. + logFatalErrs(err, endpoint, false) } server := &storageRESTServer{storage: storage} diff --git a/cmd/xl-storage-disk-id-check.go b/cmd/xl-storage-disk-id-check.go index 03681f8ce..b1640b8e3 100644 --- a/cmd/xl-storage-disk-id-check.go +++ b/cmd/xl-storage-disk-id-check.go @@ -43,10 +43,18 @@ func (p *xlStorageDiskIDCheck) IsLocal() bool { return p.storage.IsLocal() } +func (p *xlStorageDiskIDCheck) Endpoint() Endpoint { + return p.storage.Endpoint() +} + func (p *xlStorageDiskIDCheck) Hostname() string { return p.storage.Hostname() } +func (p *xlStorageDiskIDCheck) Healing() bool { + return p.storage.Healing() +} + func (p *xlStorageDiskIDCheck) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) { if err := p.checkDiskStale(); err != nil { return dataUsageCache{}, err diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index fea10963f..6a311a842 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -92,8 +92,7 @@ type xlStorage struct { activeIOCount int32 diskPath string - hostname string - endpoint string + endpoint Endpoint pool sync.Pool @@ -249,7 +248,6 @@ func newLocalXLStorage(path string) (*xlStorage, error) { // Initialize a new storage disk. func newXLStorage(ep Endpoint) (*xlStorage, error) { path := ep.Path - hostname := ep.Host var err error if path, err = getValidPath(path, true); err != nil { return nil, err @@ -262,8 +260,7 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) { p := &xlStorage{ diskPath: path, - hostname: hostname, - endpoint: ep.String(), + endpoint: ep, pool: sync.Pool{ New: func() interface{} { b := disk.AlignedBlock(readBlockSize) @@ -319,7 +316,11 @@ func (s *xlStorage) String() string { } func (s *xlStorage) Hostname() string { - return s.hostname + return s.endpoint.Host +} + +func (s *xlStorage) Endpoint() Endpoint { + return s.endpoint } func (*xlStorage) Close() error { @@ -334,6 +335,13 @@ func (s *xlStorage) IsLocal() bool { return true } +func (s *xlStorage) Healing() bool { + healingFile := pathJoin(s.diskPath, minioMetaBucket, + bucketMetaPrefix, healingTrackerFilename) + _, err := os.Stat(healingFile) + return err == nil +} + func (s *xlStorage) waitForLowActiveIO() { max := lowActiveIOWaitMaxN for atomic.LoadInt32(&s.activeIOCount) >= s.maxActiveIOCount { @@ -439,6 +447,7 @@ type DiskInfo struct { Free uint64 Used uint64 RootDisk bool + Healing bool Endpoint string MountPath string ID string @@ -462,9 +471,10 @@ func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) { Total: di.Total, Free: di.Free, Used: di.Total - di.Free, + Healing: s.Healing(), RootDisk: s.rootDisk, MountPath: s.diskPath, - Endpoint: s.endpoint, + Endpoint: s.endpoint.String(), } diskID, err := s.GetDiskID() diff --git a/go.sum b/go.sum index 29bf655f2..08b109de4 100644 --- a/go.sum +++ b/go.sum @@ -469,7 +469,6 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -532,7 +531,6 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -601,8 +599,6 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200425043458-8463f397d07c h1:iHhCR0b26amDCiiO+kBguKZom9aMF+NrFxh9zeKR/XU= golang.org/x/tools v0.0.0-20200425043458-8463f397d07c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200814172026-c4923e618c08 h1:sfBQLM20fzeXhOixVQirwEbuW4PGStP773EXQpsBB6E= -golang.org/x/tools v0.0.0-20200814172026-c4923e618c08/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200925191224-5d1fdd8fa346 h1:hzJjkvxUIF3bSt+v8N5tBQNx/605vszZJ+3XsIamzZo= golang.org/x/tools v0.0.0-20200925191224-5d1fdd8fa346/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=