minio/cmd/xl-v1-healing-common.go

/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"encoding/hex"
	"path/filepath"
	"time"
)

// commonTime returns a maximally occurring time from a list of time.
func commonTime(modTimes []time.Time) (modTime time.Time, count int) {
	var maxima int // Counter for remembering max occurrence of elements.
	timeOccurenceMap := make(map[time.Time]int)
	// Ignore the uuid sentinel and count the rest.
	for _, time := range modTimes {
		if time == timeSentinel {
			continue
		}
		timeOccurenceMap[time]++
	}
	// Find the common cardinality from previously collected
	// occurrences of elements.
	for time, count := range timeOccurenceMap {
		if count == maxima && time.After(modTime) {
			maxima = count
			modTime = time

		} else if count > maxima {
			maxima = count
			modTime = time
		}
	}
	// Return the collected common uuid.
	return modTime, maxima
}

// Beginning of unix time is treated as sentinel value here.
var timeSentinel = time.Unix(0, 0).UTC()

// Boot modTimes up to disk count, setting the value to time sentinel.
func bootModtimes(diskCount int) []time.Time {
	modTimes := make([]time.Time, diskCount)
	// Boots up all the modtimes.
	for i := range modTimes {
		modTimes[i] = timeSentinel
	}
	return modTimes
}

// Extracts list of times from xlMetaV1 slice and returns, skips
// slice elements which have errors.
func listObjectModtimes(partsMetadata []xlMetaV1, errs []error) (modTimes []time.Time) {
	modTimes = bootModtimes(len(partsMetadata))
	for index, metadata := range partsMetadata {
		if errs[index] != nil {
			continue
		}
		// Once the file is found, save the uuid saved on disk.
		modTimes[index] = metadata.Stat.ModTime
	}
	return modTimes
}

// Notes:
// There are 5 possible states a disk could be in,
// 1. __online__             - has the latest copy of xl.json - returned by listOnlineDisks
//
// 2. __offline__            - err == errDiskNotFound
//
// 3. __availableWithParts__ - has the latest copy of xl.json and has all
//                             parts with checksums matching; returned by disksWithAllParts
//
// 4. __outdated__           - returned by outDatedDisk, provided []StorageAPI
//                             returned by diskWithAllParts is passed for latestDisks.
//    - has an old copy of xl.json
//    - doesn't have xl.json (errFileNotFound)
//    - has the latest xl.json but one or more parts are corrupt
//
// 5. __missingParts__       - has the latest copy of xl.json but has some parts
// missing.  This is identified separately since this may need manual
// inspection to understand the root cause. E.g, this could be due to
// backend filesystem corruption.

// listOnlineDisks - returns
// - a slice of disks where disk having 'older' xl.json (or nothing)
// are set to nil.
// - latest (in time) of the maximally occurring modTime(s).
func listOnlineDisks(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error) (onlineDisks []StorageAPI, modTime time.Time) {
	onlineDisks = make([]StorageAPI, len(disks))

	// List all the file commit ids from parts metadata.
	modTimes := listObjectModtimes(partsMetadata, errs)

	// Reduce list of UUIDs to a single common value.
	modTime, _ = commonTime(modTimes)

	// Create a new online disks slice, which have common uuid.
	for index, t := range modTimes {
		if t == modTime {
			onlineDisks[index] = disks[index]
		} else {
			onlineDisks[index] = nil
		}
	}
	return onlineDisks, modTime
}

// outDatedDisks - return disks which don't have the latest object (i.e xl.json).
// disks that are offline are not 'marked' outdated.
func outDatedDisks(disks, latestDisks []StorageAPI, errs []error, partsMetadata []xlMetaV1,
	bucket, object string) (outDatedDisks []StorageAPI) {

	outDatedDisks = make([]StorageAPI, len(disks))
	for index, latestDisk := range latestDisks {
		if latestDisk != nil {
			continue
		}
		// disk either has an older xl.json or doesn't have one.
		switch errorCause(errs[index]) {
		case nil, errFileNotFound:
			outDatedDisks[index] = disks[index]
		}
	}

	return outDatedDisks
}

// Returns if the object should be healed.
func xlShouldHeal(partsMetadata []xlMetaV1, errs []error) bool {
	modTime, _ := commonTime(listObjectModtimes(partsMetadata, errs))
	for index := range partsMetadata {
		if errs[index] == errDiskNotFound {
			continue
		}
		if errs[index] != nil {
			return true
		}
		if modTime != partsMetadata[index].Stat.ModTime {
			return true
		}
	}
	return false
}

// xlHealStat - returns a structure which describes how many data,
// parity erasure blocks are missing and if it is possible to heal
// with the blocks present.
func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealObjectInfo {
	// Less than quorum erasure coded blocks of the object have the same create time.
	// This object can't be healed with the information we have.
	modTime, count := commonTime(listObjectModtimes(partsMetadata, errs))
	if count < xl.readQuorum {
		return HealObjectInfo{
			Status:              quorumUnavailable,
			MissingDataCount:    0,
			MissingPartityCount: 0,
		}
	}

	// If there isn't a valid xlMeta then we can't heal the object.
	xlMeta, err := pickValidXLMeta(partsMetadata, modTime)
	if err != nil {
		return HealObjectInfo{
			Status:              corrupted,
			MissingDataCount:    0,
			MissingPartityCount: 0,
		}
	}

	// Compute heal statistics like bytes to be healed, missing
	// data and missing parity count.
	missingDataCount := 0
	missingParityCount := 0

	for i, err := range errs {
		// xl.json is not found, which implies the erasure
		// coded blocks are unavailable in the corresponding disk.
		// First half of the disks are data and the rest are parity.
		if realErr := errorCause(err); realErr == errFileNotFound || realErr == errDiskNotFound {
			if xlMeta.Erasure.Distribution[i]-1 < xl.dataBlocks {
				missingDataCount++
			} else {
				missingParityCount++
			}
		}
	}

	// This object can be healed. We have enough object metadata
	// to reconstruct missing erasure coded blocks.
	return HealObjectInfo{
		Status:              canHeal,
		MissingDataCount:    missingDataCount,
		MissingPartityCount: missingParityCount,
	}
}

// disksWithAllParts - This function needs to be called with
// []StorageAPI returned by listOnlineDisks. Returns,
// - disks which have all parts specified in the latest xl.json.
// - errs updated to have errFileNotFound in place of disks that had
// missing parts.
// - non-nil error if any of the online disks failed during
// calculating blake2b checksum.
func disksWithAllParts(onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket, object string) ([]StorageAPI, []error, error) {
	availableDisks := make([]StorageAPI, len(onlineDisks))
	for index, onlineDisk := range onlineDisks {
		if onlineDisk == nil {
			continue
		}
		// disk has a valid xl.json but may not have all the
		// parts. This is considered an outdated disk, since
		// it needs healing too.
		for pIndex, part := range partsMetadata[index].Parts {
			// compute blake2b sum of part.
			partPath := filepath.Join(object, part.Name)
			hash := newHash(blake2bAlgo)
			blakeBytes, hErr := hashSum(onlineDisk, bucket, partPath, hash)
			if hErr == errFileNotFound {
				errs[index] = errFileNotFound
				continue
			}

			if hErr != nil && hErr != errFileNotFound {
				return nil, nil, traceError(hErr)
			}

			partChecksum := partsMetadata[index].Erasure.Checksum[pIndex].Hash
			blakeSum := hex.EncodeToString(blakeBytes)
			// if blake2b sum doesn't match for a part
			// then this disk is outdated and needs
			// healing.
			if blakeSum != partChecksum {
				errs[index] = errFileNotFound
				break
			}
			availableDisks[index] = onlineDisk
		}
	}

	return availableDisks, errs, nil
}