You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
minio/cmd/object-api-common.go

307 lines
7.8 KiB

/*
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"net"
"net/url"
"runtime"
"sync"
humanize "github.com/dustin/go-humanize"
)
const (
// Block size used for all internal operations version 1.
blockSizeV1 = 10 * humanize.MiByte
// Staging buffer read size for all internal operations version 1.
readSizeV1 = 1 * humanize.MiByte
// Buckets meta prefix.
bucketMetaPrefix = "buckets"
)
// Global object layer mutex, used for safely updating object layer.
var globalObjLayerMutex *sync.RWMutex
// Global object layer, only accessed by newObjectLayerFn().
var globalObjectAPI ObjectLayer
func init() {
// Initialize this once per server initialization.
globalObjLayerMutex = &sync.RWMutex{}
}
// Check if the disk is remote.
func isRemoteDisk(disk StorageAPI) bool {
_, ok := disk.(*networkStorage)
return ok
}
// Checks if the object is a directory, this logic uses
// if size == 0 and object ends with slashSeparator then
// returns true.
func isObjectDir(object string, size int64) bool {
return hasSuffix(object, slashSeparator) && size == 0
}
// Converts just bucket, object metadata into ObjectInfo datatype.
func dirObjectInfo(bucket, object string, size int64, metadata map[string]string) ObjectInfo {
// This is a special case with size as '0' and object ends with
// a slash separator, we treat it like a valid operation and
// return success.
md5Sum := metadata["md5Sum"]
delete(metadata, "md5Sum")
return ObjectInfo{
Bucket: bucket,
Name: object,
ModTime: UTCNow(),
ContentType: "application/octet-stream",
IsDir: true,
Size: size,
MD5Sum: md5Sum,
UserDefined: metadata,
}
}
// House keeping code for FS/XL and distributed Minio setup.
func houseKeeping(storageDisks []StorageAPI) error {
var wg = &sync.WaitGroup{}
// Initialize errs to collect errors inside go-routine.
var errs = make([]error, len(storageDisks))
// Initialize all disks in parallel.
for index, disk := range storageDisks {
if disk == nil {
continue
}
// Skip remote disks.
if isRemoteDisk(disk) {
continue
}
wg.Add(1)
go func(index int, disk StorageAPI) {
// Indicate this wait group is done.
defer wg.Done()
// Cleanup all temp entries upon start.
err := cleanupDir(disk, minioMetaTmpBucket, "")
if err != nil {
if !isErrIgnored(errorCause(err), errDiskNotFound, errVolumeNotFound, errFileNotFound) {
errs[index] = err
}
}
}(index, disk)
}
// Wait for all cleanup to finish.
wg.Wait()
// Return upon first error.
for _, err := range errs {
if err == nil {
continue
}
return toObjectErr(err, minioMetaTmpBucket, "*")
}
// Return success here.
return nil
}
// Check if a network path is local to this node.
func isLocalStorage(ep *url.URL) bool {
if ep.Host == "" {
return true
}
if globalMinioHost != "" && globalMinioPort != "" {
// if --address host:port was specified for distXL we short
// circuit only the endPoint that matches host:port
return net.JoinHostPort(globalMinioHost, globalMinioPort) == ep.Host
}
// Split host to extract host information.
host, _, err := net.SplitHostPort(ep.Host)
if err != nil {
errorIf(err, "Cannot split host port")
return false
}
// Resolve host to address to check if the IP is loopback.
// If address resolution fails, assume it's a non-local host.
addrs, err := net.LookupHost(host)
if err != nil {
errorIf(err, "Failed to lookup host")
return false
}
for _, addr := range addrs {
if ip := net.ParseIP(addr); ip.IsLoopback() {
return true
}
}
iaddrs, err := net.InterfaceAddrs()
if err != nil {
errorIf(err, "Unable to list interface addresses")
return false
}
for _, addr := range addrs {
for _, iaddr := range iaddrs {
ip, _, err := net.ParseCIDR(iaddr.String())
if err != nil {
errorIf(err, "Unable to parse CIDR")
return false
}
if ip.String() == addr {
return true
}
}
}
return false
}
// Fetch the path component from *url.URL*.
func getPath(ep *url.URL) string {
if ep == nil {
return ""
}
var diskPath string
// For windows ep.Path is usually empty
if runtime.GOOS == globalWindowsOSName {
switch ep.Scheme {
case "":
// Eg. "minio server .\export"
diskPath = ep.Path
case httpScheme, httpsScheme:
// For full URLs windows drive is part of URL path.
// Eg: http://ip:port/C:\mydrive
// For windows trim off the preceding "/".
diskPath = ep.Path[1:]
default:
// For the rest url splits drive letter into
// Scheme contruct the disk path back.
diskPath = ep.Scheme + ":" + ep.Opaque
}
} else {
// For other operating systems ep.Path is non empty.
diskPath = ep.Path
}
return diskPath
}
// Depending on the disk type network or local, initialize storage API.
func newStorageAPI(ep *url.URL) (storage StorageAPI, err error) {
if isLocalStorage(ep) {
return newPosix(getPath(ep))
}
return newStorageRPC(ep)
}
var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)
// Initializes meta volume on all input storage disks.
func initMetaVolume(storageDisks []StorageAPI) error {
// This happens for the first time, but keep this here since this
// is the only place where it can be made expensive optimizing all
// other calls. Create minio meta volume, if it doesn't exist yet.
var wg = &sync.WaitGroup{}
// Initialize errs to collect errors inside go-routine.
var errs = make([]error, len(storageDisks))
// Initialize all disks in parallel.
for index, disk := range storageDisks {
if disk == nil {
// Ignore create meta volume on disks which are not found.
continue
}
wg.Add(1)
go func(index int, disk StorageAPI) {
// Indicate this wait group is done.
defer wg.Done()
// Attempt to create `.minio.sys`.
err := disk.MakeVol(minioMetaBucket)
if err != nil {
if !isErrIgnored(err, initMetaVolIgnoredErrs...) {
errs[index] = err
return
}
}
err = disk.MakeVol(minioMetaTmpBucket)
if err != nil {
if !isErrIgnored(err, initMetaVolIgnoredErrs...) {
errs[index] = err
return
}
}
err = disk.MakeVol(minioMetaMultipartBucket)
if err != nil {
if !isErrIgnored(err, initMetaVolIgnoredErrs...) {
errs[index] = err
return
}
}
}(index, disk)
}
// Wait for all cleanup to finish.
wg.Wait()
// Return upon first error.
for _, err := range errs {
if err == nil {
continue
}
return toObjectErr(err, minioMetaBucket)
}
// Return success here.
return nil
}
// Cleanup a directory recursively.
func cleanupDir(storage StorageAPI, volume, dirPath string) error {
var delFunc func(string) error
// Function to delete entries recursively.
delFunc = func(entryPath string) error {
if !hasSuffix(entryPath, slashSeparator) {
// Delete the file entry.
return traceError(storage.DeleteFile(volume, entryPath))
}
// If it's a directory, list and call delFunc() for each entry.
entries, err := storage.ListDir(volume, entryPath)
// If entryPath prefix never existed, safe to ignore.
if err == errFileNotFound {
return nil
} else if err != nil { // For any other errors fail.
return traceError(err)
} // else on success..
// Recurse and delete all other entries.
for _, entry := range entries {
if err = delFunc(pathJoin(entryPath, entry)); err != nil {
return err
}
}
return nil
}
err := delFunc(retainSlash(pathJoin(dirPath)))
return err
}