minio/pkg/madmin/info-commands.go

/*
 * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package madmin

import (
	"context"
	"encoding/json"
	"errors"
	"io/ioutil"
	"math"
	"net/http"
	"net/url"
	"strconv"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/minio/minio/pkg/cpu"
	"github.com/minio/minio/pkg/disk"
	"github.com/minio/minio/pkg/mem"
)

const (
	// DefaultNetPerfSize - default payload size used for network performance.
	DefaultNetPerfSize = 100 * humanize.MiByte
	// DefaultDrivePerfSize - default file size for testing drive performance
	DefaultDrivePerfSize = 100 * humanize.MiByte
)

// BackendType - represents different backend types.
type BackendType int

// Enum for different backend types.
const (
	Unknown BackendType = iota
	// Filesystem backend.
	FS
	// Multi disk Erasure (single, distributed) backend.
	Erasure

	// Add your own backend.
)

// DriveInfo - represents each drive info, describing
// status, uuid and endpoint.
type DriveInfo HealDriveInfo

// StorageInfo - represents total capacity of underlying storage.
type StorageInfo struct {
	Used []uint64 // Used total used per disk.

	Total []uint64 // Total disk space per disk.

	Available []uint64 // Total disk space available per disk.

	MountPaths []string // Disk mountpoints

	// Backend type.
	Backend struct {
		// Represents various backend types, currently on FS and Erasure.
		Type BackendType

		// Following fields are only meaningful if BackendType is Erasure.
		OnlineDisks      BackendDisks // Online disks during server startup.
		OfflineDisks     BackendDisks // Offline disks during server startup.
		StandardSCData   int          // Data disks for currently configured Standard storage class.
		StandardSCParity int          // Parity disks for currently configured Standard storage class.
		RRSCData         int          // Data disks for currently configured Reduced Redundancy storage class.
		RRSCParity       int          // Parity disks for currently configured Reduced Redundancy storage class.

		// List of all disk status, this is only meaningful if BackendType is Erasure.
		Sets [][]DriveInfo
	}
}

// BackendDisks - represents the map of endpoint-disks.
type BackendDisks map[string]int

// Sum - Return the sum of the disks in the endpoint-disk map.
func (d1 BackendDisks) Sum() (sum int) {
	for _, count := range d1 {
		sum += count
	}
	return sum
}

// Merge - Reduces two endpoint-disk maps.
func (d1 BackendDisks) Merge(d2 BackendDisks) BackendDisks {
	if len(d2) == 0 {
		d2 = make(BackendDisks)
	}
	for i1, v1 := range d1 {
		if v2, ok := d2[i1]; ok {
			d2[i1] = v2 + v1
			continue
		}
		d2[i1] = v1
	}
	return d2
}

// StorageInfo - Connect to a minio server and call Storage Info Management API
// to fetch server's information represented by StorageInfo structure
func (adm *AdminClient) StorageInfo(ctx context.Context) (StorageInfo, error) {
	resp, err := adm.executeMethod(ctx, http.MethodGet, requestData{relPath: adminAPIPrefix + "/storageinfo"})
	defer closeResponse(resp)
	if err != nil {
		return StorageInfo{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return StorageInfo{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var storageInfo StorageInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return StorageInfo{}, err
	}

	err = json.Unmarshal(respBytes, &storageInfo)
	if err != nil {
		return StorageInfo{}, err
	}

	return storageInfo, nil
}

type objectHistogramInterval struct {
	name       string
	start, end int64
}

// ObjectsHistogramIntervals contains the list of intervals
// of an histogram analysis of objects sizes.
var ObjectsHistogramIntervals = []objectHistogramInterval{
	{"LESS_THAN_1024_B", -1, 1024 - 1},
	{"BETWEEN_1024_B_AND_1_MB", 1024, 1024*1024 - 1},
	{"BETWEEN_1_MB_AND_10_MB", 1024 * 1024, 1024*1024*10 - 1},
	{"BETWEEN_10_MB_AND_64_MB", 1024 * 1024 * 10, 1024*1024*64 - 1},
	{"BETWEEN_64_MB_AND_128_MB", 1024 * 1024 * 64, 1024*1024*128 - 1},
	{"BETWEEN_128_MB_AND_512_MB", 1024 * 1024 * 128, 1024*1024*512 - 1},
	{"GREATER_THAN_512_MB", 1024 * 1024 * 512, math.MaxInt64},
}

// DataUsageInfo represents data usage of an Object API
type DataUsageInfo struct {
	// LastUpdate is the timestamp of when the data usage info was last updated.
	// This does not indicate a full scan.
	LastUpdate       time.Time `json:"lastUpdate"`
	ObjectsCount     uint64    `json:"objectsCount"`
	ObjectsTotalSize uint64    `json:"objectsTotalSize"`

	// ObjectsSizesHistogram contains information on objects across all buckets.
	// See ObjectsHistogramIntervals.
	ObjectsSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`

	BucketsCount uint64 `json:"bucketsCount"`

	// BucketsSizes is "bucket name" -> size.
	BucketsSizes map[string]uint64 `json:"bucketsSizes"`
}

// DataUsageInfo - returns data usage of the current object API
func (adm *AdminClient) DataUsageInfo(ctx context.Context) (DataUsageInfo, error) {
	resp, err := adm.executeMethod(ctx, http.MethodGet, requestData{relPath: adminAPIPrefix + "/datausageinfo"})
	defer closeResponse(resp)
	if err != nil {
		return DataUsageInfo{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return DataUsageInfo{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var dataUsageInfo DataUsageInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return DataUsageInfo{}, err
	}

	err = json.Unmarshal(respBytes, &dataUsageInfo)
	if err != nil {
		return DataUsageInfo{}, err
	}

	return dataUsageInfo, nil
}

// AccountAccess contains information about
type AccountAccess struct {
	AccountName string `json:"accountName"`
	Read        bool   `json:"read"`
	Write       bool   `json:"write"`
	Custom      bool   `json:"custom"`
}

// BucketAccountingUsage represents the accounting usage of a particular bucket
type BucketAccountingUsage struct {
	Size       uint64          `json:"size"`
	AccessList []AccountAccess `json:"accessList"`
}

// AccountingUsageInfo returns the accounting usage info, currently it returns
// the type of access of different accounts to the different buckets.
func (adm *AdminClient) AccountingUsageInfo(ctx context.Context) (map[string]BucketAccountingUsage, error) {
	resp, err := adm.executeMethod(ctx, http.MethodGet, requestData{relPath: adminAPIPrefix + "/accountingusageinfo"})
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return nil, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var accountingUsageInfo map[string]BucketAccountingUsage

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(respBytes, &accountingUsageInfo)
	if err != nil {
		return nil, err
	}

	return accountingUsageInfo, nil
}

// ServerDrivesPerfInfo holds informantion about address and write speed of
// all drives in a single server node
type ServerDrivesPerfInfo struct {
	Addr  string             `json:"addr"`
	Error string             `json:"error,omitempty"`
	Perf  []disk.Performance `json:"perf"`
	Size  int64              `json:"size,omitempty"`
}

// ServerDrivesPerfInfo - Returns drive's read and write performance information
func (adm *AdminClient) ServerDrivesPerfInfo(ctx context.Context, size int64) ([]ServerDrivesPerfInfo, error) {
	v := url.Values{}
	v.Set("perfType", string("drive"))

	v.Set("size", strconv.FormatInt(size, 10))

	resp, err := adm.executeMethod(ctx, http.MethodGet,
		requestData{
			relPath:     adminAPIPrefix + "/performance",
			queryValues: v,
		},
	)

	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return nil, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var info []ServerDrivesPerfInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(respBytes, &info)
	if err != nil {
		return nil, err
	}

	return info, nil
}

// ServerCPULoadInfo holds information about address and cpu load of
// a single server node
type ServerCPULoadInfo struct {
	Addr         string     `json:"addr"`
	Error        string     `json:"error,omitempty"`
	Load         []cpu.Load `json:"load"`
	HistoricLoad []cpu.Load `json:"historicLoad"`
}

// ServerCPULoadInfo - Returns cpu utilization information
func (adm *AdminClient) ServerCPULoadInfo(ctx context.Context) ([]ServerCPULoadInfo, error) {
	v := url.Values{}
	v.Set("perfType", string("cpu"))
	resp, err := adm.executeMethod(ctx,
		http.MethodGet, requestData{
			relPath:     adminAPIPrefix + "/performance",
			queryValues: v,
		},
	)

	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return nil, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var info []ServerCPULoadInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(respBytes, &info)
	if err != nil {
		return nil, err
	}

	return info, nil
}

// ServerMemUsageInfo holds information about address and memory utilization of
// a single server node
type ServerMemUsageInfo struct {
	Addr          string      `json:"addr"`
	Error         string      `json:"error,omitempty"`
	Usage         []mem.Usage `json:"usage"`
	HistoricUsage []mem.Usage `json:"historicUsage"`
}

// ServerMemUsageInfo - Returns mem utilization information
func (adm *AdminClient) ServerMemUsageInfo(ctx context.Context) ([]ServerMemUsageInfo, error) {
	v := url.Values{}
	v.Set("perfType", string("mem"))
	resp, err := adm.executeMethod(ctx,
		http.MethodGet,
		requestData{
			relPath:     adminAPIPrefix + "/performance",
			queryValues: v,
		},
	)

	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return nil, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var info []ServerMemUsageInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(respBytes, &info)
	if err != nil {
		return nil, err
	}

	return info, nil
}

// NetPerfInfo network performance information.
type NetPerfInfo struct {
	Addr           string `json:"addr"`
	ReadThroughput uint64 `json:"readThroughput"`
	Error          string `json:"error,omitempty"`
}

// NetPerfInfo - Returns network performance information of all cluster nodes.
func (adm *AdminClient) NetPerfInfo(ctx context.Context, size int) (map[string][]NetPerfInfo, error) {
	v := url.Values{}
	v.Set("perfType", "net")
	if size > 0 {
		v.Set("size", strconv.Itoa(size))
	}
	resp, err := adm.executeMethod(ctx,
		http.MethodGet,
		requestData{
			relPath:     adminAPIPrefix + "/performance",
			queryValues: v,
		},
	)

	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	// Check response http status code
	if resp.StatusCode == http.StatusMethodNotAllowed {
		return nil, errors.New("NetPerfInfo is meant for multi-node MinIO deployments")
	}

	if resp.StatusCode != http.StatusOK {
		return nil, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	info := map[string][]NetPerfInfo{}

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(respBytes, &info)
	if err != nil {
		return nil, err
	}

	return info, nil
}

// InfoMessage container to hold server admin related information.
type InfoMessage struct {
	Mode         string             `json:"mode,omitempty"`
	Domain       []string           `json:"domain,omitempty"`
	Region       string             `json:"region,omitempty"`
	SQSARN       []string           `json:"sqsARN,omitempty"`
	DeploymentID string             `json:"deploymentID,omitempty"`
	Buckets      Buckets            `json:"buckets,omitempty"`
	Objects      Objects            `json:"objects,omitempty"`
	Usage        Usage              `json:"usage,omitempty"`
	Services     Services           `json:"services,omitempty"`
	Backend      interface{}        `json:"backend,omitempty"`
	Servers      []ServerProperties `json:"servers,omitempty"`
}

// Services contains different services information
type Services struct {
	Vault         Vault                         `json:"vault,omitempty"`
	LDAP          LDAP                          `json:"ldap,omitempty"`
	Logger        []Logger                      `json:"logger,omitempty"`
	Audit         []Audit                       `json:"audit,omitempty"`
	Notifications []map[string][]TargetIDStatus `json:"notifications,omitempty"`
}

// Buckets contains the number of buckets
type Buckets struct {
	Count uint64 `json:"count,omitempty"`
}

// Objects contains the number of objects
type Objects struct {
	Count uint64 `json:"count,omitempty"`
}

// Usage contains the tottal size used
type Usage struct {
	Size uint64 `json:"size,omitempty"`
}

// Vault - Fetches the Vault status
type Vault struct {
	Status  string `json:"status,omitempty"`
	Encrypt string `json:"encryp,omitempty"`
	Decrypt string `json:"decrypt,omitempty"`
}

// LDAP contains ldap status
type LDAP struct {
	Status string `json:"status,omitempty"`
}

// Status of endpoint
type Status struct {
	Status string `json:"status,omitempty"`
}

// Audit contains audit logger status
type Audit map[string]Status

// Logger contains logger status
type Logger map[string]Status

// TargetIDStatus containsid and status
type TargetIDStatus map[string]Status

// backendType - indicates the type of backend storage
type backendType string

const (
	// FsType - Backend is FS Type
	FsType = backendType("FS")
	// ErasureType - Backend is Erasure type
	ErasureType = backendType("Erasure")
)

// FSBackend contains specific FS storage information
type FSBackend struct {
	Type backendType `json:"backendType,omitempty"`
}

// XLBackend contains specific erasure storage information
type XLBackend struct {
	Type         backendType `json:"backendType,omitempty"`
	OnlineDisks  int         `json:"onlineDisks,omitempty"`
	OfflineDisks int         `json:"offlineDisks,omitempty"`
	// Data disks for currently configured Standard storage class.
	StandardSCData int `json:"standardSCData,omitempty"`
	// Parity disks for currently configured Standard storage class.
	StandardSCParity int `json:"standardSCParity,omitempty"`
	// Data disks for currently configured Reduced Redundancy storage class.
	RRSCData int `json:"rrSCData,omitempty"`
	// Parity disks for currently configured Reduced Redundancy storage class.
	RRSCParity int `json:"rrSCParity,omitempty"`
}

// ServerProperties holds server information
type ServerProperties struct {
	State    string            `json:"state,omitempty"`
	Endpoint string            `json:"endpoint,omitempty"`
	Uptime   int64             `json:"uptime,omitempty"`
	Version  string            `json:"version,omitempty"`
	CommitID string            `json:"commitID,omitempty"`
	Network  map[string]string `json:"network,omitempty"`
	Disks    []Disk            `json:"disks,omitempty"`
}

// Disk holds Disk information
type Disk struct {
	DrivePath       string  `json:"path,omitempty"`
	State           string  `json:"state,omitempty"`
	UUID            string  `json:"uuid,omitempty"`
	Model           string  `json:"model,omitempty"`
	TotalSpace      uint64  `json:"totalspace,omitempty"`
	UsedSpace       uint64  `json:"usedspace,omitempty"`
	ReadThroughput  float64 `json:"readthroughput,omitempty"`
	WriteThroughPut float64 `json:"writethroughput,omitempty"`
	ReadLatency     float64 `json:"readlatency,omitempty"`
	WriteLatency    float64 `json:"writelatency,omitempty"`
	Utilization     float64 `json:"utilization,omitempty"`
}

// ServerInfo - Connect to a minio server and call Server Admin Info Management API
// to fetch server's information represented by infoMessage structure
func (adm *AdminClient) ServerInfo(ctx context.Context) (InfoMessage, error) {
	resp, err := adm.executeMethod(ctx,
		http.MethodGet,
		requestData{relPath: adminAPIPrefix + "/info"},
	)
	defer closeResponse(resp)
	if err != nil {
		return InfoMessage{}, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return InfoMessage{}, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var message InfoMessage

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return InfoMessage{}, err
	}

	err = json.Unmarshal(respBytes, &message)
	if err != nil {
		return InfoMessage{}, err
	}

	return message, nil
}