minio/cmd/gateway/s3/gateway-s3-metadata.go

/*
 * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"time"

	minio "github.com/minio/minio/cmd"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/hash"
	"github.com/valyala/fastjson"
)

var (
	errGWMetaNotFound      = errors.New("dare.meta file not found")
	errGWMetaInvalidFormat = errors.New("dare.meta format is invalid")
)

// A gwMetaV1 represents `gw.json` metadata header.
type gwMetaV1 struct {
	Version string         `json:"version"` // Version of the current `gw.json`.
	Format  string         `json:"format"`  // Format of the current `gw.json`.
	Stat    minio.StatInfo `json:"stat"`    // Stat of the current object `gw.json`.
	ETag    string         `json:"etag"`    // ETag of the current object

	// Metadata map for current object `gw.json`.
	Meta map[string]string `json:"meta,omitempty"`
	// Captures all the individual object `gw.json`.
	Parts []minio.ObjectPartInfo `json:"parts,omitempty"`
}

// Gateway metadata constants.
const (
	// Gateway meta version.
	gwMetaVersion = "1.0.0"

	// Gateway meta version.
	gwMetaVersion100 = "1.0.0"

	// Gateway meta format string.
	gwMetaFormat = "gw"

	// Add new constants here.
)

// newGWMetaV1 - initializes new gwMetaV1, adds version.
func newGWMetaV1() (gwMeta gwMetaV1) {
	gwMeta = gwMetaV1{}
	gwMeta.Version = gwMetaVersion
	gwMeta.Format = gwMetaFormat
	return gwMeta
}

// IsValid - tells if the format is sane by validating the version
// string, format fields.
func (m gwMetaV1) IsValid() bool {
	return ((m.Version == gwMetaVersion || m.Version == gwMetaVersion100) &&
		m.Format == gwMetaFormat)
}

// Converts metadata to object info.
func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
	filterKeys := append([]string{
		"ETag",
		"Content-Length",
		"Last-Modified",
		"Content-Type",
		"Expires",
	}, defaultFilterKeys...)
	objInfo := minio.ObjectInfo{
		IsDir:           false,
		Bucket:          bucket,
		Name:            object,
		Size:            m.Stat.Size,
		ModTime:         m.Stat.ModTime,
		ContentType:     m.Meta["content-type"],
		ContentEncoding: m.Meta["content-encoding"],
		ETag:            minio.CanonicalizeETag(m.ETag),
		UserDefined:     minio.CleanMetadataKeys(m.Meta, filterKeys...),
		Parts:           m.Parts,
	}

	if sc, ok := m.Meta["x-amz-storage-class"]; ok {
		objInfo.StorageClass = sc
	}
	var (
		t time.Time
		e error
	)
	if exp, ok := m.Meta["expires"]; ok {
		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
			objInfo.Expires = t.UTC()
		}
	}
	// Success.
	return objInfo
}

// ObjectToPartOffset - translate offset of an object to offset of its individual part.
func (m gwMetaV1) ObjectToPartOffset(ctx context.Context, offset int64) (partIndex int, partOffset int64, err error) {
	if offset == 0 {
		// Special case - if offset is 0, then partIndex and partOffset are always 0.
		return 0, 0, nil
	}
	partOffset = offset
	// Seek until object offset maps to a particular part offset.
	for i, part := range m.Parts {
		partIndex = i
		// Offset is smaller than size we have reached the proper part offset.
		if partOffset < part.Size {
			return partIndex, partOffset, nil
		}
		// Continue to towards the next part.
		partOffset -= part.Size
	}
	logger.LogIf(ctx, minio.InvalidRange{})
	// Offset beyond the size of the object return InvalidRange.
	return 0, 0, minio.InvalidRange{}
}

// parses gateway metadata stat info from metadata json
func parseGWStat(v *fastjson.Value) (si minio.StatInfo, err error) {
	// obtain stat info.
	st := v.GetObject("stat")
	var mb []byte
	mb, err = st.Get("modTime").StringBytes()
	if err != nil {
		return si, err
	}
	// fetching modTime.
	si.ModTime, err = time.Parse(time.RFC3339, string(mb))
	if err != nil {
		return si, err
	}
	// obtain Stat.Size .
	si.Size, err = st.Get("size").Int64()
	if err != nil {
		return si, err
	}
	return si, nil
}

// parses gateway metadata version from metadata json
func parseGWVersion(v *fastjson.Value) string {
	return string(v.GetStringBytes("version"))
}

// parses gateway ETag from metadata json
func parseGWETag(v *fastjson.Value) string {
	return string(v.GetStringBytes("etag"))
}

// parses gateway metadata format from metadata json
func parseGWFormat(v *fastjson.Value) string {
	return string(v.GetStringBytes("format"))
}

// parses gateway metadata json to get list of ObjectPartInfo
func parseGWParts(v *fastjson.Value) []minio.ObjectPartInfo {
	// Parse the GW Parts.
	partsResult := v.GetArray("parts")
	partInfo := make([]minio.ObjectPartInfo, len(partsResult))
	for i, p := range partsResult {
		partInfo[i] = minio.ObjectPartInfo{
			Number: p.GetInt("number"),
			Name:   string(p.GetStringBytes("name")),
			ETag:   string(p.GetStringBytes("etag")),
			Size:   p.GetInt64("size"),
		}
	}
	return partInfo
}

// parses gateway metadata json to get the metadata map
func parseGWMetaMap(v *fastjson.Value) map[string]string {
	metaMap := make(map[string]string)
	// Get gwMetaV1.Meta map.
	v.GetObject("meta").Visit(func(k []byte, kv *fastjson.Value) {
		metaMap[string(k)] = string(kv.GetStringBytes())
	})
	return metaMap
}

var gwParserPool fastjson.ParserPool

// Constructs GWMetaV1 using `fastjson` lib to retrieve each field.
func gwMetaUnmarshalJSON(ctx context.Context, gwMetaBuf []byte) (gwMeta gwMetaV1, err error) {
	parser := gwParserPool.Get()
	defer gwParserPool.Put(parser)

	var v *fastjson.Value
	v, err = parser.ParseBytes(gwMetaBuf)
	if err != nil {
		return gwMeta, err
	}

	// obtain version.
	gwMeta.Version = parseGWVersion(v)
	// obtain format.
	gwMeta.Format = parseGWFormat(v)
	// Parse gwMetaV1.Stat .
	stat, err := parseGWStat(v)
	if err != nil {
		logger.LogIf(ctx, err)
		return gwMeta, err
	}
	gwMeta.ETag = parseGWETag(v)
	gwMeta.Stat = stat

	// Parse the GW Parts.
	gwMeta.Parts = parseGWParts(v)
	// parse gwMetaV1.
	gwMeta.Meta = parseGWMetaMap(v)

	return gwMeta, nil
}

// readGWMeta reads `dare.meta` and returns back GW metadata structure.
func readGWMetadata(ctx context.Context, buf bytes.Buffer) (gwMeta gwMetaV1, err error) {
	if buf.Len() == 0 {
		return gwMetaV1{}, errGWMetaNotFound
	}
	gwMeta, err = gwMetaUnmarshalJSON(ctx, buf.Bytes())
	if err != nil {
		return gwMetaV1{}, err
	}
	if !gwMeta.IsValid() {
		return gwMetaV1{}, errGWMetaInvalidFormat
	}
	// Return structured `dare.meta`.
	return gwMeta, nil
}

// getGWMetadata - unmarshals dare.meta into a *minio.PutObjReader
func getGWMetadata(ctx context.Context, bucket, prefix string, gwMeta gwMetaV1) (*minio.PutObjReader, error) {
	// Marshal json.
	metadataBytes, err := json.Marshal(&gwMeta)
	if err != nil {
		logger.LogIf(ctx, err)
		return nil, err
	}
	hashReader, err := hash.NewReader(bytes.NewReader(metadataBytes), int64(len(metadataBytes)), "", "", int64(len(metadataBytes)), false)
	if err != nil {
		return nil, err
	}
	return minio.NewPutObjReader(hashReader, nil, nil), nil
}