fix: use getObjectNInfo to avoid bytes.Buffer usage (#11428)

few places were still using legacy call GetObject()
which was mainly designed for client response writer,
use GetObjectNInfo() for internal calls instead.
master
Harshavardhana 4 years ago committed by GitHub
parent af9cb5f5f2
commit 8bb580abfc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 17
      cmd/config-common.go
  2. 12
      cmd/data-crawler.go
  3. 9
      cmd/data-usage.go
  4. 20
      cmd/metacache-bucket.go

@ -20,6 +20,8 @@ import (
"bytes"
"context"
"errors"
"io/ioutil"
"net/http"
"github.com/minio/minio/pkg/hash"
)
@ -27,9 +29,9 @@ import (
var errConfigNotFound = errors.New("config file not found")
func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
var buffer bytes.Buffer
// Read entire content by setting size to -1
if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
r, err := objAPI.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
if err != nil {
// Treat object not found as config not found.
if isErrObjectNotFound(err) {
return nil, errConfigNotFound
@ -37,13 +39,16 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
return nil, err
}
defer r.Close()
// Return config not found on empty content.
if buffer.Len() == 0 {
buf, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
if len(buf) == 0 {
return nil, errConfigNotFound
}
return buffer.Bytes(), nil
return buf, nil
}
type objectDeleter interface {

@ -23,6 +23,7 @@ import (
"errors"
"math"
"math/rand"
"net/http"
"os"
"path"
"strings"
@ -84,16 +85,19 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
// Load current bloom cycle
nextBloomCycle := intDataUpdateTracker.current() + 1
var buf bytes.Buffer
err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageBloomName, 0, -1, &buf, "", ObjectOptions{})
br, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageBloomName, nil, http.Header{}, readLock, ObjectOptions{})
if err != nil {
if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
logger.LogIf(ctx, err)
}
} else {
if buf.Len() == 8 {
nextBloomCycle = binary.LittleEndian.Uint64(buf.Bytes())
if br.ObjInfo.Size == 8 {
if err = binary.Read(br, binary.LittleEndian, &nextBloomCycle); err != nil {
logger.LogIf(ctx, err)
}
}
br.Close()
}
crawlTimer := time.NewTimer(dataCrawlStartDelay)

@ -20,6 +20,7 @@ import (
"bytes"
"context"
"encoding/json"
"net/http"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio/cmd/logger"
@ -59,20 +60,18 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
}
func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
var dataUsageInfoJSON bytes.Buffer
err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
r, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageObjName, nil, http.Header{}, readLock, ObjectOptions{})
if err != nil {
if isErrObjectNotFound(err) || isErrBucketNotFound(err) {
return DataUsageInfo{}, nil
}
return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
}
defer r.Close()
var dataUsageInfo DataUsageInfo
var json = jsoniter.ConfigCompatibleWithStandardLibrary
err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
if err != nil {
if err = json.NewDecoder(r).Decode(&dataUsageInfo); err != nil {
return DataUsageInfo{}, err
}

@ -21,7 +21,7 @@ import (
"context"
"errors"
"fmt"
"io"
"net/http"
"path"
"runtime/debug"
"strings"
@ -95,25 +95,19 @@ func loadBucketMetaCache(ctx context.Context, bucket string) (*bucketMetacache,
logger.LogIf(ctx, fmt.Errorf("loadBucketMetaCache: object layer not ready. bucket: %q", bucket))
}
}
var meta bucketMetacache
var decErr error
var wg sync.WaitGroup
wg.Add(1)
r, w := io.Pipe()
go func() {
defer wg.Done()
// Use global context for this.
r, err := objAPI.GetObjectNInfo(GlobalContext, minioMetaBucket, pathJoin("buckets", bucket, ".metacache", "index.s2"), nil, http.Header{}, readLock, ObjectOptions{})
if err == nil {
dec := s2DecPool.Get().(*s2.Reader)
dec.Reset(r)
decErr = meta.DecodeMsg(msgp.NewReader(dec))
dec.Reset(nil)
r.Close()
s2DecPool.Put(dec)
r.CloseWithError(decErr)
}()
// Use global context for this.
err := objAPI.GetObject(GlobalContext, minioMetaBucket, pathJoin("buckets", bucket, ".metacache", "index.s2"), 0, -1, w, "", ObjectOptions{})
logger.LogIf(ctx, w.CloseWithError(err))
wg.Wait()
}
if err != nil {
switch err.(type) {
case ObjectNotFound:

Loading…
Cancel
Save