restirct max size of http header and user metadata (#4634) (#4680)

S3 only allows http headers with a size of 8 KB and user-defined metadata
with a size of 2 KB. This change adds a new API error and returns this
error to clients which sends to large http requests.

Fixes #4634
master
Andreas Auernhammer 7 years ago committed by Dee Koder
parent b694c1a4d7
commit 3a73c675a6
  1. 7
      cmd/api-errors.go
  2. 46
      cmd/generic-handlers.go
  3. 35
      cmd/generic-handlers_test.go
  4. 18
      cmd/handler-utils.go
  5. 2
      cmd/routers.go

@ -116,6 +116,7 @@ const (
ErrInvalidDuration
ErrNotSupported
ErrBucketAlreadyExists
ErrMetadataTooLarge
// Add new error codes here.
// Bucket notification related errors.
@ -630,7 +631,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
Description: "Cannot respond to plain-text request from TLS-encrypted server",
HTTPStatusCode: http.StatusBadRequest,
},
ErrMetadataTooLarge: {
Code: "InvalidArgument",
Description: "Your metadata headers exceed the maximum allowed metadata size.",
HTTPStatusCode: http.StatusBadRequest,
},
// Add your error structure here.
}

@ -62,6 +62,52 @@ func (h requestSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
h.handler.ServeHTTP(w, r)
}
const (
// Maximum size for http headers - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
maxHeaderSize = 8 * 1024
// Maximum size for user-defined metadata - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
maxUserDataSize = 2 * 1024
)
type requestHeaderSizeLimitHandler struct {
http.Handler
}
func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
return requestHeaderSizeLimitHandler{h}
}
// ServeHTTP restricts the size of the http header to 8 KB and the size
// of the user-defined metadata to 2 KB.
func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if isHTTPHeaderSizeTooLarge(r.Header) {
writeErrorResponse(w, ErrMetadataTooLarge, r.URL)
return
}
h.Handler.ServeHTTP(w, r)
}
// isHTTPHeaderSizeTooLarge returns true if the provided
// header is larger than 8 KB or the user-defined metadata
// is larger than 2 KB.
func isHTTPHeaderSizeTooLarge(header http.Header) bool {
var size, usersize int
for key := range header {
length := len(key) + len(header.Get(key))
size += length
for _, prefix := range userMetadataKeyPrefixes {
if strings.HasPrefix(key, prefix) {
usersize += length
break
}
}
if usersize > maxUserDataSize || size > maxHeaderSize {
return true
}
}
return false
}
// Reserved bucket.
const (
minioReservedBucket = "minio"

@ -18,6 +18,7 @@ package cmd
import (
"net/http"
"strconv"
"testing"
)
@ -88,3 +89,37 @@ func TestGuessIsBrowser(t *testing.T) {
t.Fatal("Test shouldn't report as browser for a non browser request.")
}
}
var isHTTPHeaderSizeTooLargeTests = []struct {
header http.Header
shouldFail bool
}{
{header: generateHeader(0, 0), shouldFail: false},
{header: generateHeader(1024, 0), shouldFail: false},
{header: generateHeader(2048, 0), shouldFail: false},
{header: generateHeader(8*1024+1, 0), shouldFail: true},
{header: generateHeader(0, 1024), shouldFail: false},
{header: generateHeader(0, 2048), shouldFail: true},
{header: generateHeader(0, 2048+1), shouldFail: true},
}
func generateHeader(size, usersize int) http.Header {
header := http.Header{}
for i := 0; i < size; i++ {
header.Add(strconv.Itoa(i), "")
}
userlength := 0
for i := 0; userlength < usersize; i++ {
userlength += len(userMetadataKeyPrefixes[0] + strconv.Itoa(i))
header.Add(userMetadataKeyPrefixes[0]+strconv.Itoa(i), "")
}
return header
}
func TestIsHTTPHeaderSizeTooLarge(t *testing.T) {
for i, test := range isHTTPHeaderSizeTooLargeTests {
if res := isHTTPHeaderSizeTooLarge(test.header); res != test.shouldFail {
t.Errorf("Test %d: Expected %v got %v", i, res, test.shouldFail)
}
}
}

@ -97,6 +97,14 @@ func path2BucketAndObject(path string) (bucket, object string) {
return bucket, object
}
// userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys.
// All values stored with a key starting with one of the following prefixes
// must be extracted from the header.
var userMetadataKeyPrefixes = []string{
"X-Amz-Meta-",
"X-Minio-Meta-",
}
// extractMetadataFromHeader extracts metadata from HTTP header.
func extractMetadataFromHeader(header http.Header) (map[string]string, error) {
if header == nil {
@ -119,11 +127,11 @@ func extractMetadataFromHeader(header http.Header) (map[string]string, error) {
if key != http.CanonicalHeaderKey(key) {
return nil, traceError(errInvalidArgument)
}
if strings.HasPrefix(key, "X-Amz-Meta-") {
metadata[key] = header.Get(key)
}
if strings.HasPrefix(key, "X-Minio-Meta-") {
metadata[key] = header.Get(key)
for _, prefix := range userMetadataKeyPrefixes {
if strings.HasPrefix(key, prefix) {
metadata[key] = header.Get(key)
break
}
}
}
return metadata, nil

@ -91,6 +91,8 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
setHTTPStatsHandler,
// Limits all requests size to a maximum fixed limit
setRequestSizeLimitHandler,
// Limits all header sizes to a maximum fixed limit
setRequestHeaderSizeLimitHandler,
// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
setCrossDomainPolicy,
// Redirect some pre-defined browser request paths to a static location prefix.

Loading…
Cancel
Save