From 3a73c675a62dd5b79c7c059161a0975d5195e56c Mon Sep 17 00:00:00 2001 From: Andreas Auernhammer Date: Tue, 22 Aug 2017 16:53:35 -0700 Subject: [PATCH] restirct max size of http header and user metadata (#4634) (#4680) S3 only allows http headers with a size of 8 KB and user-defined metadata with a size of 2 KB. This change adds a new API error and returns this error to clients which sends to large http requests. Fixes #4634 --- cmd/api-errors.go | 7 +++++- cmd/generic-handlers.go | 46 ++++++++++++++++++++++++++++++++++++ cmd/generic-handlers_test.go | 35 +++++++++++++++++++++++++++ cmd/handler-utils.go | 18 ++++++++++---- cmd/routers.go | 2 ++ 5 files changed, 102 insertions(+), 6 deletions(-) diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 7bb60f87e..1b1000281 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -116,6 +116,7 @@ const ( ErrInvalidDuration ErrNotSupported ErrBucketAlreadyExists + ErrMetadataTooLarge // Add new error codes here. // Bucket notification related errors. @@ -630,7 +631,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{ Description: "Cannot respond to plain-text request from TLS-encrypted server", HTTPStatusCode: http.StatusBadRequest, }, - + ErrMetadataTooLarge: { + Code: "InvalidArgument", + Description: "Your metadata headers exceed the maximum allowed metadata size.", + HTTPStatusCode: http.StatusBadRequest, + }, // Add your error structure here. } diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 4306c8ed3..504740863 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -62,6 +62,52 @@ func (h requestSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques h.handler.ServeHTTP(w, r) } +const ( + // Maximum size for http headers - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html + maxHeaderSize = 8 * 1024 + // Maximum size for user-defined metadata - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html + maxUserDataSize = 2 * 1024 +) + +type requestHeaderSizeLimitHandler struct { + http.Handler +} + +func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler { + return requestHeaderSizeLimitHandler{h} +} + +// ServeHTTP restricts the size of the http header to 8 KB and the size +// of the user-defined metadata to 2 KB. +func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if isHTTPHeaderSizeTooLarge(r.Header) { + writeErrorResponse(w, ErrMetadataTooLarge, r.URL) + return + } + h.Handler.ServeHTTP(w, r) +} + +// isHTTPHeaderSizeTooLarge returns true if the provided +// header is larger than 8 KB or the user-defined metadata +// is larger than 2 KB. +func isHTTPHeaderSizeTooLarge(header http.Header) bool { + var size, usersize int + for key := range header { + length := len(key) + len(header.Get(key)) + size += length + for _, prefix := range userMetadataKeyPrefixes { + if strings.HasPrefix(key, prefix) { + usersize += length + break + } + } + if usersize > maxUserDataSize || size > maxHeaderSize { + return true + } + } + return false +} + // Reserved bucket. const ( minioReservedBucket = "minio" diff --git a/cmd/generic-handlers_test.go b/cmd/generic-handlers_test.go index f9872709e..f8b98649a 100644 --- a/cmd/generic-handlers_test.go +++ b/cmd/generic-handlers_test.go @@ -18,6 +18,7 @@ package cmd import ( "net/http" + "strconv" "testing" ) @@ -88,3 +89,37 @@ func TestGuessIsBrowser(t *testing.T) { t.Fatal("Test shouldn't report as browser for a non browser request.") } } + +var isHTTPHeaderSizeTooLargeTests = []struct { + header http.Header + shouldFail bool +}{ + {header: generateHeader(0, 0), shouldFail: false}, + {header: generateHeader(1024, 0), shouldFail: false}, + {header: generateHeader(2048, 0), shouldFail: false}, + {header: generateHeader(8*1024+1, 0), shouldFail: true}, + {header: generateHeader(0, 1024), shouldFail: false}, + {header: generateHeader(0, 2048), shouldFail: true}, + {header: generateHeader(0, 2048+1), shouldFail: true}, +} + +func generateHeader(size, usersize int) http.Header { + header := http.Header{} + for i := 0; i < size; i++ { + header.Add(strconv.Itoa(i), "") + } + userlength := 0 + for i := 0; userlength < usersize; i++ { + userlength += len(userMetadataKeyPrefixes[0] + strconv.Itoa(i)) + header.Add(userMetadataKeyPrefixes[0]+strconv.Itoa(i), "") + } + return header +} + +func TestIsHTTPHeaderSizeTooLarge(t *testing.T) { + for i, test := range isHTTPHeaderSizeTooLargeTests { + if res := isHTTPHeaderSizeTooLarge(test.header); res != test.shouldFail { + t.Errorf("Test %d: Expected %v got %v", i, res, test.shouldFail) + } + } +} diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go index e8b60f7e1..1c34c16a4 100644 --- a/cmd/handler-utils.go +++ b/cmd/handler-utils.go @@ -97,6 +97,14 @@ func path2BucketAndObject(path string) (bucket, object string) { return bucket, object } +// userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys. +// All values stored with a key starting with one of the following prefixes +// must be extracted from the header. +var userMetadataKeyPrefixes = []string{ + "X-Amz-Meta-", + "X-Minio-Meta-", +} + // extractMetadataFromHeader extracts metadata from HTTP header. func extractMetadataFromHeader(header http.Header) (map[string]string, error) { if header == nil { @@ -119,11 +127,11 @@ func extractMetadataFromHeader(header http.Header) (map[string]string, error) { if key != http.CanonicalHeaderKey(key) { return nil, traceError(errInvalidArgument) } - if strings.HasPrefix(key, "X-Amz-Meta-") { - metadata[key] = header.Get(key) - } - if strings.HasPrefix(key, "X-Minio-Meta-") { - metadata[key] = header.Get(key) + for _, prefix := range userMetadataKeyPrefixes { + if strings.HasPrefix(key, prefix) { + metadata[key] = header.Get(key) + break + } } } return metadata, nil diff --git a/cmd/routers.go b/cmd/routers.go index af0cd1654..b87ce3241 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -91,6 +91,8 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) { setHTTPStatsHandler, // Limits all requests size to a maximum fixed limit setRequestSizeLimitHandler, + // Limits all header sizes to a maximum fixed limit + setRequestHeaderSizeLimitHandler, // Adds 'crossdomain.xml' policy handler to serve legacy flash clients. setCrossDomainPolicy, // Redirect some pre-defined browser request paths to a static location prefix.