From 7fea9cb550521d555ea19386fc2e5b6aa40b9c70 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 8 Oct 2015 19:56:41 -0700 Subject: [PATCH] Add rpc signature handler --- common.go | 139 ++++++++++ controller-main.go | 2 +- controller-router.go | 13 +- controller-rpc-signature-handler.go | 243 ++++++++++++++++++ controller-rpc-signature.go | 123 +++++++++ controller-rpc.go | 44 +--- controller_rpc_test.go | 48 +++- ...generic-handlers.go => generic-handlers.go | 27 +- .../donut/donut-metadata.md | 0 pkg/signature/signature-v4.go | 14 +- rpc-request.go | 109 +++++++- server-api-signature-handler.go | 5 +- server-main.go | 2 +- server-router.go | 22 +- server_signV4_donut_cache_test.go | 2 +- server_signV4_donut_test.go | 118 +-------- server-api-typed-errors.go => typed-errors.go | 3 + 17 files changed, 720 insertions(+), 194 deletions(-) create mode 100644 common.go create mode 100644 controller-rpc-signature-handler.go create mode 100644 controller-rpc-signature.go rename server-api-generic-handlers.go => generic-handlers.go (85%) rename donut-metadata.md => pkg/donut/donut-metadata.md (100%) rename server-api-typed-errors.go => typed-errors.go (95%) diff --git a/common.go b/common.go new file mode 100644 index 000000000..f8b81f1dc --- /dev/null +++ b/common.go @@ -0,0 +1,139 @@ +/* + * Minio Cloud Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "crypto/hmac" + "encoding/hex" + "io" + "regexp" + "strings" + "unicode/utf8" + + "github.com/minio/minio/pkg/crypto/sha256" +) + +/// +/// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 +/// +/// User-Agent: +/// +/// This is ignored from signing because signing this causes problems with generating pre-signed URLs +/// (that are executed by other agents) or when customers pass requests through proxies, which may +/// modify the user-agent. +/// +/// Content-Length: +/// +/// This is ignored from signing because generating a pre-signed URL should not provide a content-length +/// constraint, specifically when vending a S3 pre-signed PUT URL. The corollary to this is that when +/// sending regular requests (non-pre-signed), the signature contains a checksum of the body, which +/// implicitly validates the payload length (since changing the number of bytes would change the checksum) +/// and therefore this header is not valuable in the signature. +/// +/// Content-Type: +/// +/// Signing this header causes quite a number of problems in browser environments, where browsers +/// like to modify and normalize the content-type header in different ways. There is more information +/// on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding this field simplifies logic +/// and reduces the possibility of future bugs +/// +/// Authorization: +/// +/// Is skipped for obvious reasons +/// +var ignoredHeaders = map[string]bool{ + "Authorization": true, + "Content-Type": true, + "Content-Length": true, + "User-Agent": true, +} + +// sum256Reader calculate sha256 sum for an input read seeker +func sum256Reader(reader io.ReadSeeker) ([]byte, error) { + h := sha256.New() + var err error + + start, _ := reader.Seek(0, 1) + defer reader.Seek(start, 0) + + for err == nil { + length := 0 + byteBuffer := make([]byte, 1024*1024) + length, err = reader.Read(byteBuffer) + byteBuffer = byteBuffer[0:length] + h.Write(byteBuffer) + } + + if err != io.EOF { + return nil, err + } + + return h.Sum(nil), nil +} + +// sum256 calculate sha256 sum for an input byte array +func sum256(data []byte) []byte { + hash := sha256.New() + hash.Write(data) + return hash.Sum(nil) +} + +// sumHMAC calculate hmac between two input byte array +func sumHMAC(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} + +// getURLEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences +// +// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 +// non english characters cannot be parsed due to the nature in which url.Encode() is written +// +// This function on the other hand is a direct replacement for url.Encode() technique to support +// pretty much every UTF-8 character. +func getURLEncodedName(name string) string { + // if object matches reserved string, no need to encode them + reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") + if reservedNames.MatchString(name) { + return name + } + var encodedName string + for _, s := range name { + if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) + encodedName = encodedName + string(s) + continue + } + switch s { + case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) + encodedName = encodedName + string(s) + continue + default: + len := utf8.RuneLen(s) + if len < 0 { + return name + } + u := make([]byte, len) + utf8.EncodeRune(u, s) + for _, r := range u { + hex := hex.EncodeToString([]byte{r}) + encodedName = encodedName + "%" + strings.ToUpper(hex) + } + } + } + return encodedName +} diff --git a/controller-main.go b/controller-main.go index c3ca3a8bc..ba64a9a91 100644 --- a/controller-main.go +++ b/controller-main.go @@ -100,7 +100,7 @@ func configureControllerRPC(conf minioConfig, rpcHandler http.Handler) (*http.Se // startController starts a minio controller func startController(conf minioConfig) *probe.Error { - rpcServer, err := configureControllerRPC(conf, getControllerRPCHandler()) + rpcServer, err := configureControllerRPC(conf, getControllerRPCHandler(conf.Anonymous)) if err != nil { return err.Trace() } diff --git a/controller-router.go b/controller-router.go index 87ad8bb65..019f97c23 100644 --- a/controller-router.go +++ b/controller-router.go @@ -25,7 +25,14 @@ import ( ) // getControllerRPCHandler rpc handler for controller -func getControllerRPCHandler() http.Handler { +func getControllerRPCHandler(anonymous bool) http.Handler { + var mwHandlers = []MiddlewareHandler{ + TimeValidityHandler, + } + if !anonymous { + mwHandlers = append(mwHandlers, RPCSignatureHandler) + } + s := jsonrpc.NewServer() codec := json.NewCodec() s.RegisterCodec(codec, "application/json") @@ -35,5 +42,7 @@ func getControllerRPCHandler() http.Handler { // Add new RPC services here mux.Handle("/rpc", s) mux.Handle("/{file:.*}", http.FileServer(assetFS())) - return mux + + rpcHandler := registerCustomMiddleware(mux, mwHandlers...) + return rpcHandler } diff --git a/controller-rpc-signature-handler.go b/controller-rpc-signature-handler.go new file mode 100644 index 000000000..6654dcc72 --- /dev/null +++ b/controller-rpc-signature-handler.go @@ -0,0 +1,243 @@ +/* + * Minio Cloud Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "encoding/hex" + "io" + "net/http" + "sort" + "strings" + "time" + + "github.com/minio/minio/pkg/crypto/sha256" + "github.com/minio/minio/pkg/probe" +) + +type rpcSignatureHandler struct { + handler http.Handler +} + +// RPCSignatureHandler to validate authorization header for the incoming request. +func RPCSignatureHandler(h http.Handler) http.Handler { + return signatureHandler{h} +} + +type rpcSignature struct { + AccessKeyID string + SecretAccessKey string + Signature string + SignedHeaders []string + Request *http.Request +} + +// getCanonicalHeaders generate a list of request headers with their values +func (r *rpcSignature) getCanonicalHeaders(signedHeaders map[string][]string) string { + var headers []string + vals := make(map[string][]string) + for k, vv := range signedHeaders { + headers = append(headers, strings.ToLower(k)) + vals[strings.ToLower(k)] = vv + } + headers = append(headers, "host") + sort.Strings(headers) + + var buf bytes.Buffer + for _, k := range headers { + buf.WriteString(k) + buf.WriteByte(':') + switch { + case k == "host": + buf.WriteString(r.Request.Host) + fallthrough + default: + for idx, v := range vals[k] { + if idx > 0 { + buf.WriteByte(',') + } + buf.WriteString(v) + } + buf.WriteByte('\n') + } + } + return buf.String() +} + +// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names +func (r *rpcSignature) getSignedHeaders(signedHeaders map[string][]string) string { + var headers []string + for k := range signedHeaders { + headers = append(headers, strings.ToLower(k)) + } + headers = append(headers, "host") + sort.Strings(headers) + return strings.Join(headers, ";") +} + +// extractSignedHeaders extract signed headers from Authorization header +func (r rpcSignature) extractSignedHeaders() map[string][]string { + extractedSignedHeadersMap := make(map[string][]string) + for _, header := range r.SignedHeaders { + val, ok := r.Request.Header[http.CanonicalHeaderKey(header)] + if !ok { + // if not found continue, we will fail later + continue + } + extractedSignedHeadersMap[header] = val + } + return extractedSignedHeadersMap +} + +// getCanonicalRequest generate a canonical request of style +// +// canonicalRequest = +// \n +// \n +// \n +// \n +// \n +// +// +func (r *rpcSignature) getCanonicalRequest() string { + payload := r.Request.Header.Get(http.CanonicalHeaderKey("x-amz-content-sha256")) + r.Request.URL.RawQuery = strings.Replace(r.Request.URL.Query().Encode(), "+", "%20", -1) + encodedPath := getURLEncodedName(r.Request.URL.Path) + // convert any space strings back to "+" + encodedPath = strings.Replace(encodedPath, "+", "%20", -1) + canonicalRequest := strings.Join([]string{ + r.Request.Method, + encodedPath, + r.Request.URL.RawQuery, + r.getCanonicalHeaders(r.extractSignedHeaders()), + r.getSignedHeaders(r.extractSignedHeaders()), + payload, + }, "\n") + return canonicalRequest +} + +// getScope generate a string of a specific date, an AWS region, and a service +func (r rpcSignature) getScope(t time.Time) string { + scope := strings.Join([]string{ + t.Format(yyyymmdd), + "milkyway", + "rpc", + "rpc_request", + }, "/") + return scope +} + +// getStringToSign a string based on selected query values +func (r rpcSignature) getStringToSign(canonicalRequest string, t time.Time) string { + stringToSign := authHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n" + stringToSign = stringToSign + r.getScope(t) + "\n" + stringToSign = stringToSign + hex.EncodeToString(sha256.Sum256([]byte(canonicalRequest))) + return stringToSign +} + +// getSigningKey hmac seed to calculate final signature +func (r rpcSignature) getSigningKey(t time.Time) []byte { + secret := r.SecretAccessKey + date := sumHMAC([]byte("MINIO"+secret), []byte(t.Format(yyyymmdd))) + region := sumHMAC(date, []byte("milkyway")) + service := sumHMAC(region, []byte("rpc")) + signingKey := sumHMAC(service, []byte("rpc_request")) + return signingKey +} + +// getSignature final signature in hexadecimal form +func (r rpcSignature) getSignature(signingKey []byte, stringToSign string) string { + return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) +} + +func (r rpcSignature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) { + // set new calulated payload + r.Request.Header.Set("X-Minio-Content-Sha256", hashedPayload) + + // Add date if not present throw error + var date string + if date = r.Request.Header.Get(http.CanonicalHeaderKey("x-minio-date")); date == "" { + if date = r.Request.Header.Get("Date"); date == "" { + return false, probe.NewError(errMissingDateHeader) + } + } + t, err := time.Parse(iso8601Format, date) + if err != nil { + return false, probe.NewError(err) + } + canonicalRequest := r.getCanonicalRequest() + stringToSign := r.getStringToSign(canonicalRequest, t) + signingKey := r.getSigningKey(t) + newSignature := r.getSignature(signingKey, stringToSign) + + if newSignature != r.Signature { + return false, nil + } + return true, nil +} + +func isRequestSignatureRPC(req *http.Request) bool { + if _, ok := req.Header["Authorization"]; ok { + return ok + } + return false +} + +func (s rpcSignatureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var signature *rpcSignature + if isRequestSignatureRPC(r) { + // Init signature V4 verification + var err *probe.Error + signature, err = initSignatureRPC(r) + if err != nil { + switch err.ToGoError() { + case errInvalidRegion: + errorIf(err.Trace(), "Unknown region in authorization header.", nil) + writeErrorResponse(w, r, AuthorizationHeaderMalformed, r.URL.Path) + return + case errAccessKeyIDInvalid: + errorIf(err.Trace(), "Invalid access key id.", nil) + writeErrorResponse(w, r, InvalidAccessKeyID, r.URL.Path) + return + default: + errorIf(err.Trace(), "Initializing signature v4 failed.", nil) + writeErrorResponse(w, r, InternalError, r.URL.Path) + return + } + } + buffer := new(bytes.Buffer) + if _, err := io.Copy(buffer, r.Body); err != nil { + errorIf(probe.NewError(err), "Unable to read payload from request body.", nil) + writeErrorResponse(w, r, InternalError, r.URL.Path) + return + } + value := sha256.Sum256(buffer.Bytes()) + ok, err := signature.DoesSignatureMatch(hex.EncodeToString(value[:])) + if err != nil { + errorIf(err.Trace(), "Unable to verify signature.", nil) + writeErrorResponse(w, r, InternalError, r.URL.Path) + return + } + if !ok { + writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path) + return + } + s.handler.ServeHTTP(w, r) + return + } + writeErrorResponse(w, r, AccessDenied, r.URL.Path) +} diff --git a/controller-rpc-signature.go b/controller-rpc-signature.go new file mode 100644 index 000000000..c0453c1f9 --- /dev/null +++ b/controller-rpc-signature.go @@ -0,0 +1,123 @@ +/* + * Minio Cloud Storage, (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "net/http" + "strings" + + "github.com/minio/minio/pkg/probe" +) + +const ( + rpcAuthHeaderPrefix = "MINIORPC" +) + +// getRPCCredentialsFromAuth parse credentials tag from authorization value +// Authorization: +// Authorization: MINIORPC Credential=admin/20130524/milkyway/rpc/rpc_request, +// SignedHeaders=host;x-minio-date, Signature=fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024 +func getRPCCredentialsFromAuth(authValue string) ([]string, *probe.Error) { + if authValue == "" { + return nil, probe.NewError(errMissingAuthHeaderValue) + } + authFields := strings.Split(strings.TrimSpace(authValue), ",") + if len(authFields) != 3 { + return nil, probe.NewError(errInvalidAuthHeaderValue) + } + authPrefixFields := strings.Fields(authFields[0]) + if len(authPrefixFields) != 2 { + return nil, probe.NewError(errMissingFieldsAuthHeader) + } + if authPrefixFields[0] != rpcAuthHeaderPrefix { + return nil, probe.NewError(errInvalidAuthHeaderPrefix) + } + credentials := strings.Split(strings.TrimSpace(authPrefixFields[1]), "=") + if len(credentials) != 2 { + return nil, probe.NewError(errMissingFieldsCredentialTag) + } + if len(strings.Split(strings.TrimSpace(authFields[1]), "=")) != 2 { + return nil, probe.NewError(errMissingFieldsSignedHeadersTag) + } + if len(strings.Split(strings.TrimSpace(authFields[2]), "=")) != 2 { + return nil, probe.NewError(errMissingFieldsSignatureTag) + } + credentialElements := strings.Split(strings.TrimSpace(credentials[1]), "/") + if len(credentialElements) != 5 { + return nil, probe.NewError(errCredentialTagMalformed) + } + return credentialElements, nil +} + +// verify if rpcAuthHeader value has valid region +func isValidRPCRegion(authHeaderValue string) *probe.Error { + credentialElements, err := getRPCCredentialsFromAuth(authHeaderValue) + if err != nil { + return err.Trace() + } + region := credentialElements[2] + if region != "milkyway" { + return probe.NewError(errInvalidRegion) + } + return nil +} + +// stripRPCAccessKeyID - strip only access key id from auth header +func stripRPCAccessKeyID(authHeaderValue string) (string, *probe.Error) { + if err := isValidRegion(authHeaderValue); err != nil { + return "", err.Trace() + } + credentialElements, err := getRPCCredentialsFromAuth(authHeaderValue) + if err != nil { + return "", err.Trace() + } + accessKeyID := credentialElements[0] + if !IsValidAccessKey(accessKeyID) { + return "", probe.NewError(errAccessKeyIDInvalid) + } + return accessKeyID, nil +} + +// initSignatureRPC initializing rpc signature verification +func initSignatureRPC(req *http.Request) (*rpcSignature, *probe.Error) { + // strip auth from authorization header + authHeaderValue := req.Header.Get("Authorization") + accessKeyID, err := stripAccessKeyID(authHeaderValue) + if err != nil { + return nil, err.Trace() + } + authConfig, err := LoadConfig() + if err != nil { + return nil, err.Trace() + } + authFields := strings.Split(strings.TrimSpace(authHeaderValue), ",") + signedHeaders := strings.Split(strings.Split(strings.TrimSpace(authFields[1]), "=")[1], ";") + signature := strings.Split(strings.TrimSpace(authFields[2]), "=")[1] + for _, user := range authConfig.Users { + if user.AccessKeyID == accessKeyID { + signature := &rpcSignature{ + AccessKeyID: user.AccessKeyID, + SecretAccessKey: user.SecretAccessKey, + Signature: signature, + SignedHeaders: signedHeaders, + Request: req, + } + return signature, nil + } + } + return nil, probe.NewError(errAccessKeyIDInvalid) +} diff --git a/controller-rpc.go b/controller-rpc.go index eb305e7f3..86ba1d96e 100644 --- a/controller-rpc.go +++ b/controller-rpc.go @@ -150,6 +150,14 @@ func (s *controllerRPCService) ResetAuth(r *http.Request, args *AuthArgs, reply return nil } +func readAuthConfig() (*AuthConfig, *probe.Error) { + authConfig, err := LoadConfig() + if err != nil { + return nil, err.Trace() + } + return authConfig, nil +} + func proxyRequest(method, host string, ssl bool, res interface{}) *probe.Error { u := &url.URL{} if ssl { @@ -169,7 +177,11 @@ func proxyRequest(method, host string, ssl bool, res interface{}) *probe.Error { Method: method, Request: ServerArg{}, } - request, err := newRPCRequest(u.String(), op, nil) + authConfig, err := readAuthConfig() + if err != nil { + return err.Trace() + } + request, err := newRPCRequest(authConfig, u.String(), op, nil) if err != nil { return err.Trace() } @@ -216,39 +228,11 @@ func (s *controllerRPCService) DiscoverServers(r *http.Request, args *DiscoverAr defer close(c) for _, host := range args.Hosts { go func(c chan DiscoverRepEntry, host string) { - u := &url.URL{} - if args.SSL { - u.Scheme = "https" - } else { - u.Scheme = "http" - } - if args.Port != 0 { - u.Host = host + ":" + string(args.Port) - } else { - u.Host = host + ":9002" - } - u.Path = "/rpc" - - op := rpcOperation{ - Method: "Server.Version", - Request: ServerArg{}, - } - versionrep := VersionRep{} - request, err := newRPCRequest(u.String(), op, nil) - if err != nil { - c <- DiscoverRepEntry{host, err.ToGoError().Error()} - return - } - var resp *http.Response - resp, err = request.Do() + err := proxyRequest("Server.Version", host, args.SSL, rep) if err != nil { c <- DiscoverRepEntry{host, err.ToGoError().Error()} return } - if err := json.DecodeClientResponse(resp.Body, &versionrep); err != nil { - c <- DiscoverRepEntry{host, err.Error()} - return - } c <- DiscoverRepEntry{host, ""} }(c, host) } diff --git a/controller_rpc_test.go b/controller_rpc_test.go index 9ebde48b0..2e7389fed 100644 --- a/controller_rpc_test.go +++ b/controller_rpc_test.go @@ -17,6 +17,7 @@ package main import ( + "io" "io/ioutil" "net/http" "net/http/httptest" @@ -28,8 +29,11 @@ import ( ) type ControllerRPCSuite struct { - root string - url *url.URL + root string + url *url.URL + req *http.Request + body io.ReadSeeker + config *AuthConfig } var _ = Suite(&ControllerRPCSuite{}) @@ -45,8 +49,24 @@ func (s *ControllerRPCSuite) SetUpSuite(c *C) { s.root = root SetAuthConfigPath(root) - testControllerRPC = httptest.NewServer(getControllerRPCHandler()) - testServerRPC = httptest.NewUnstartedServer(getServerRPCHandler()) + secretAccessKey, perr := generateSecretAccessKey() + c.Assert(perr, IsNil) + + authConf := &AuthConfig{} + authConf.Users = make(map[string]*AuthUser) + authConf.Users["admin"] = &AuthUser{ + Name: "admin", + AccessKeyID: "admin", + SecretAccessKey: string(secretAccessKey), + } + s.config = authConf + + SetAuthConfigPath(root) + perr = SaveConfig(authConf) + c.Assert(perr, IsNil) + + testControllerRPC = httptest.NewServer(getControllerRPCHandler(false)) + testServerRPC = httptest.NewUnstartedServer(getServerRPCHandler(false)) testServerRPC.Config.Addr = ":9002" testServerRPC.Start() @@ -66,7 +86,7 @@ func (s *ControllerRPCSuite) TestMemStats(c *C) { Method: "Controller.GetServerMemStats", Request: ControllerArgs{Host: s.url.Host}, } - req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err := req.Do() @@ -84,7 +104,7 @@ func (s *ControllerRPCSuite) TestDiskStats(c *C) { Method: "Controller.GetServerDiskStats", Request: ControllerArgs{Host: s.url.Host}, } - req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err := req.Do() @@ -102,7 +122,7 @@ func (s *ControllerRPCSuite) TestSysInfo(c *C) { Method: "Controller.GetServerSysInfo", Request: ControllerArgs{Host: s.url.Host}, } - req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err := req.Do() @@ -120,7 +140,7 @@ func (s *ControllerRPCSuite) TestServerList(c *C) { Method: "Controller.ListServers", Request: ControllerArgs{Host: s.url.Host}, } - req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err := req.Do() @@ -138,7 +158,7 @@ func (s *ControllerRPCSuite) TestServerAdd(c *C) { Method: "Controller.AddServer", Request: ControllerArgs{Host: s.url.Host}, } - req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err := req.Do() @@ -156,7 +176,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) { Method: "Controller.GenerateAuth", Request: AuthArgs{User: "newuser"}, } - req, err := newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err := newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err := req.Do() @@ -175,7 +195,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) { Method: "Controller.FetchAuth", Request: AuthArgs{User: "newuser"}, } - req, err = newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err = newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err = req.Do() @@ -194,7 +214,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) { Method: "Controller.ResetAuth", Request: AuthArgs{User: "newuser"}, } - req, err = newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err = newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err = req.Do() @@ -216,7 +236,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) { Method: "Controller.GenerateAuth", Request: AuthArgs{User: "newuser"}, } - req, err = newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err = newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err = req.Do() @@ -228,7 +248,7 @@ func (s *ControllerRPCSuite) TestAuth(c *C) { Method: "Controller.GenerateAuth", Request: AuthArgs{User: ""}, } - req, err = newRPCRequest(testControllerRPC.URL+"/rpc", op, http.DefaultTransport) + req, err = newRPCRequest(s.config, testControllerRPC.URL+"/rpc", op, http.DefaultTransport) c.Assert(err, IsNil) c.Assert(req.Get("Content-Type"), Equals, "application/json") resp, err = req.Do() diff --git a/server-api-generic-handlers.go b/generic-handlers.go similarity index 85% rename from server-api-generic-handlers.go rename to generic-handlers.go index 05a9fff25..51840157e 100644 --- a/server-api-generic-handlers.go +++ b/generic-handlers.go @@ -22,17 +22,23 @@ import ( "strings" "time" + router "github.com/gorilla/mux" "github.com/rs/cors" ) // MiddlewareHandler - useful to chain different middleware http.Handler type MiddlewareHandler func(http.Handler) http.Handler -type timeHandler struct { - handler http.Handler +func registerCustomMiddleware(mux *router.Router, mwHandlers ...MiddlewareHandler) http.Handler { + var f http.Handler + f = mux + for _, mw := range mwHandlers { + f = mw(f) + } + return f } -type validateAuthHandler struct { +type timeHandler struct { handler http.Handler } @@ -54,6 +60,19 @@ func parseDate(req *http.Request) (time.Time, error) { return time.Parse(iso8601Format, amzDate) } } + minioDate := req.Header.Get(http.CanonicalHeaderKey("x-minio-date")) + switch { + case minioDate != "": + if _, err := time.Parse(time.RFC1123, minioDate); err == nil { + return time.Parse(time.RFC1123, minioDate) + } + if _, err := time.Parse(time.RFC1123Z, minioDate); err == nil { + return time.Parse(time.RFC1123Z, minioDate) + } + if _, err := time.Parse(iso8601Format, minioDate); err == nil { + return time.Parse(iso8601Format, minioDate) + } + } date := req.Header.Get("Date") switch { case date != "": @@ -78,7 +97,7 @@ func TimeValidityHandler(h http.Handler) http.Handler { func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Verify if date headers are set, if not reject the request if r.Header.Get("Authorization") != "" { - if r.Header.Get(http.CanonicalHeaderKey("x-amz-date")) == "" && r.Header.Get("Date") == "" { + if r.Header.Get(http.CanonicalHeaderKey("x-amz-date")) == "" && r.Header.Get(http.CanonicalHeaderKey("x-minio-date")) == "" && r.Header.Get("Date") == "" { // there is no way to knowing if this is a valid request, could be a attack reject such clients writeErrorResponse(w, r, RequestTimeTooSkewed, r.URL.Path) return diff --git a/donut-metadata.md b/pkg/donut/donut-metadata.md similarity index 100% rename from donut-metadata.md rename to pkg/donut/donut-metadata.md diff --git a/pkg/signature/signature-v4.go b/pkg/signature/signature-v4.go index 92555ee56..c53c7e2b6 100644 --- a/pkg/signature/signature-v4.go +++ b/pkg/signature/signature-v4.go @@ -97,7 +97,7 @@ func getURLEncodedName(name string) string { } // getCanonicalHeaders generate a list of request headers with their values -func (r *Signature) getCanonicalHeaders(signedHeaders map[string][]string) string { +func (r Signature) getCanonicalHeaders(signedHeaders map[string][]string) string { var headers []string vals := make(map[string][]string) for k, vv := range signedHeaders { @@ -129,7 +129,7 @@ func (r *Signature) getCanonicalHeaders(signedHeaders map[string][]string) strin } // getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names -func (r *Signature) getSignedHeaders(signedHeaders map[string][]string) string { +func (r Signature) getSignedHeaders(signedHeaders map[string][]string) string { var headers []string for k := range signedHeaders { headers = append(headers, strings.ToLower(k)) @@ -190,7 +190,7 @@ func (r *Signature) getCanonicalRequest() string { // \n // // -func (r *Signature) getPresignedCanonicalRequest(presignedQuery string) string { +func (r Signature) getPresignedCanonicalRequest(presignedQuery string) string { rawQuery := strings.Replace(presignedQuery, "+", "%20", -1) encodedPath := getURLEncodedName(r.Request.URL.Path) // convert any space strings back to "+" @@ -207,7 +207,7 @@ func (r *Signature) getPresignedCanonicalRequest(presignedQuery string) string { } // getScope generate a string of a specific date, an AWS region, and a service -func (r *Signature) getScope(t time.Time) string { +func (r Signature) getScope(t time.Time) string { scope := strings.Join([]string{ t.Format(yyyymmdd), "milkyway", @@ -218,7 +218,7 @@ func (r *Signature) getScope(t time.Time) string { } // getStringToSign a string based on selected query values -func (r *Signature) getStringToSign(canonicalRequest string, t time.Time) string { +func (r Signature) getStringToSign(canonicalRequest string, t time.Time) string { stringToSign := authHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n" stringToSign = stringToSign + r.getScope(t) + "\n" stringToSign = stringToSign + hex.EncodeToString(sha256.Sum256([]byte(canonicalRequest))) @@ -226,7 +226,7 @@ func (r *Signature) getStringToSign(canonicalRequest string, t time.Time) string } // getSigningKey hmac seed to calculate final signature -func (r *Signature) getSigningKey(t time.Time) []byte { +func (r Signature) getSigningKey(t time.Time) []byte { secret := r.SecretAccessKey date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) region := sumHMAC(date, []byte("milkyway")) @@ -236,7 +236,7 @@ func (r *Signature) getSigningKey(t time.Time) []byte { } // getSignature final signature in hexadecimal form -func (r *Signature) getSignature(signingKey []byte, stringToSign string) string { +func (r Signature) getSignature(signingKey []byte, stringToSign string) string { return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) } diff --git a/rpc-request.go b/rpc-request.go index c8a3113d2..be54f4520 100644 --- a/rpc-request.go +++ b/rpc-request.go @@ -18,7 +18,12 @@ package main import ( "bytes" + "encoding/hex" + "fmt" "net/http" + "sort" + "strings" + "time" "github.com/gorilla/rpc/v2/json" "github.com/minio/minio/pkg/probe" @@ -37,18 +42,116 @@ type rpcRequest struct { } // newRPCRequest initiate a new client RPC request -func newRPCRequest(url string, op rpcOperation, transport http.RoundTripper) (*rpcRequest, *probe.Error) { +func newRPCRequest(config *AuthConfig, url string, op rpcOperation, transport http.RoundTripper) (*rpcRequest, *probe.Error) { + t := time.Now().UTC() params, err := json.EncodeClientRequest(op.Method, op.Request) if err != nil { return nil, probe.NewError(err) } - req, err := http.NewRequest("POST", url, bytes.NewReader(params)) + + body := bytes.NewReader(params) + req, err := http.NewRequest("POST", url, body) if err != nil { return nil, probe.NewError(err) } + req.Header.Set("x-minio-date", t.Format(iso8601Format)) + + // save for subsequent use + hash := func() string { + sum256Bytes, _ := sum256Reader(body) + return hex.EncodeToString(sum256Bytes) + } + + hashedPayload := hash() + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-amz-content-sha256", hashedPayload) + + var headers []string + vals := make(map[string][]string) + for k, vv := range req.Header { + if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { + continue // ignored header + } + headers = append(headers, strings.ToLower(k)) + vals[strings.ToLower(k)] = vv + } + headers = append(headers, "host") + sort.Strings(headers) + + var canonicalHeaders bytes.Buffer + for _, k := range headers { + canonicalHeaders.WriteString(k) + canonicalHeaders.WriteByte(':') + switch { + case k == "host": + canonicalHeaders.WriteString(req.URL.Host) + fallthrough + default: + for idx, v := range vals[k] { + if idx > 0 { + canonicalHeaders.WriteByte(',') + } + canonicalHeaders.WriteString(v) + } + canonicalHeaders.WriteByte('\n') + } + } + + signedHeaders := strings.Join(headers, ";") + + req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) + encodedPath := getURLEncodedName(req.URL.Path) + // convert any space strings back to "+" + encodedPath = strings.Replace(encodedPath, "+", "%20", -1) + + // + // canonicalRequest = + // \n + // \n + // \n + // \n + // \n + // + // + canonicalRequest := strings.Join([]string{ + req.Method, + encodedPath, + req.URL.RawQuery, + canonicalHeaders.String(), + signedHeaders, + hashedPayload, + }, "\n") + + scope := strings.Join([]string{ + t.Format(yyyymmdd), + "milkyway", + "rpc", + "rpc_request", + }, "/") + + stringToSign := rpcAuthHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n" + stringToSign = stringToSign + scope + "\n" + stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest))) + + fmt.Println(config) + date := sumHMAC([]byte("MINIO"+config.Users["admin"].SecretAccessKey), []byte(t.Format(yyyymmdd))) + region := sumHMAC(date, []byte("milkyway")) + service := sumHMAC(region, []byte("rpc")) + signingKey := sumHMAC(service, []byte("rpc_request")) + + signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) + + // final Authorization header + parts := []string{ + rpcAuthHeaderPrefix + " Credential=" + config.Users["admin"].AccessKeyID + "/" + scope, + "SignedHeaders=" + signedHeaders, + "Signature=" + signature, + } + auth := strings.Join(parts, ", ") + req.Header.Set("Authorization", auth) + rpcReq := &rpcRequest{} rpcReq.req = req - rpcReq.req.Header.Set("Content-Type", "application/json") if transport == nil { transport = http.DefaultTransport } diff --git a/server-api-signature-handler.go b/server-api-signature-handler.go index 93e029664..c9272cce4 100644 --- a/server-api-signature-handler.go +++ b/server-api-signature-handler.go @@ -17,10 +17,10 @@ package main import ( - "crypto/sha256" "encoding/hex" "net/http" + "github.com/minio/minio/pkg/crypto/sha256" "github.com/minio/minio/pkg/probe" signv4 "github.com/minio/minio/pkg/signature" ) @@ -73,8 +73,7 @@ func (s signatureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } } - value := sha256.Sum256([]byte("")) - ok, err := signature.DoesSignatureMatch(hex.EncodeToString(value[:])) + ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256.Sum256([]byte("")))) if err != nil { errorIf(err.Trace(), "Unable to verify signature.", nil) writeErrorResponse(w, r, InternalError, r.URL.Path) diff --git a/server-main.go b/server-main.go index ce3edeae1..8da3430c2 100644 --- a/server-main.go +++ b/server-main.go @@ -136,7 +136,7 @@ func startServer(conf minioConfig) *probe.Error { if err != nil { return err.Trace() } - rpcServer, err := configureServerRPC(conf, getServerRPCHandler()) + rpcServer, err := configureServerRPC(conf, getServerRPCHandler(conf.Anonymous)) // start ticket master go startTM(minioAPI) diff --git a/server-router.go b/server-router.go index 568867704..47560e7c4 100644 --- a/server-router.go +++ b/server-router.go @@ -50,15 +50,6 @@ func registerAPI(mux *router.Router, a API) { mux.HandleFunc("/{bucket}/{object:.*}", a.DeleteObjectHandler).Methods("DELETE") } -func registerCustomMiddleware(mux *router.Router, mwHandlers ...MiddlewareHandler) http.Handler { - var f http.Handler - f = mux - for _, mw := range mwHandlers { - f = mw(f) - } - return f -} - // APIOperation container for individual operations read by Ticket Master type APIOperation struct { ProceedCh chan struct{} @@ -99,12 +90,21 @@ func getAPIHandler(anonymous bool, api API) http.Handler { return apiHandler } -func getServerRPCHandler() http.Handler { +func getServerRPCHandler(anonymous bool) http.Handler { + var mwHandlers = []MiddlewareHandler{ + TimeValidityHandler, + } + if !anonymous { + mwHandlers = append(mwHandlers, RPCSignatureHandler) + } + s := jsonrpc.NewServer() s.RegisterCodec(json.NewCodec(), "application/json") s.RegisterService(new(serverRPCService), "Server") s.RegisterService(new(donutRPCService), "Donut") mux := router.NewRouter() mux.Handle("/rpc", s) - return mux + + rpcHandler := registerCustomMiddleware(mux, mwHandlers...) + return rpcHandler } diff --git a/server_signV4_donut_cache_test.go b/server_signV4_donut_cache_test.go index a420c7678..79efb751f 100644 --- a/server_signV4_donut_cache_test.go +++ b/server_signV4_donut_cache_test.go @@ -159,7 +159,7 @@ func (s *MyAPIDonutCacheSuite) newRequest(method, urlStr string, contentLength i signedHeaders := strings.Join(headers, ";") req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) - encodedPath, _ := urlEncodeName(req.URL.Path) + encodedPath := getURLEncodedName(req.URL.Path) // convert any space strings back to "+" encodedPath = strings.Replace(encodedPath, "+", "%20", -1) diff --git a/server_signV4_donut_test.go b/server_signV4_donut_test.go index 16ac3217b..0ed8f26a8 100644 --- a/server_signV4_donut_test.go +++ b/server_signV4_donut_test.go @@ -18,19 +18,14 @@ package main import ( "bytes" - "crypto/hmac" - "crypto/sha256" - "errors" "io" "io/ioutil" "os" "path/filepath" - "regexp" "sort" "strconv" "strings" "time" - "unicode/utf8" "encoding/hex" "encoding/xml" @@ -113,117 +108,6 @@ func (s *MyAPISignatureV4Suite) TearDownSuite(c *C) { testSignatureV4Server.Close() } -/// -/// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 -/// -/// User-Agent: -/// -/// This is ignored from signing because signing this causes problems with generating pre-signed URLs -/// (that are executed by other agents) or when customers pass requests through proxies, which may -/// modify the user-agent. -/// -/// Content-Length: -/// -/// This is ignored from signing because generating a pre-signed URL should not provide a content-length -/// constraint, specifically when vending a S3 pre-signed PUT URL. The corollary to this is that when -/// sending regular requests (non-pre-signed), the signature contains a checksum of the body, which -/// implicitly validates the payload length (since changing the number of bytes would change the checksum) -/// and therefore this header is not valuable in the signature. -/// -/// Content-Type: -/// -/// Signing this header causes quite a number of problems in browser environments, where browsers -/// like to modify and normalize the content-type header in different ways. There is more information -/// on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding this field simplifies logic -/// and reduces the possibility of future bugs -/// -/// Authorization: -/// -/// Is skipped for obvious reasons -/// -var ignoredHeaders = map[string]bool{ - "Authorization": true, - "Content-Type": true, - "Content-Length": true, - "User-Agent": true, -} - -// urlEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences -// -// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 -// non english characters cannot be parsed due to the nature in which url.Encode() is written -// -// This function on the other hand is a direct replacement for url.Encode() technique to support -// pretty much every UTF-8 character. -func urlEncodeName(name string) (string, error) { - // if object matches reserved string, no need to encode them - reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") - if reservedNames.MatchString(name) { - return name, nil - } - var encodedName string - for _, s := range name { - if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) - encodedName = encodedName + string(s) - continue - } - switch s { - case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) - encodedName = encodedName + string(s) - continue - default: - len := utf8.RuneLen(s) - if len < 0 { - return "", errors.New("invalid utf-8") - } - u := make([]byte, len) - utf8.EncodeRune(u, s) - for _, r := range u { - hex := hex.EncodeToString([]byte{r}) - encodedName = encodedName + "%" + strings.ToUpper(hex) - } - } - } - return encodedName, nil -} - -// sum256Reader calculate sha256 sum for an input read seeker -func sum256Reader(reader io.ReadSeeker) ([]byte, error) { - h := sha256.New() - var err error - - start, _ := reader.Seek(0, 1) - defer reader.Seek(start, 0) - - for err == nil { - length := 0 - byteBuffer := make([]byte, 1024*1024) - length, err = reader.Read(byteBuffer) - byteBuffer = byteBuffer[0:length] - h.Write(byteBuffer) - } - - if err != io.EOF { - return nil, err - } - - return h.Sum(nil), nil -} - -// sum256 calculate sha256 sum for an input byte array -func sum256(data []byte) []byte { - hash := sha256.New() - hash.Write(data) - return hash.Sum(nil) -} - -// sumHMAC calculate hmac between two input byte array -func sumHMAC(key []byte, data []byte) []byte { - hash := hmac.New(sha256.New, key) - hash.Write(data) - return hash.Sum(nil) -} - func (s *MyAPISignatureV4Suite) newRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { t := time.Now().UTC() req, err := http.NewRequest(method, urlStr, nil) @@ -294,7 +178,7 @@ func (s *MyAPISignatureV4Suite) newRequest(method, urlStr string, contentLength signedHeaders := strings.Join(headers, ";") req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) - encodedPath, _ := urlEncodeName(req.URL.Path) + encodedPath := getURLEncodedName(req.URL.Path) // convert any space strings back to "+" encodedPath = strings.Replace(encodedPath, "+", "%20", -1) diff --git a/server-api-typed-errors.go b/typed-errors.go similarity index 95% rename from server-api-typed-errors.go rename to typed-errors.go index 0d86c55df..8e4b2de88 100644 --- a/server-api-typed-errors.go +++ b/typed-errors.go @@ -68,3 +68,6 @@ var errPolicyAlreadyExpired = errors.New("Policy already expired") // errPolicyMissingFields means that form values and policy header have some fields missing. var errPolicyMissingFields = errors.New("Some fields are missing or do not match in policy") + +// errMissingDateHeader means that date header is missing +var errMissingDateHeader = errors.New("Missing date header on the request")