From f0a8dbecae75d2c222cada6e5506877037420288 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sun, 4 Oct 2015 23:11:18 -0700 Subject: [PATCH] Add --json output formatter for server --- controller-main.go | 56 ++++++++++++++++++++++++++----------- controller-rpc.go | 8 +++--- flags.go | 19 +++++++------ main.go | 6 ++-- notifier.go | 4 +-- pkg/donut/donut-v2.go | 44 ++++++++++++++++------------- pkg/donut/multipart.go | 17 ++++++----- server-auth-config.go | 6 ++-- server-auth.go | 30 +++++++++++++++++--- server_auth_test.go | 4 +-- server_donut_test.go | 12 -------- server_signature_v4_test.go | 16 ++--------- 12 files changed, 127 insertions(+), 95 deletions(-) diff --git a/controller-main.go b/controller-main.go index 40d3165d6..c3ca3a8bc 100644 --- a/controller-main.go +++ b/controller-main.go @@ -18,6 +18,8 @@ package main import ( "crypto/tls" + "encoding/json" + "fmt" "net" "net/http" "os" @@ -42,7 +44,7 @@ EXAMPLES: 1. Start minio controller $ minio {{.Name}} - 2. Colored output of generated keys + 2. Fetch stored access keys $ minio {{.Name}} keys `, } @@ -117,18 +119,16 @@ func genAuthFirstTime() (*AuthConfig, *probe.Error) { config := &AuthConfig{} config.Version = "0.0.1" config.Users = make(map[string]*AuthUser) - accessKeyID, err := GenerateAccessKeyID() - if err != nil { - return nil, err.Trace() - } - secretAccessKey, err := GenerateSecretAccessKey() - if err != nil { - return nil, err.Trace() - } + config.Users["admin"] = &AuthUser{ Name: "admin", - AccessKeyID: string(accessKeyID), - SecretAccessKey: string(secretAccessKey), + AccessKeyID: "admin", + SecretAccessKey: string(mustGenerateSecretAccessKey()), + } + config.Users["user"] = &AuthUser{ + Name: "user", + AccessKeyID: string(mustGenerateAccessKeyID()), + SecretAccessKey: string(mustGenerateSecretAccessKey()), } if err := SaveConfig(config); err != nil { return nil, err.Trace() @@ -144,6 +144,21 @@ func getAuth() (*AuthConfig, *probe.Error) { return config, nil } +type accessKeys struct { + *AuthUser +} + +func (a accessKeys) String() string { + return colorizeMessage(fmt.Sprintf("Username: %s, AccessKey: %s, SecretKey: %s", a.Name, a.AccessKeyID, a.SecretAccessKey)) +} + +// JSON - json formatted output +func (a accessKeys) JSON() string { + b, err := json.Marshal(a) + errorIf(probe.NewError(err), "Unable to marshal json", nil) + return string(b) +} + // firstTimeAuth first time authorization func firstTimeAuth() *probe.Error { conf, err := genAuthFirstTime() @@ -151,11 +166,17 @@ func firstTimeAuth() *probe.Error { return err.Trace() } if conf != nil { + Println("Running for first time, generating access keys.") for _, user := range conf.Users { - Println(colorizeMessage("AccessKey: " + user.AccessKeyID)) - Println(colorizeMessage("SecretKey: " + user.SecretAccessKey)) - Println(colorizeMessage("$ minio controller keys")) + if globalJSONFlag { + Println(accessKeys{user}.JSON()) + } else { + Println(accessKeys{user}) + } + } + Println("To fetch your keys again.") + Println(" $ minio controller keys") } return nil } @@ -186,8 +207,11 @@ func controllerMain(c *cli.Context) { fatalIf(err.Trace(), "Failed to fetch keys for minio controller.", nil) if conf != nil { for _, user := range conf.Users { - Println(colorizeMessage("AccessKey: " + user.AccessKeyID)) - Println(colorizeMessage("SecretKey: " + user.SecretAccessKey)) + if globalJSONFlag { + Println(accessKeys{user}.JSON()) + } else { + Println(accessKeys{user}) + } } } return diff --git a/controller-rpc.go b/controller-rpc.go index 72931440d..eb305e7f3 100644 --- a/controller-rpc.go +++ b/controller-rpc.go @@ -49,13 +49,13 @@ func generateAuth(args *AuthArgs, reply *AuthRep) *probe.Error { if _, ok := config.Users[args.User]; ok { return probe.NewError(errors.New("Credentials already set, if you wish to change this invoke Reset() method")) } - accessKeyID, err := GenerateAccessKeyID() + accessKeyID, err := generateAccessKeyID() if err != nil { return err.Trace() } reply.AccessKeyID = string(accessKeyID) - secretAccessKey, err := GenerateSecretAccessKey() + secretAccessKey, err := generateSecretAccessKey() if err != nil { return err.Trace() } @@ -97,12 +97,12 @@ func resetAuth(args *AuthArgs, reply *AuthRep) *probe.Error { if _, ok := config.Users[args.User]; !ok { return probe.NewError(errors.New("User not found")) } - accessKeyID, err := GenerateAccessKeyID() + accessKeyID, err := generateAccessKeyID() if err != nil { return err.Trace() } reply.AccessKeyID = string(accessKeyID) - secretAccessKey, err := GenerateSecretAccessKey() + secretAccessKey, err := generateSecretAccessKey() if err != nil { return err.Trace() } diff --git a/flags.go b/flags.go index 8977ed63b..63cbf27b3 100644 --- a/flags.go +++ b/flags.go @@ -25,42 +25,43 @@ var ( addressFlag = cli.StringFlag{ Name: "address", Value: ":9000", - Usage: "ADDRESS:PORT for cloud storage access", + Usage: "ADDRESS:PORT for cloud storage access.", } addressControllerFlag = cli.StringFlag{ Name: "address-controller", Hide: true, Value: ":9001", - Usage: "ADDRESS:PORT for management console access", + Usage: "ADDRESS:PORT for management console access.", } addressServerRPCFlag = cli.StringFlag{ Name: "address-server-rpc", Hide: true, Value: ":9002", - Usage: "ADDRESS:PORT for management console access", + Usage: "ADDRESS:PORT for management console access.", } ratelimitFlag = cli.IntFlag{ Name: "ratelimit", + Hide: true, Value: 16, - Usage: "Limit for total concurrent requests: [DEFAULT: 16]", + Usage: "Limit for total concurrent requests: [DEFAULT: 16].", } certFlag = cli.StringFlag{ Name: "cert", - Usage: "Provide your domain certificate", + Usage: "Provide your domain certificate.", } keyFlag = cli.StringFlag{ Name: "key", - Usage: "Provide your domain private key", + Usage: "Provide your domain private key.", } - debugFlag = cli.BoolFlag{ - Name: "debug", - Usage: "print debug information", + jsonFlag = cli.BoolFlag{ + Name: "json", + Usage: "Enable json formatted output.", } ) diff --git a/main.go b/main.go index adc4e3c93..5b9a40575 100644 --- a/main.go +++ b/main.go @@ -98,11 +98,11 @@ func registerApp() *cli.App { // register all flags registerFlag(addressFlag) registerFlag(addressControllerFlag) + registerFlag(addressServerRPCFlag) registerFlag(ratelimitFlag) registerFlag(certFlag) registerFlag(keyFlag) - registerFlag(debugFlag) - registerFlag(addressServerRPCFlag) + registerFlag(jsonFlag) // set up app app := cli.NewApp() @@ -151,7 +151,7 @@ VERSION: func main() { app := registerApp() app.Before = func(c *cli.Context) error { - // get flag and set global defaults here. + globalJSONFlag = c.GlobalBool("json") return nil } app.ExtraInfo = func() map[string]string { diff --git a/notifier.go b/notifier.go index dd41a1cfe..a3ddc56af 100644 --- a/notifier.go +++ b/notifier.go @@ -42,8 +42,8 @@ func colorizeMessage(message string) string { terminal, err := ts.GetSize() if err != nil { - globalJSONFlag = true - return "" + // no coloring needed just send as is + return message } var msg string switch { diff --git a/pkg/donut/donut-v2.go b/pkg/donut/donut-v2.go index 77d0e2ffc..21e648e5c 100644 --- a/pkg/donut/donut-v2.go +++ b/pkg/donut/donut-v2.go @@ -187,21 +187,19 @@ func (donut API) GetObject(w io.Writer, bucket string, object string, start, len } return 0, probe.NewError(ObjectNotFound{Object: object}) } - { - var err error - if start == 0 && length == 0 { - written, err = io.CopyN(w, bytes.NewBuffer(data), int64(donut.objects.Len(objectKey))) - if err != nil { - return 0, probe.NewError(err) - } - } else { - written, err = io.CopyN(w, bytes.NewBuffer(data[start:]), length) - if err != nil { - return 0, probe.NewError(err) - } + var err error + if start == 0 && length == 0 { + written, err = io.CopyN(w, bytes.NewBuffer(data), int64(donut.objects.Len(objectKey))) + if err != nil { + return 0, probe.NewError(err) } return written, nil } + written, err = io.CopyN(w, bytes.NewBuffer(data[start:]), length) + if err != nil { + return 0, probe.NewError(err) + } + return written, nil } // GetBucketMetadata - @@ -353,14 +351,16 @@ func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, s var length int byteBuffer := make([]byte, 1024*1024) length, err = data.Read(byteBuffer) - hash.Write(byteBuffer[0:length]) - sha256hash.Write(byteBuffer[0:length]) - ok := donut.objects.Append(objectKey, byteBuffer[0:length]) - if !ok { - return ObjectMetadata{}, probe.NewError(InternalError{}) + if length != 0 { + hash.Write(byteBuffer[0:length]) + sha256hash.Write(byteBuffer[0:length]) + ok := donut.objects.Append(objectKey, byteBuffer[0:length]) + if !ok { + return ObjectMetadata{}, probe.NewError(InternalError{}) + } + totalLength += int64(length) + go debug.FreeOSMemory() } - totalLength += int64(length) - go debug.FreeOSMemory() } if size != 0 { if totalLength != size { @@ -377,15 +377,21 @@ func (donut API) createObject(bucket, key, contentType, expectedMD5Sum string, s // Verify if the written object is equal to what is expected, only if it is requested as such if strings.TrimSpace(expectedMD5Sum) != "" { if err := isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), md5Sum); err != nil { + // Delete perhaps the object is already saved, due to the nature of append() + donut.objects.Delete(objectKey) return ObjectMetadata{}, probe.NewError(BadDigest{}) } } if signature != nil { ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256hash.Sum(nil))) if err != nil { + // Delete perhaps the object is already saved, due to the nature of append() + donut.objects.Delete(objectKey) return ObjectMetadata{}, err.Trace() } if !ok { + // Delete perhaps the object is already saved, due to the nature of append() + donut.objects.Delete(objectKey) return ObjectMetadata{}, probe.NewError(signv4.DoesNotMatch{}) } } diff --git a/pkg/donut/multipart.go b/pkg/donut/multipart.go index 0dc2826da..80e4448fa 100644 --- a/pkg/donut/multipart.go +++ b/pkg/donut/multipart.go @@ -23,6 +23,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/xml" + "io" "io/ioutil" "math/rand" @@ -190,14 +191,16 @@ func (donut API) createObjectPart(bucket, key, uploadID string, partID int, cont var length int byteBuffer := make([]byte, 1024*1024) length, err = data.Read(byteBuffer) // do not read error return error here, we will handle this error later - hash.Write(byteBuffer[0:length]) - sha256hash.Write(byteBuffer[0:length]) - ok := donut.multiPartObjects[uploadID].Append(partID, byteBuffer[0:length]) - if !ok { - return "", probe.NewError(InternalError{}) + if length != 0 { + hash.Write(byteBuffer[0:length]) + sha256hash.Write(byteBuffer[0:length]) + ok := donut.multiPartObjects[uploadID].Append(partID, byteBuffer[0:length]) + if !ok { + return "", probe.NewError(InternalError{}) + } + totalLength += int64(length) + go debug.FreeOSMemory() } - totalLength += int64(length) - go debug.FreeOSMemory() } if totalLength != size { donut.multiPartObjects[uploadID].Delete(partID) diff --git a/server-auth-config.go b/server-auth-config.go index 1af01a6d7..35c7ed83c 100644 --- a/server-auth-config.go +++ b/server-auth-config.go @@ -27,9 +27,9 @@ import ( // AuthUser container type AuthUser struct { - Name string - AccessKeyID string - SecretAccessKey string + Name string `json:"name"` + AccessKeyID string `json:"accessKeyId"` + SecretAccessKey string `json:"secretAccessKey"` } // AuthConfig auth keys diff --git a/server-auth.go b/server-auth.go index 5d41b7ae3..619a47b7b 100644 --- a/server-auth.go +++ b/server-auth.go @@ -23,9 +23,9 @@ import ( "github.com/minio/minio/pkg/probe" ) -// GenerateAccessKeyID - generate random alpha numeric value using only uppercase characters +// generateAccessKeyID - generate random alpha numeric value using only uppercase characters // takes input as size in integer -func GenerateAccessKeyID() ([]byte, *probe.Error) { +func generateAccessKeyID() ([]byte, *probe.Error) { alpha := make([]byte, MinioAccessID) _, err := rand.Read(alpha) if err != nil { @@ -37,8 +37,8 @@ func GenerateAccessKeyID() ([]byte, *probe.Error) { return alpha, nil } -// GenerateSecretAccessKey - generate random base64 numeric value from a random seed. -func GenerateSecretAccessKey() ([]byte, *probe.Error) { +// generateSecretAccessKey - generate random base64 numeric value from a random seed. +func generateSecretAccessKey() ([]byte, *probe.Error) { rb := make([]byte, MinioSecretID) _, err := rand.Read(rb) if err != nil { @@ -46,3 +46,25 @@ func GenerateSecretAccessKey() ([]byte, *probe.Error) { } return []byte(base64.StdEncoding.EncodeToString(rb))[:MinioSecretID], nil } + +// mustGenerateAccessKeyID - must generate random alpha numeric value using only uppercase characters +// takes input as size in integer +func mustGenerateAccessKeyID() []byte { + alpha := make([]byte, MinioAccessID) + _, err := rand.Read(alpha) + fatalIf(probe.NewError(err), "Unable to get random number from crypto/rand.", nil) + + for i := 0; i < MinioAccessID; i++ { + alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))] + } + return alpha +} + +// mustGenerateSecretAccessKey - generate random base64 numeric value from a random seed. +func mustGenerateSecretAccessKey() []byte { + rb := make([]byte, MinioSecretID) + _, err := rand.Read(rb) + fatalIf(probe.NewError(err), "Unable to get random number from crypto/rand.", nil) + + return []byte(base64.StdEncoding.EncodeToString(rb))[:MinioSecretID] +} diff --git a/server_auth_test.go b/server_auth_test.go index 19b74652a..d93406336 100644 --- a/server_auth_test.go +++ b/server_auth_test.go @@ -23,10 +23,10 @@ type MySuite struct{} var _ = Suite(&MySuite{}) func (s *MySuite) TestAuth(c *C) { - secretID, err := GenerateSecretAccessKey() + secretID, err := generateSecretAccessKey() c.Assert(err, IsNil) - accessID, err := GenerateAccessKeyID() + accessID, err := generateAccessKeyID() c.Assert(err, IsNil) c.Assert(len(secretID), Equals, MinioSecretID) diff --git a/server_donut_test.go b/server_donut_test.go index 700461e8b..60fe2a528 100644 --- a/server_donut_test.go +++ b/server_donut_test.go @@ -893,16 +893,4 @@ func (s *MyAPIDonutSuite) TestObjectMultipart(c *C) { response, err = client.Do(request) c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) - - /* - request, err = http.NewRequest("GET", testAPIDonutServer.URL+"/objectmultiparts/object", nil) - c.Assert(err, IsNil) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - object, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(string(object), Equals, ("hello worldhello world")) - */ } diff --git a/server_signature_v4_test.go b/server_signature_v4_test.go index fb550347b..a02a72018 100644 --- a/server_signature_v4_test.go +++ b/server_signature_v4_test.go @@ -66,9 +66,9 @@ func (s *MyAPISignatureV4Suite) SetUpSuite(c *C) { perr := donut.SaveConfig(conf) c.Assert(perr, IsNil) - accessKeyID, perr := GenerateAccessKeyID() + accessKeyID, perr := generateAccessKeyID() c.Assert(perr, IsNil) - secretAccessKey, perr := GenerateSecretAccessKey() + secretAccessKey, perr := generateSecretAccessKey() c.Assert(perr, IsNil) authConf := &AuthConfig{} @@ -1133,16 +1133,4 @@ func (s *MyAPISignatureV4Suite) TestObjectMultipart(c *C) { response, err = client.Do(request) c.Assert(err, IsNil) c.Assert(response.StatusCode, Equals, http.StatusOK) - - /* - request, err = s.newRequest("GET", testSignatureV4Server.URL+"/objectmultiparts/object", 0, nil) - c.Assert(err, IsNil) - - response, err = client.Do(request) - c.Assert(err, IsNil) - c.Assert(response.StatusCode, Equals, http.StatusOK) - object, err := ioutil.ReadAll(response.Body) - c.Assert(err, IsNil) - c.Assert(string(object), Equals, ("hello worldhello world")) - */ }