From 5e1e5ad786dbff95434b7d68acc88247a5e78920 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 23 Feb 2015 17:44:55 -0800 Subject: [PATCH] More updates on documentation --- pkg/utils/config/config.go | 24 +++++++++++++++++++----- pkg/utils/crypto/keys/common.go | 4 ++++ pkg/utils/crypto/keys/keys.go | 8 ++++++++ pkg/utils/crypto/signers/signers.go | 9 +++++++-- pkg/utils/helpers/common.go | 4 ++++ pkg/utils/helpers/execpipe.go | 3 ++- pkg/utils/split/split.go | 17 +++++++++++------ pkg/utils/unitconv/unitconv.go | 3 +++ 8 files changed, 58 insertions(+), 14 deletions(-) diff --git a/pkg/utils/config/config.go b/pkg/utils/config/config.go index 41f1bb0b3..aaca1881a 100644 --- a/pkg/utils/config/config.go +++ b/pkg/utils/config/config.go @@ -39,6 +39,7 @@ type User struct { SecretKey string } +// Initialize config directory and template config func (c *Config) SetupConfig() error { confPath := path.Join(helpers.HomeDir(), ".minio") if err := os.MkdirAll(confPath, os.ModeDir); err != nil { @@ -58,10 +59,12 @@ func (c *Config) SetupConfig() error { return nil } +// Get config file location func (c *Config) GetConfigPath() string { return c.configPath } +// Verify if user exists func (c *Config) IsUserExists(username string) bool { for _, user := range c.Users { if user.Name == username { @@ -71,6 +74,7 @@ func (c *Config) IsUserExists(username string) bool { return false } +// Get user based on accesskey func (c *Config) GetKey(accessKey string) User { value, ok := c.Users[accessKey] if !ok { @@ -79,6 +83,7 @@ func (c *Config) GetKey(accessKey string) User { return value } +// Get user based on username func (c *Config) GetUser(username string) User { for _, user := range c.Users { if user.Name == username { @@ -88,6 +93,7 @@ func (c *Config) GetUser(username string) User { return User{} } +// Add a new user into existing User list func (c *Config) AddUser(user User) { var currentUsers map[string]User if len(c.Users) == 0 { @@ -99,12 +105,14 @@ func (c *Config) AddUser(user User) { c.Users = currentUsers } +// Write encoded json in config file func (c *Config) WriteConfig() error { + c.configLock.Lock() + defer c.configLock.Unlock() + var file *os.File var err error - c.configLock.Lock() - defer c.configLock.Unlock() file, err = os.OpenFile(c.configFile, os.O_WRONLY, 0666) defer file.Close() if err != nil { @@ -116,13 +124,14 @@ func (c *Config) WriteConfig() error { return nil } +// Read json config file and decode func (c *Config) ReadConfig() error { - var file *os.File - var err error - c.configLock.RLock() defer c.configLock.RUnlock() + var file *os.File + var err error + file, err = os.OpenFile(c.configFile, os.O_RDONLY, 0666) defer file.Close() if err != nil { @@ -143,6 +152,9 @@ func (c *Config) ReadConfig() error { } } +/// helpers + +// Load all users into memory func Loadusers() map[string]User { c := Config{} c.SetupConfig() @@ -150,6 +162,7 @@ func Loadusers() map[string]User { return c.Users } +// Load a given user based on accessKey func Loadkey(accessKeyId string) User { c := Config{} c.SetupConfig() @@ -157,6 +170,7 @@ func Loadkey(accessKeyId string) User { return c.GetKey(accessKeyId) } +// Load a given user based on username func Loaduser(username string) User { c := Config{} c.SetupConfig() diff --git a/pkg/utils/crypto/keys/common.go b/pkg/utils/crypto/keys/common.go index 4562d834e..454d71681 100644 --- a/pkg/utils/crypto/keys/common.go +++ b/pkg/utils/crypto/keys/common.go @@ -21,10 +21,14 @@ const ( MINIO_SECRET_ID = 40 ) +/// helpers + +// Is alphanumeric? func isalnum(c byte) bool { return '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' } +// validate access key for only alphanumeric characters func ValidateAccessKey(key []byte) bool { for _, char := range key { if isalnum(char) { diff --git a/pkg/utils/crypto/keys/keys.go b/pkg/utils/crypto/keys/keys.go index c4e423bd5..0a18b6054 100644 --- a/pkg/utils/crypto/keys/keys.go +++ b/pkg/utils/crypto/keys/keys.go @@ -21,9 +21,14 @@ import ( "encoding/base64" ) +// Static alphaNumeric table used for generating unique keys var alphaNumericTable = []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ") var alphaNumericTableFull = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") +/// helpers + +// Generate random alpha numeric value using only uppercase characters +// takes input as size in integer func GetRandomAlphaNumeric(size int) ([]byte, error) { alpha := make([]byte, size) _, err := rand.Read(alpha) @@ -37,6 +42,8 @@ func GetRandomAlphaNumeric(size int) ([]byte, error) { return alpha, nil } +// Generate random alpha numeric value using all alphanumeric characters +// takes input as size in integer func GetRandomAlphaNumericFull(size int) ([]byte, error) { alphaFull := make([]byte, size) _, err := rand.Read(alphaFull) @@ -49,6 +56,7 @@ func GetRandomAlphaNumericFull(size int) ([]byte, error) { return alphaFull, nil } +// Generate random base64 numeric value from a random seed. func GetRandomBase64(size int) ([]byte, error) { rb := make([]byte, size) _, err := rand.Read(rb) diff --git a/pkg/utils/crypto/signers/signers.go b/pkg/utils/crypto/signers/signers.go index 2848fd8f2..9379afceb 100644 --- a/pkg/utils/crypto/signers/signers.go +++ b/pkg/utils/crypto/signers/signers.go @@ -32,6 +32,7 @@ import ( "github.com/minio-io/minio/pkg/utils/config" ) +// Sign a given http request using HMAC style signatures func SignRequest(user config.User, req *http.Request) { if date := req.Header.Get("Date"); date == "" { req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) @@ -48,7 +49,7 @@ func SignRequest(user config.User, req *http.Request) { req.Header.Set("Authorization", authHeader.String()) } -// This package implements verification side of Object API Signature request +// Validate an API request by validating its signature using HMAC signatures func ValidateRequest(user config.User, req *http.Request) (bool, error) { // Verify if date headers are set, if not reject the request if req.Header.Get("x-amz-date") == "" { @@ -101,6 +102,7 @@ func getStringToSign(req *http.Request) string { return buf.String() } +// Lower all upper case letters func hasPrefixCaseInsensitive(s, pfx string) bool { if len(pfx) > len(s) { return false @@ -113,6 +115,7 @@ func hasPrefixCaseInsensitive(s, pfx string) bool { return shead == pfx || shead == strings.ToLower(pfx) } +// Canonicalize amazon special headers, headers starting with 'x-amz-' func writeCanonicalizedAmzHeaders(buf *bytes.Buffer, req *http.Request) { amzHeaders := make([]string, 0) vals := make(map[string][]string) @@ -146,7 +149,7 @@ func writeCanonicalizedAmzHeaders(buf *bytes.Buffer, req *http.Request) { } } -// Must be sorted: +// Resource list must be sorted: var subResList = []string{"acl", "lifecycle", "location", "logging", "notification", "partNumber", "policy", "requestPayment", "torrent", "uploadId", "uploads", "versionId", "versioning", "versions", "website"} // From the Amazon docs: @@ -155,6 +158,7 @@ var subResList = []string{"acl", "lifecycle", "location", "logging", "notificati // + // [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; func writeCanonicalizedResource(buf *bytes.Buffer, req *http.Request) { + // Grab bucket name from hostname bucket := bucketFromHostname(req) if bucket != "" { buf.WriteByte('/') @@ -182,6 +186,7 @@ func writeCanonicalizedResource(buf *bytes.Buffer, req *http.Request) { } } +// Convert subdomain http request into bucketname if possible func bucketFromHostname(req *http.Request) string { host := req.Host if host == "" { diff --git a/pkg/utils/helpers/common.go b/pkg/utils/helpers/common.go index 3b3eee3c6..5de35ff0f 100644 --- a/pkg/utils/helpers/common.go +++ b/pkg/utils/helpers/common.go @@ -24,6 +24,7 @@ import ( "strings" ) +// Get current user home directory func HomeDir() string { if runtime.GOOS == "windows" { home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") @@ -35,16 +36,19 @@ func HomeDir() string { return os.Getenv("HOME") } +// Create a new temp directory func MakeTempTestDir() (string, error) { return ioutil.TempDir("/tmp", "minio-test-") } +// Assert wrapper for error not being null func Assert(err error) { if err != nil { log.Fatal(err) } } +// Camelcase input string func FirstUpper(str string) string { return strings.ToUpper(str[0:1]) + str[1:] } diff --git a/pkg/utils/helpers/execpipe.go b/pkg/utils/helpers/execpipe.go index 04dcced7c..00d4da89c 100644 --- a/pkg/utils/helpers/execpipe.go +++ b/pkg/utils/helpers/execpipe.go @@ -26,7 +26,8 @@ import ( // A simple ExecPipe() pipes exec.Cmd together - somewhat similar to how bash pipes "|" behave. // Each command's standard output is connected to the standard input of the next command // and the output of the final command is returned - +// +// TODO: handle errors properly func ExecPipe(cmds ...*exec.Cmd) (pipeLineOutput io.Reader, pipeLineError error) { // Require at least one command if len(cmds) < 1 { diff --git a/pkg/utils/split/split.go b/pkg/utils/split/split.go index 0dede2b9b..6c01b9dda 100644 --- a/pkg/utils/split/split.go +++ b/pkg/utils/split/split.go @@ -33,12 +33,6 @@ type SplitMessage struct { Err error } -type JoinMessage struct { - Reader io.Reader - Length int64 - Err error -} - // SplitStream reads from io.Reader, splits the data into chunks, and sends // each chunk to the channel. This method runs until an EOF or error occurs. If // an error occurs, the method sends the error over the channel and returns. @@ -98,6 +92,17 @@ func splitStreamGoRoutine(reader io.Reader, chunkSize uint64, ch chan SplitMessa close(ch) } +// JoinFiles reads from a given directory, joins data in chunks with prefix and sends +// an io.Reader. +// +// var err error +// for err == nil { +// buf := make([]byte, 1024*1024) +// reader := JoinFiles("mydirectory", "mypreferred-prefix") +// _, err = reader.Read(buf) +// fmt.Println(buf) +// } +// func JoinFiles(dirname string, inputPrefix string) io.Reader { reader, writer := io.Pipe() fileInfos, readError := ioutil.ReadDir(dirname) diff --git a/pkg/utils/unitconv/unitconv.go b/pkg/utils/unitconv/unitconv.go index a40cfc6ca..7e1f59bd9 100644 --- a/pkg/utils/unitconv/unitconv.go +++ b/pkg/utils/unitconv/unitconv.go @@ -24,6 +24,7 @@ import ( "strings" ) +// Units of various bytes in ascending order const ( UNIT_BYTE = 1 << (10 * iota) UNIT_KILOBYTE @@ -33,6 +34,7 @@ const ( UNIT_PETABYTE ) +// Convert bytes length in integer to human readable string func BytesToString(bytes uint64) string { var unit string = "B" var value uint64 = 0 @@ -58,6 +60,7 @@ func BytesToString(bytes uint64) string { return fmt.Sprintf("%d%s", value, unit) } +// Convert human readable string to bytes length in integer func StringToBytes(s string) (uint64, error) { var bytes uint64 var err error