Add support for reading and saving config on Gateway. (#4463)

This is also a first step towards supporting bucket
notification for gateway.
master
Harshavardhana 8 years ago
parent 4fb5fc72d7
commit f99f218999
  1. 117
      cmd/common-main.go
  2. 25
      cmd/gateway-azure.go
  3. 6
      cmd/gateway-gcs-errors.go
  4. 23
      cmd/gateway-gcs.go
  5. 144
      cmd/gateway-handlers.go
  6. 222
      cmd/gateway-main.go
  7. 23
      cmd/gateway-main_test.go
  8. 32
      cmd/gateway-s3.go
  9. 31
      cmd/gateway-startup-msg.go
  10. 20
      cmd/gateway-startup-msg_test.go
  11. 96
      cmd/server-main.go
  12. 8
      cmd/server-startup-msg.go
  13. 2
      cmd/server-startup-msg_test.go

@ -0,0 +1,117 @@
/*
* Minio Cloud Storage, (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cmd
import (
"errors"
"os"
"path/filepath"
"strings"
"time"
"github.com/minio/cli"
)
// Check for updates and print a notification message
func checkUpdate(mode string) {
// Its OK to ignore any errors during getUpdateInfo() here.
if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
log.Println(updateMsg)
}
}
}
func enableLoggers() {
fileLogTarget := serverConfig.Logger.GetFile()
if fileLogTarget.Enable {
err := InitFileLogger(&fileLogTarget)
fatalIf(err, "Unable to initialize file logger")
log.AddTarget(fileLogTarget)
}
consoleLogTarget := serverConfig.Logger.GetConsole()
if consoleLogTarget.Enable {
InitConsoleLogger(&consoleLogTarget)
}
log.SetConsoleTarget(consoleLogTarget)
}
func initConfig() {
// Config file does not exist, we create it fresh and return upon success.
if isFile(getConfigFile()) {
fatalIf(migrateConfig(), "Config migration failed.")
fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
} else {
fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
log.Println("Created minio configuration file successfully at " + getConfigDir())
}
}
func handleCommonCmdArgs(ctx *cli.Context) {
// Set configuration directory.
{
// Get configuration directory from command line argument.
configDir := ctx.String("config-dir")
if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
configDir = ctx.GlobalString("config-dir")
}
if configDir == "" {
fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
}
// Disallow relative paths, figure out absolute paths.
configDirAbs, err := filepath.Abs(configDir)
fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
setConfigDir(configDirAbs)
}
}
func handleCommonEnvVars() {
// Start profiler if env is set.
if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
globalProfiler = startProfiler(profiler)
}
// Check if object cache is disabled.
globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
accessKey := os.Getenv("MINIO_ACCESS_KEY")
secretKey := os.Getenv("MINIO_SECRET_KEY")
if accessKey != "" && secretKey != "" {
cred, err := createCredential(accessKey, secretKey)
fatalIf(err, "Invalid access/secret Key set in environment.")
// credential Envs are set globally.
globalIsEnvCreds = true
globalActiveCred = cred
}
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
browserFlag, err := ParseBrowserFlag(browser)
if err != nil {
fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
}
// browser Envs are set globally, this does not represent
// if browser is turned off or on.
globalIsEnvBrowser = true
globalIsBrowserEnabled = bool(browserFlag)
}
}

@ -26,14 +26,12 @@ import (
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"os"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
"github.com/Azure/azure-sdk-for-go/storage" "github.com/Azure/azure-sdk-for-go/storage"
"github.com/minio/cli"
"github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/policy"
"github.com/minio/sha256-simd" "github.com/minio/sha256-simd"
) )
@ -157,32 +155,30 @@ func azureToObjectError(err error, params ...string) error {
} }
// Inits azure blob storage client and returns AzureObjects. // Inits azure blob storage client and returns AzureObjects.
func newAzureLayer(args cli.Args) (GatewayLayer, error) { func newAzureLayer(host string) (GatewayLayer, error) {
var err error var err error
var endpoint = storage.DefaultBaseURL
// Default endpoint parameters var secure = true
endPoint := storage.DefaultBaseURL
secure := true
// If user provided some parameters // If user provided some parameters
if args.Present() { if host != "" {
endPoint, secure, err = parseGatewayEndpoint(args.First()) endpoint, secure, err = parseGatewayEndpoint(host)
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
account := os.Getenv("MINIO_ACCESS_KEY") creds := serverConfig.GetCredential()
key := os.Getenv("MINIO_SECRET_KEY") if !creds.IsValid() && !globalIsEnvCreds {
if account == "" || key == "" { return nil, errors.New("Azure backend account and secret keys should be set through ENVs")
return nil, errors.New("No Azure account and key set")
} }
c, err := storage.NewClient(account, key, endPoint, globalAzureAPIVersion, secure) c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
if err != nil { if err != nil {
return &azureObjects{}, err return &azureObjects{}, err
} }
return &azureObjects{ return &azureObjects{
client: c.GetBlobService(), client: c.GetBlobService(),
metaInfo: azureMultipartMetaInfo{ metaInfo: azureMultipartMetaInfo{
@ -195,7 +191,6 @@ func newAzureLayer(args cli.Args) (GatewayLayer, error) {
// Shutdown - save any gateway metadata to disk // Shutdown - save any gateway metadata to disk
// if necessary and reload upon next restart. // if necessary and reload upon next restart.
func (a *azureObjects) Shutdown() error { func (a *azureObjects) Shutdown() error {
// TODO
return nil return nil
} }

@ -19,9 +19,9 @@ package cmd
import "errors" import "errors"
var ( var (
// Project is format is not valid // ProjectID format is not valid.
errGCSInvalidProjectID = errors.New("invalid project id") errGCSInvalidProjectID = errors.New("GCS project id is either empty or invalid")
// Multipart identifier is not in the correct form // Multipart identifier is not in the correct form.
errGCSNotValidMultipartIdentifier = errors.New("Not a valid multipart identifier") errGCSNotValidMultipartIdentifier = errors.New("Not a valid multipart identifier")
) )

@ -33,7 +33,6 @@ import (
"google.golang.org/api/googleapi" "google.golang.org/api/googleapi"
"google.golang.org/api/iterator" "google.golang.org/api/iterator"
"github.com/minio/cli"
minio "github.com/minio/minio-go" minio "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/policy"
) )
@ -175,29 +174,20 @@ type gcsGateway struct {
ctx context.Context ctx context.Context
} }
// newGCSGateway returns gcs gatewaylayer const googleStorageEndpoint = "storage.googleapis.com"
func newGCSGateway(args cli.Args) (GatewayLayer, error) {
if !args.Present() {
return nil, fmt.Errorf("ProjectID expected")
}
endpoint := "storage.googleapis.com"
secure := true
projectID := args.First()
if !isValidGCSProjectID(projectID) {
fatalIf(errGCSInvalidProjectID, "Unable to initialize GCS gateway")
}
// newGCSGateway returns gcs gatewaylayer
func newGCSGateway(projectID string) (GatewayLayer, error) {
ctx := context.Background() ctx := context.Background()
// Creates a client. // Initialize a GCS client.
client, err := storage.NewClient(ctx) client, err := storage.NewClient(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
anonClient, err := minio.NewCore(endpoint, "", "", secure) // Initialize a anonymous client with minio core APIs.
anonClient, err := minio.NewCore(googleStorageEndpoint, "", "", true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -213,7 +203,6 @@ func newGCSGateway(args cli.Args) (GatewayLayer, error) {
// Shutdown - save any gateway metadata to disk // Shutdown - save any gateway metadata to disk
// if necessary and reload upon next restart. // if necessary and reload upon next restart.
func (l *gcsGateway) Shutdown() error { func (l *gcsGateway) Shutdown() error {
// TODO
return nil return nil
} }

@ -19,12 +19,12 @@ package cmd
import ( import (
"io" "io"
"io/ioutil" "io/ioutil"
"net"
"net/http" "net/http"
"strconv" "strconv"
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"encoding/xml"
router "github.com/gorilla/mux" router "github.com/gorilla/mux"
"github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/policy"
@ -140,7 +140,7 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
} }
// Reads the object at startOffset and writes to mw. // Reads the object at startOffset and writes to mw.
if err := getObject(bucket, object, startOffset, length, writer); err != nil { if err = getObject(bucket, object, startOffset, length, writer); err != nil {
errorIf(err, "Unable to write to client.") errorIf(err, "Unable to write to client.")
if !dataWritten { if !dataWritten {
// Error response only if no data has been written to client yet. i.e if // Error response only if no data has been written to client yet. i.e if
@ -157,6 +157,23 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
// call wrter.Write(nil) to set appropriate headers. // call wrter.Write(nil) to set appropriate headers.
writer.Write(nil) writer.Write(nil)
} }
// Get host and port from Request.RemoteAddr.
host, port, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host, port = "", ""
}
// Notify object accessed via a GET request.
eventNotify(eventData{
Type: ObjectAccessedGet,
Bucket: bucket,
ObjInfo: objInfo,
ReqParams: extractReqParams(r),
UserAgent: r.UserAgent(),
Host: host,
Port: port,
})
} }
// PutObjectHandler - PUT Object // PutObjectHandler - PUT Object
@ -293,6 +310,23 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
w.Header().Set("ETag", "\""+objInfo.ETag+"\"") w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
writeSuccessResponseHeadersOnly(w) writeSuccessResponseHeadersOnly(w)
// Get host and port from Request.RemoteAddr.
host, port, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host, port = "", ""
}
// Notify object created event.
eventNotify(eventData{
Type: ObjectCreatedPut,
Bucket: bucket,
ObjInfo: objInfo,
ReqParams: extractReqParams(r),
UserAgent: r.UserAgent(),
Host: host,
Port: port,
})
} }
// HeadObjectHandler - HEAD Object // HeadObjectHandler - HEAD Object
@ -361,97 +395,23 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
// Successful response. // Successful response.
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
}
// DeleteMultipleObjectsHandler - deletes multiple objects.
func (api gatewayAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
vars := router.Vars(r)
bucket := vars["bucket"]
objectAPI := api.ObjectAPI()
if objectAPI == nil {
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
return
}
if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone {
writeErrorResponse(w, s3Error, r.URL)
return
}
// Content-Length is required and should be non-zero
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
if r.ContentLength <= 0 {
writeErrorResponse(w, ErrMissingContentLength, r.URL)
return
}
// Content-Md5 is requied should be set // Get host and port from Request.RemoteAddr.
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html host, port, err := net.SplitHostPort(r.RemoteAddr)
if _, ok := r.Header["Content-Md5"]; !ok { if err != nil {
writeErrorResponse(w, ErrMissingContentMD5, r.URL) host, port = "", ""
return }
}
// Notify object accessed via a HEAD request.
// Allocate incoming content length bytes. eventNotify(eventData{
deleteXMLBytes := make([]byte, r.ContentLength) Type: ObjectAccessedHead,
Bucket: bucket,
// Read incoming body XML bytes. ObjInfo: objInfo,
if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil { ReqParams: extractReqParams(r),
errorIf(err, "Unable to read HTTP body.") UserAgent: r.UserAgent(),
writeErrorResponse(w, ErrInternalError, r.URL) Host: host,
return Port: port,
} })
// Unmarshal list of keys to be deleted.
deleteObjects := &DeleteObjectsRequest{}
if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
errorIf(err, "Unable to unmarshal delete objects request XML.")
writeErrorResponse(w, ErrMalformedXML, r.URL)
return
}
var dErrs = make([]error, len(deleteObjects.Objects))
// Delete all requested objects in parallel.
for index, object := range deleteObjects.Objects {
dErr := objectAPI.DeleteObject(bucket, object.ObjectName)
if dErr != nil {
dErrs[index] = dErr
}
}
// Collect deleted objects and errors if any.
var deletedObjects []ObjectIdentifier
var deleteErrors []DeleteError
for index, err := range dErrs {
object := deleteObjects.Objects[index]
// Success deleted objects are collected separately.
if err == nil {
deletedObjects = append(deletedObjects, object)
continue
}
if _, ok := errorCause(err).(ObjectNotFound); ok {
// If the object is not found it should be
// accounted as deleted as per S3 spec.
deletedObjects = append(deletedObjects, object)
continue
}
errorIf(err, "Unable to delete object. %s", object.ObjectName)
// Error during delete should be collected separately.
deleteErrors = append(deleteErrors, DeleteError{
Code: errorCodeResponse[toAPIErrorCode(err)].Code,
Message: errorCodeResponse[toAPIErrorCode(err)].Description,
Key: object.ObjectName,
})
}
// Generate response
response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
encodedSuccessResponse := encodeResponse(response)
// Write success response.
writeSuccessResponseXML(w, encodedSuccessResponse)
} }
// PutBucketPolicyHandler - PUT Bucket policy // PutBucketPolicyHandler - PUT Bucket policy

@ -20,7 +20,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"net/url" "net/url"
"os"
"runtime" "runtime"
"strings" "strings"
@ -60,20 +59,6 @@ EXAMPLES:
$ {{.HelpName}} https://azure.example.com $ {{.HelpName}} https://azure.example.com
` `
var azureBackendCmd = cli.Command{
Name: "azure",
Usage: "Microsoft Azure Blob Storage.",
Action: azureGatewayMain,
CustomHelpTemplate: azureGatewayTemplate,
Flags: append(serverFlags,
cli.BoolFlag{
Name: "quiet",
Usage: "Disable startup banner.",
},
),
HideHelpCommand: true,
}
const s3GatewayTemplate = `NAME: const s3GatewayTemplate = `NAME:
{{.HelpName}} - {{.Usage}} {{.HelpName}} - {{.Usage}}
@ -134,40 +119,41 @@ EXAMPLES:
` `
var s3BackendCmd = cli.Command{ var (
Name: "s3", azureBackendCmd = cli.Command{
Usage: "Amazon Simple Storage Service (S3).", Name: "azure",
Action: s3GatewayMain, Usage: "Microsoft Azure Blob Storage.",
CustomHelpTemplate: s3GatewayTemplate, Action: azureGatewayMain,
Flags: append(serverFlags, CustomHelpTemplate: azureGatewayTemplate,
cli.BoolFlag{ Flags: append(serverFlags, globalFlags...),
Name: "quiet", HideHelpCommand: true,
Usage: "Disable startup banner.", }
},
),
HideHelpCommand: true,
}
var gcsBackendCmd = cli.Command{ s3BackendCmd = cli.Command{
Name: "gcs", Name: "s3",
Usage: "Google Cloud Storage.", Usage: "Amazon Simple Storage Service (S3).",
Action: gcsGatewayMain, Action: s3GatewayMain,
CustomHelpTemplate: gcsGatewayTemplate, CustomHelpTemplate: s3GatewayTemplate,
Flags: append(serverFlags, Flags: append(serverFlags, globalFlags...),
cli.BoolFlag{ HideHelpCommand: true,
Name: "quiet", }
Usage: "Disable startup banner.", gcsBackendCmd = cli.Command{
}, Name: "gcs",
), Usage: "Google Cloud Storage.",
HideHelpCommand: true, Action: gcsGatewayMain,
} CustomHelpTemplate: gcsGatewayTemplate,
Flags: append(serverFlags, globalFlags...),
HideHelpCommand: true,
}
var gatewayCmd = cli.Command{ gatewayCmd = cli.Command{
Name: "gateway", Name: "gateway",
Usage: "Start object storage gateway.", Usage: "Start object storage gateway.",
HideHelpCommand: true, Flags: append(serverFlags, globalFlags...),
Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd}, HideHelpCommand: true,
} Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd},
}
)
// Represents the type of the gateway backend. // Represents the type of the gateway backend.
type gatewayBackend string type gatewayBackend string
@ -179,38 +165,6 @@ const (
// Add more backends here. // Add more backends here.
) )
// Returns access and secretkey set from environment variables.
func mustGetGatewayConfigFromEnv() (string, string, string) {
// Fetch access keys from environment variables.
accessKey := os.Getenv("MINIO_ACCESS_KEY")
secretKey := os.Getenv("MINIO_SECRET_KEY")
if accessKey == "" || secretKey == "" {
fatalIf(errors.New("Missing credentials"), "Access and secret keys are mandatory to run Minio gateway server.")
}
region := globalMinioDefaultRegion
if v := os.Getenv("MINIO_REGION"); v != "" {
region = v
}
return accessKey, secretKey, region
}
// Set browser setting from environment variables
func mustSetBrowserSettingFromEnv() {
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
browserFlag, err := ParseBrowserFlag(browser)
if err != nil {
fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
}
// browser Envs are set globally, this does not represent
// if browser is turned off or on.
globalIsEnvBrowser = true
globalIsBrowserEnabled = bool(browserFlag)
}
}
// Initialize gateway layer depending on the backend type. // Initialize gateway layer depending on the backend type.
// Supported backend types are // Supported backend types are
// //
@ -218,46 +172,19 @@ func mustSetBrowserSettingFromEnv() {
// - AWS S3. // - AWS S3.
// - Google Cloud Storage. // - Google Cloud Storage.
// - Add your favorite backend here. // - Add your favorite backend here.
func newGatewayLayer(backendType string, args cli.Args) (GatewayLayer, error) { func newGatewayLayer(backendType gatewayBackend, arg string) (GatewayLayer, error) {
switch gatewayBackend(backendType) { switch backendType {
case azureBackend: case azureBackend:
return newAzureLayer(args) return newAzureLayer(arg)
case s3Backend: case s3Backend:
return newS3Gateway(args) return newS3Gateway(arg)
case gcsBackend: case gcsBackend:
return newGCSGateway(args) return newGCSGateway(arg)
} }
return nil, fmt.Errorf("Unrecognized backend type %s", backendType) return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
} }
// Initialize a new gateway config.
//
// DO NOT save this config, this is meant to be
// only used in memory.
func newGatewayConfig(accessKey, secretKey, region string) error {
// Initialize server config.
srvCfg := newServerConfigV19()
// If env is set for a fresh start, save them to config file.
srvCfg.SetCredential(credential{
AccessKey: accessKey,
SecretKey: secretKey,
})
// Set custom region.
srvCfg.SetRegion(region)
// hold the mutex lock before a new config is assigned.
// Save the new config globally.
// unlock the mutex.
serverConfigMu.Lock()
serverConfig = srvCfg
serverConfigMu.Unlock()
return nil
}
// Return endpoint. // Return endpoint.
func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) { func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) {
schemeSpecified := len(strings.Split(arg, "://")) > 1 schemeSpecified := len(strings.Split(arg, "://")) > 1
@ -317,6 +244,9 @@ func azureGatewayMain(ctx *cli.Context) {
cli.ShowCommandHelpAndExit(ctx, "azure", 1) cli.ShowCommandHelpAndExit(ctx, "azure", 1)
} }
// Validate gateway arguments.
fatalIf(validateGatewayArguments(ctx.String("address"), ctx.Args().First()), "Invalid argument")
gatewayMain(ctx, azureBackend) gatewayMain(ctx, azureBackend)
} }
@ -326,13 +256,21 @@ func s3GatewayMain(ctx *cli.Context) {
cli.ShowCommandHelpAndExit(ctx, "s3", 1) cli.ShowCommandHelpAndExit(ctx, "s3", 1)
} }
// Validate gateway arguments.
fatalIf(validateGatewayArguments(ctx.String("address"), ctx.Args().First()), "Invalid argument")
gatewayMain(ctx, s3Backend) gatewayMain(ctx, s3Backend)
} }
// Handler for 'minio gateway gcs' command line // Handler for 'minio gateway gcs' command line
func gcsGatewayMain(ctx *cli.Context) { func gcsGatewayMain(ctx *cli.Context) {
if ctx.Args().Present() && ctx.Args().First() == "help" { if ctx.Args().Present() && ctx.Args().First() == "help" {
cli.ShowCommandHelpAndExit(ctx, "s3", 1) cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
}
if !isValidGCSProjectID(ctx.Args().First()) {
errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
} }
gatewayMain(ctx, gcsBackend) gatewayMain(ctx, gcsBackend)
@ -340,43 +278,45 @@ func gcsGatewayMain(ctx *cli.Context) {
// Handler for 'minio gateway'. // Handler for 'minio gateway'.
func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
// Fetch access and secret key from env.
accessKey, secretKey, region := mustGetGatewayConfigFromEnv()
// Fetch browser env setting
mustSetBrowserSettingFromEnv()
// Initialize new gateway config.
newGatewayConfig(accessKey, secretKey, region)
// Get quiet flag from command line argument. // Get quiet flag from command line argument.
quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet") quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
if quietFlag { if quietFlag {
log.EnableQuiet() log.EnableQuiet()
} }
serverAddr := ctx.String("address") // Handle common command args.
endpointAddr := ctx.Args().First() handleCommonCmdArgs(ctx)
err := validateGatewayArguments(serverAddr, endpointAddr)
fatalIf(err, "Invalid argument") // Handle common env vars.
handleCommonEnvVars()
// Create certs path for SSL configuration. // Create certs path.
fatalIf(createConfigDir(), "Unable to create configuration directory") fatalIf(createConfigDir(), "Unable to create configuration directories.")
// Initialize gateway config.
initConfig()
// Enable loggers as per configuration file.
enableLoggers()
// Init the error tracing module.
initError()
// Check and load SSL certificates.
var err error
globalPublicCerts, globalRootCAs, globalIsSSL, err = getSSLConfig()
fatalIf(err, "Invalid SSL key file")
initNSLock(false) // Enable local namespace lock. initNSLock(false) // Enable local namespace lock.
newObject, err := newGatewayLayer(backendType, ctx.Args()[1:]) newObject, err := newGatewayLayer(backendType, ctx.Args().First())
fatalIf(err, "Unable to initialize gateway layer") fatalIf(err, "Unable to initialize gateway layer")
router := mux.NewRouter().SkipClean(true) router := mux.NewRouter().SkipClean(true)
// credentials Envs are set globally.
globalIsEnvCreds = true
// Register web router when its enabled. // Register web router when its enabled.
if globalIsBrowserEnabled { if globalIsBrowserEnabled {
aerr := registerWebRouter(router) fatalIf(registerWebRouter(router), "Unable to configure web browser")
fatalIf(aerr, "Unable to configure web browser")
} }
registerGatewayAPIRouter(router, newObject) registerGatewayAPIRouter(router, newObject)
@ -409,10 +349,7 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
} }
apiServer := NewServerMux(serverAddr, registerHandlers(router, handlerFns...)) apiServer := NewServerMux(ctx.String("address"), registerHandlers(router, handlerFns...))
_, _, globalIsSSL, err = getSSLConfig()
fatalIf(err, "Invalid SSL key file")
// Start server, automatically configures TLS if certs are available. // Start server, automatically configures TLS if certs are available.
go func() { go func() {
@ -420,9 +357,7 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
if globalIsSSL { if globalIsSSL {
cert, key = getPublicCertFile(), getPrivateKeyFile() cert, key = getPublicCertFile(), getPrivateKeyFile()
} }
fatalIf(apiServer.ListenAndServe(cert, key), "Failed to start minio server")
aerr := apiServer.ListenAndServe(cert, key)
fatalIf(aerr, "Failed to start minio server")
}() }()
// Once endpoints are finalized, initialize the new object api. // Once endpoints are finalized, initialize the new object api.
@ -441,9 +376,12 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
case s3Backend: case s3Backend:
mode = globalMinioModeGatewayS3 mode = globalMinioModeGatewayS3
} }
// Check update mode.
checkUpdate(mode) checkUpdate(mode)
apiEndpoints := getAPIEndpoints(apiServer.Addr)
printGatewayStartupMessage(apiEndpoints, accessKey, secretKey, backendType) // Print gateway startup message.
printGatewayStartupMessage(getAPIEndpoints(apiServer.Addr), backendType)
} }
<-globalServiceDoneCh <-globalServiceDoneCh

@ -17,7 +17,6 @@
package cmd package cmd
import ( import (
"os"
"strings" "strings"
"testing" "testing"
) )
@ -53,28 +52,6 @@ func TestParseGatewayEndpoint(t *testing.T) {
} }
} }
func TestSetBrowserFromEnv(t *testing.T) {
browser := os.Getenv("MINIO_BROWSER")
os.Setenv("MINIO_BROWSER", "on")
mustSetBrowserSettingFromEnv()
if globalIsBrowserEnabled != true {
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, false)
}
os.Setenv("MINIO_BROWSER", "off")
mustSetBrowserSettingFromEnv()
if globalIsBrowserEnabled != false {
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, true)
}
os.Setenv("MINIO_BROWSER", "")
mustSetBrowserSettingFromEnv()
if globalIsBrowserEnabled != false {
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, true)
}
os.Setenv("MINIO_BROWSER", browser)
}
// Test validateGatewayArguments // Test validateGatewayArguments
func TestValidateGatewayArguments(t *testing.T) { func TestValidateGatewayArguments(t *testing.T) {
nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool { nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {

@ -20,12 +20,10 @@ import (
"errors" "errors"
"io" "io"
"net/http" "net/http"
"os"
"path" "path"
"encoding/hex" "encoding/hex"
"github.com/minio/cli"
minio "github.com/minio/minio-go" minio "github.com/minio/minio-go"
"github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/policy"
) )
@ -101,31 +99,33 @@ type s3Objects struct {
} }
// newS3Gateway returns s3 gatewaylayer // newS3Gateway returns s3 gatewaylayer
func newS3Gateway(args cli.Args) (GatewayLayer, error) { func newS3Gateway(host string) (GatewayLayer, error) {
var err error var err error
var endpoint string
var secure = true
// Default endpoint parameters // Validate host parameters.
endpoint := "s3.amazonaws.com" if host != "" {
secure := true // Override default params if the host is provided
endpoint, secure, err = parseGatewayEndpoint(host)
// Check if user provided some parameters
if args.Present() {
// Override default params if the endpoint is provided
endpoint, secure, err = parseGatewayEndpoint(args.First())
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
accessKey := os.Getenv("MINIO_ACCESS_KEY") // Default endpoint parameters
secretKey := os.Getenv("MINIO_SECRET_KEY") if endpoint == "" {
if accessKey == "" || secretKey == "" { endpoint = "s3.amazonaws.com"
return nil, errors.New("No S3 access and secret key") }
creds := serverConfig.GetCredential()
if !creds.IsValid() && !globalIsEnvCreds {
return nil, errors.New("S3 backend account and secret keys should be set through ENVs")
} }
// Initialize minio client object. // Initialize minio client object.
client, err := minio.NewCore(endpoint, accessKey, secretKey, secure) client, err := minio.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
if err != nil { if err != nil {
return nil, err return nil, err
} }

@ -18,28 +18,19 @@ package cmd
import ( import (
"fmt" "fmt"
"runtime"
"strings" "strings"
) )
// Prints the formatted startup message. // Prints the formatted startup message.
func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey string, backendType gatewayBackend) { func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBackend) {
strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
// Prints credential. // Prints credential.
printGatewayCommonMsg(apiEndPoints, accessKey, secretKey) printGatewayCommonMsg(strippedAPIEndpoints)
// Prints `mc` cli configuration message chooses // Prints `mc` cli configuration message chooses
// first endpoint as default. // first endpoint as default.
endPoint := apiEndPoints[0] printCLIAccessMsg(strippedAPIEndpoints[0], fmt.Sprintf("my%s", backendType))
// Configure 'mc', following block prints platform specific information for minio client.
log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
if runtime.GOOS == globalWindowsOSName {
mcMessage := fmt.Sprintf("$ mc.exe config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey)
log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
} else {
mcMessage := fmt.Sprintf("$ mc config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey)
log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
}
// Prints documentation message. // Prints documentation message.
printObjectAPIMsg() printObjectAPIMsg()
@ -52,10 +43,16 @@ func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey stri
} }
// Prints common server startup message. Prints credential, region and browser access. // Prints common server startup message. Prints credential, region and browser access.
func printGatewayCommonMsg(apiEndpoints []string, accessKey, secretKey string) { func printGatewayCommonMsg(apiEndpoints []string) {
// Get saved credentials.
cred := serverConfig.GetCredential()
apiEndpointStr := strings.Join(apiEndpoints, " ") apiEndpointStr := strings.Join(apiEndpoints, " ")
// Colorize the message and print. // Colorize the message and print.
log.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr))) log.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", accessKey))) log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", secretKey))) log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
log.Println(colorBlue("\nBrowser Access:"))
log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
} }

@ -20,12 +20,24 @@ import "testing"
// Test printing Gateway common message. // Test printing Gateway common message.
func TestPrintGatewayCommonMessage(t *testing.T) { func TestPrintGatewayCommonMessage(t *testing.T) {
apiEndpoints := []string{"127.0.0.1:9000"} root, err := newTestConfig(globalMinioDefaultRegion)
printGatewayCommonMsg(apiEndpoints, "abcd1", "abcd123") if err != nil {
t.Fatal(err)
}
defer removeAll(root)
apiEndpoints := []string{"http://127.0.0.1:9000"}
printGatewayCommonMsg(apiEndpoints)
} }
// Test print gateway startup message. // Test print gateway startup message.
func TestPrintGatewayStartupMessage(t *testing.T) { func TestPrintGatewayStartupMessage(t *testing.T) {
apiEndpoints := []string{"127.0.0.1:9000"} root, err := newTestConfig(globalMinioDefaultRegion)
printGatewayStartupMessage(apiEndpoints, "abcd1", "abcd123", "azure") if err != nil {
t.Fatal(err)
}
defer removeAll(root)
apiEndpoints := []string{"http://127.0.0.1:9000"}
printGatewayStartupMessage(apiEndpoints, "azure")
} }

@ -17,12 +17,8 @@
package cmd package cmd
import ( import (
"errors"
"os" "os"
"path/filepath"
"runtime" "runtime"
"strings"
"time"
"github.com/minio/cli" "github.com/minio/cli"
"github.com/minio/dsync" "github.com/minio/dsync"
@ -78,61 +74,9 @@ EXAMPLES:
`, `,
} }
// Check for updates and print a notification message
func checkUpdate(mode string) {
// Its OK to ignore any errors during getUpdateInfo() here.
if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
log.Println(updateMsg)
}
}
}
func enableLoggers() {
fileLogTarget := serverConfig.Logger.GetFile()
if fileLogTarget.Enable {
err := InitFileLogger(&fileLogTarget)
fatalIf(err, "Unable to initialize file logger")
log.AddTarget(fileLogTarget)
}
consoleLogTarget := serverConfig.Logger.GetConsole()
if consoleLogTarget.Enable {
InitConsoleLogger(&consoleLogTarget)
}
log.SetConsoleTarget(consoleLogTarget)
}
func initConfig() {
// Config file does not exist, we create it fresh and return upon success.
if isFile(getConfigFile()) {
fatalIf(migrateConfig(), "Config migration failed.")
fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
} else {
fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
log.Println("Created minio configuration file successfully at " + getConfigDir())
}
}
func serverHandleCmdArgs(ctx *cli.Context) { func serverHandleCmdArgs(ctx *cli.Context) {
// Set configuration directory. // Handle common command args.
{ handleCommonCmdArgs(ctx)
// Get configuration directory from command line argument.
configDir := ctx.String("config-dir")
if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
configDir = ctx.GlobalString("config-dir")
}
if configDir == "" {
fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
}
// Disallow relative paths, figure out absolute paths.
configDirAbs, err := filepath.Abs(configDir)
fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
setConfigDir(configDirAbs)
}
// Server address. // Server address.
serverAddr := ctx.String("address") serverAddr := ctx.String("address")
@ -159,36 +103,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
} }
func serverHandleEnvVars() { func serverHandleEnvVars() {
// Start profiler if env is set. // Handle common environment variables.
if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" { handleCommonEnvVars()
globalProfiler = startProfiler(profiler)
}
// Check if object cache is disabled.
globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
accessKey := os.Getenv("MINIO_ACCESS_KEY")
secretKey := os.Getenv("MINIO_SECRET_KEY")
if accessKey != "" && secretKey != "" {
cred, err := createCredential(accessKey, secretKey)
fatalIf(err, "Invalid access/secret Key set in environment.")
// credential Envs are set globally.
globalIsEnvCreds = true
globalActiveCred = cred
}
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
browserFlag, err := ParseBrowserFlag(browser)
if err != nil {
fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
}
// browser Envs are set globally, this does not represent
// if browser is turned off or on.
globalIsEnvBrowser = true
globalIsBrowserEnabled = bool(browserFlag)
}
if serverRegion := os.Getenv("MINIO_REGION"); serverRegion != "" { if serverRegion := os.Getenv("MINIO_REGION"); serverRegion != "" {
// region Envs are set globally. // region Envs are set globally.
@ -210,12 +126,16 @@ func serverMain(ctx *cli.Context) {
log.EnableQuiet() log.EnableQuiet()
} }
// Handle all server command args.
serverHandleCmdArgs(ctx) serverHandleCmdArgs(ctx)
// Handle all server environment vars.
serverHandleEnvVars() serverHandleEnvVars()
// Create certs path. // Create certs path.
fatalIf(createConfigDir(), "Unable to create configuration directories.") fatalIf(createConfigDir(), "Unable to create configuration directories.")
// Initialize server config.
initConfig() initConfig()
// Enable loggers as per configuration file. // Enable loggers as per configuration file.

@ -52,7 +52,7 @@ func printStartupMessage(apiEndPoints []string) {
// Prints `mc` cli configuration message chooses // Prints `mc` cli configuration message chooses
// first endpoint as default. // first endpoint as default.
printCLIAccessMsg(strippedAPIEndpoints[0]) printCLIAccessMsg(strippedAPIEndpoints[0], "myminio")
// Prints documentation message. // Prints documentation message.
printObjectAPIMsg() printObjectAPIMsg()
@ -141,17 +141,17 @@ func printEventNotifiers() {
// Prints startup message for command line access. Prints link to our documentation // Prints startup message for command line access. Prints link to our documentation
// and custom platform specific message. // and custom platform specific message.
func printCLIAccessMsg(endPoint string) { func printCLIAccessMsg(endPoint string, alias string) {
// Get saved credentials. // Get saved credentials.
cred := serverConfig.GetCredential() cred := serverConfig.GetCredential()
// Configure 'mc', following block prints platform specific information for minio client. // Configure 'mc', following block prints platform specific information for minio client.
log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide) log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
if runtime.GOOS == globalWindowsOSName { if runtime.GOOS == globalWindowsOSName {
mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKey, cred.SecretKey) mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
} else { } else {
mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKey, cred.SecretKey) mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
} }
} }

@ -140,7 +140,7 @@ func TestPrintCLIAccessMsg(t *testing.T) {
defer removeAll(root) defer removeAll(root)
apiEndpoints := []string{"http://127.0.0.1:9000"} apiEndpoints := []string{"http://127.0.0.1:9000"}
printCLIAccessMsg(apiEndpoints[0]) printCLIAccessMsg(apiEndpoints[0], "myminio")
} }
// Test print startup message. // Test print startup message.

Loading…
Cancel
Save