From 6400f506da80b3d988ed1854dc30f5be2fc82869 Mon Sep 17 00:00:00 2001 From: Timon Wong Date: Fri, 27 Oct 2017 17:07:46 -0500 Subject: [PATCH] Simplify gateway backend registration (#5111) --- cmd/gateway-azure.go | 79 +++++++++- cmd/gateway-b2.go | 63 +++++++- cmd/gateway-gcs.go | 92 +++++++++++- cmd/gateway-main.go | 299 +++++-------------------------------- cmd/gateway-main_test.go | 23 +++ cmd/gateway-s3.go | 77 +++++++++- cmd/gateway-startup-msg.go | 2 +- cmd/globals.go | 5 +- cmd/web-handlers.go | 16 +- 9 files changed, 357 insertions(+), 299 deletions(-) diff --git a/cmd/gateway-azure.go b/cmd/gateway-azure.go index 12eb442b5..276b93b6e 100644 --- a/cmd/gateway-azure.go +++ b/cmd/gateway-azure.go @@ -33,13 +33,86 @@ import ( "github.com/Azure/azure-sdk-for-go/storage" humanize "github.com/dustin/go-humanize" + "github.com/minio/cli" "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio/pkg/hash" ) -const globalAzureAPIVersion = "2016-05-31" -const azureBlockSize = 100 * humanize.MiByte -const metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure.json" +const ( + globalAzureAPIVersion = "2016-05-31" + azureBlockSize = 100 * humanize.MiByte + metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure.json" + azureBackend = "azure" +) + +func init() { + const azureGatewayTemplate = `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT] +{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +ENDPOINT: + Azure server endpoint. Default ENDPOINT is https://core.windows.net + +ENVIRONMENT VARIABLES: + ACCESS: + MINIO_ACCESS_KEY: Username or access key of Azure storage. + MINIO_SECRET_KEY: Password or secret key of Azure storage. + + BROWSER: + MINIO_BROWSER: To disable web browser access, set this value to "off". + +EXAMPLES: + 1. Start minio gateway server for Azure Blob Storage backend. + $ export MINIO_ACCESS_KEY=azureaccountname + $ export MINIO_SECRET_KEY=azureaccountkey + $ {{.HelpName}} + + 2. Start minio gateway server for Azure Blob Storage backend on custom endpoint. + $ export MINIO_ACCESS_KEY=azureaccountname + $ export MINIO_SECRET_KEY=azureaccountkey + $ {{.HelpName}} https://azure.example.com + +` + + MustRegisterGatewayCommand(cli.Command{ + Name: azureBackend, + Usage: "Microsoft Azure Blob Storage.", + Action: azureGatewayMain, + CustomHelpTemplate: azureGatewayTemplate, + Flags: append(serverFlags, globalFlags...), + HideHelpCommand: true, + }) +} + +// Handler for 'minio gateway azure' command line. +func azureGatewayMain(ctx *cli.Context) { + // Validate gateway arguments. + host := ctx.Args().First() + // Validate gateway arguments. + fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") + + startGateway(ctx, &AzureGateway{host}) +} + +// AzureGateway implements Gateway. +type AzureGateway struct { + host string +} + +// Name implements Gateway interface. +func (g *AzureGateway) Name() string { + return azureBackend +} + +// NewGatewayLayer initializes azure blob storage client and returns AzureObjects. +func (g *AzureGateway) NewGatewayLayer() (GatewayLayer, error) { + return newAzureLayer(g.host) +} // s3MetaToAzureProperties converts metadata meant for S3 PUT/COPY // object into Azure data structures - BlobMetadata and diff --git a/cmd/gateway-b2.go b/cmd/gateway-b2.go index d97e0e326..044650ebe 100644 --- a/cmd/gateway-b2.go +++ b/cmd/gateway-b2.go @@ -29,6 +29,7 @@ import ( "time" b2 "github.com/minio/blazer/base" + "github.com/minio/cli" "github.com/minio/minio-go/pkg/policy" h2 "github.com/minio/minio/pkg/hash" ) @@ -37,8 +38,65 @@ import ( const ( bucketTypePrivate = "allPrivate" bucketTypeReadOnly = "allPublic" + b2Backend = "b2" ) +func init() { + const b2GatewayTemplate = `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} +{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +ENVIRONMENT VARIABLES: + ACCESS: + MINIO_ACCESS_KEY: B2 account id. + MINIO_SECRET_KEY: B2 application key. + + BROWSER: + MINIO_BROWSER: To disable web browser access, set this value to "off". + +EXAMPLES: + 1. Start minio gateway server for B2 backend. + $ export MINIO_ACCESS_KEY=accountID + $ export MINIO_SECRET_KEY=applicationKey + $ {{.HelpName}} + +` + + MustRegisterGatewayCommand(cli.Command{ + Name: b2Backend, + Usage: "Backblaze B2.", + Action: b2GatewayMain, + CustomHelpTemplate: b2GatewayTemplate, + Flags: append(serverFlags, globalFlags...), + HideHelpCommand: true, + }) +} + +// Handler for 'minio gateway b2' command line. +func b2GatewayMain(ctx *cli.Context) { + startGateway(ctx, &B2Gateway{}) +} + +// B2Gateway implements Gateway. +type B2Gateway struct{} + +// Name implements Gateway interface. +func (g *B2Gateway) Name() string { + return b2Backend +} + +// NewGatewayLayer returns b2 gateway layer, implements GatewayLayer interface to +// talk to B2 remote backend. +func (g *B2Gateway) NewGatewayLayer() (GatewayLayer, error) { + log.Println(colorYellow("\n *** Warning: Not Ready for Production ***")) + return newB2GatewayLayer() +} + // b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers. type b2Objects struct { gatewayUnsupported @@ -49,9 +107,8 @@ type b2Objects struct { ctx context.Context } -// newB2Gateway returns b2 gateway layer, implements GatewayLayer interface to -// talk to B2 remote backend. -func newB2Gateway() (GatewayLayer, error) { +// newB2GatewayLayer returns b2 gateway layer. +func newB2GatewayLayer() (GatewayLayer, error) { ctx := context.Background() creds := serverConfig.GetCredential() diff --git a/cmd/gateway-gcs.go b/cmd/gateway-gcs.go index 1430ad0e3..a808d4342 100644 --- a/cmd/gateway-gcs.go +++ b/cmd/gateway-gcs.go @@ -30,16 +30,15 @@ import ( "strings" "time" - "golang.org/x/oauth2/google" - "cloud.google.com/go/storage" - cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" - "google.golang.org/api/googleapi" - "google.golang.org/api/iterator" - + "github.com/minio/cli" minio "github.com/minio/minio-go" "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio/pkg/hash" + "golang.org/x/oauth2/google" + cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" ) var ( @@ -78,8 +77,85 @@ const ( // Project ID key in credentials.json gcsProjectIDKey = "project_id" + + gcsBackend = "gcs" ) +func init() { + const gcsGatewayTemplate = `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [PROJECTID] +{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +PROJECTID: + GCS project-id should be provided if GOOGLE_APPLICATION_CREDENTIALS environmental variable is not set. + +ENVIRONMENT VARIABLES: + ACCESS: + MINIO_ACCESS_KEY: Username or access key of GCS. + MINIO_SECRET_KEY: Password or secret key of GCS. + + BROWSER: + MINIO_BROWSER: To disable web browser access, set this value to "off". + + GCS credentials file: + GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json + +EXAMPLES: + 1. Start minio gateway server for GCS backend. + $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json + (Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials) + $ export MINIO_ACCESS_KEY=accesskey + $ export MINIO_SECRET_KEY=secretkey + $ {{.HelpName}} mygcsprojectid + +` + + MustRegisterGatewayCommand(cli.Command{ + Name: gcsBackend, + Usage: "Google Cloud Storage.", + Action: gcsGatewayMain, + CustomHelpTemplate: gcsGatewayTemplate, + Flags: append(serverFlags, globalFlags...), + HideHelpCommand: true, + }) +} + +// Handler for 'minio gateway gcs' command line. +func gcsGatewayMain(ctx *cli.Context) { + projectID := ctx.Args().First() + if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + errorIf(errGCSProjectIDNotFound, "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json") + cli.ShowCommandHelpAndExit(ctx, "gcs", 1) + } + if projectID != "" && !isValidGCSProjectIDFormat(projectID) { + errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First()) + cli.ShowCommandHelpAndExit(ctx, "gcs", 1) + } + + startGateway(ctx, &GCSGateway{projectID}) +} + +// GCSGateway implements Gateway. +type GCSGateway struct { + projectID string +} + +// Name returns the name of gcs gatewaylayer. +func (g *GCSGateway) Name() string { + return gcsBackend +} + +// NewGatewayLayer returns gcs gatewaylayer. +func (g *GCSGateway) NewGatewayLayer() (GatewayLayer, error) { + log.Println(colorYellow("\n *** Warning: Not Ready for Production ***")) + return newGCSGatewayLayer(g.projectID) +} + // Stored in gcs.json - Contents of this file is not used anywhere. It can be // used for debugging purposes. type gcsMultipartMetaV1 struct { @@ -275,8 +351,8 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) { return googleCreds[gcsProjectIDKey], err } -// newGCSGateway returns gcs gatewaylayer -func newGCSGateway(projectID string) (GatewayLayer, error) { +// newGCSGatewayLayer returns gcs gatewaylayer +func newGCSGatewayLayer(projectID string) (GatewayLayer, error) { ctx := context.Background() var err error diff --git a/cmd/gateway-main.go b/cmd/gateway-main.go index efbff23c5..70f26b2b2 100644 --- a/cmd/gateway-main.go +++ b/cmd/gateway-main.go @@ -31,212 +31,41 @@ import ( miniohttp "github.com/minio/minio/pkg/http" ) -const azureGatewayTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT] -{{if .VisibleFlags}} -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}{{end}} -ENDPOINT: - Azure server endpoint. Default ENDPOINT is https://core.windows.net - -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of Azure storage. - MINIO_SECRET_KEY: Password or secret key of Azure storage. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - -EXAMPLES: - 1. Start minio gateway server for Azure Blob Storage backend. - $ export MINIO_ACCESS_KEY=azureaccountname - $ export MINIO_SECRET_KEY=azureaccountkey - $ {{.HelpName}} - - 2. Start minio gateway server for Azure Blob Storage backend on custom endpoint. - $ export MINIO_ACCESS_KEY=azureaccountname - $ export MINIO_SECRET_KEY=azureaccountkey - $ {{.HelpName}} https://azure.example.com -` - -const s3GatewayTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT] -{{if .VisibleFlags}} -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}{{end}} -ENDPOINT: - S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com - -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of S3 storage. - MINIO_SECRET_KEY: Password or secret key of S3 storage. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - -EXAMPLES: - 1. Start minio gateway server for AWS S3 backend. - $ export MINIO_ACCESS_KEY=accesskey - $ export MINIO_SECRET_KEY=secretkey - $ {{.HelpName}} - - 2. Start minio gateway server for S3 backend on custom endpoint. - $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F - $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG - $ {{.HelpName}} https://play.minio.io:9000 -` - -const gcsGatewayTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [PROJECTID] -{{if .VisibleFlags}} -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}{{end}} -PROJECTID: - GCS project-id should be provided if GOOGLE_APPLICATION_CREDENTIALS environmental variable is not set. - -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: Username or access key of GCS. - MINIO_SECRET_KEY: Password or secret key of GCS. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - - GCS credentials file: - GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json - -EXAMPLES: - 1. Start minio gateway server for GCS backend. - $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json - (Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials) - $ export MINIO_ACCESS_KEY=accesskey - $ export MINIO_SECRET_KEY=secretkey - $ {{.HelpName}} mygcsprojectid - -` - -const b2GatewayTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} -{{if .VisibleFlags}} -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}{{end}} -ENVIRONMENT VARIABLES: - ACCESS: - MINIO_ACCESS_KEY: B2 account id. - MINIO_SECRET_KEY: B2 application key. - - BROWSER: - MINIO_BROWSER: To disable web browser access, set this value to "off". - -EXAMPLES: - 1. Start minio gateway server for B2 backend. - $ export MINIO_ACCESS_KEY=accountID - $ export MINIO_SECRET_KEY=applicationKey - $ {{.HelpName}} - -` - var ( - azureBackendCmd = cli.Command{ - Name: "azure", - Usage: "Microsoft Azure Blob Storage.", - Action: azureGatewayMain, - CustomHelpTemplate: azureGatewayTemplate, - Flags: append(serverFlags, globalFlags...), - HideHelpCommand: true, - } - - s3BackendCmd = cli.Command{ - Name: "s3", - Usage: "Amazon Simple Storage Service (S3).", - Action: s3GatewayMain, - CustomHelpTemplate: s3GatewayTemplate, - Flags: append(serverFlags, globalFlags...), - HideHelpCommand: true, - } - - gcsBackendCmd = cli.Command{ - Name: "gcs", - Usage: "Google Cloud Storage.", - Action: gcsGatewayMain, - CustomHelpTemplate: gcsGatewayTemplate, - Flags: append(serverFlags, globalFlags...), - HideHelpCommand: true, - } - - b2BackendCmd = cli.Command{ - Name: "b2", - Usage: "Backblaze B2.", - Action: b2GatewayMain, - CustomHelpTemplate: b2GatewayTemplate, - Flags: append(serverFlags, globalFlags...), - HideHelpCommand: true, - } - gatewayCmd = cli.Command{ Name: "gateway", Usage: "Start object storage gateway.", Flags: append(serverFlags, globalFlags...), HideHelpCommand: true, - Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd, b2BackendCmd}, } ) -// Represents the type of the gateway backend. -type gatewayBackend string - -const ( - azureBackend gatewayBackend = "azure" - s3Backend gatewayBackend = "s3" - gcsBackend gatewayBackend = "gcs" - b2Backend gatewayBackend = "b2" - // Add more backends here. -) +// Gateway represents a gateway backend. +type Gateway interface { + // Name returns the unique name of the gateway. + Name() string + // NewGatewayLayer returns a new gateway layer. + NewGatewayLayer() (GatewayLayer, error) +} -// Initialize gateway layer depending on the backend type. -// Supported backend types are -// -// - Azure Blob Storage. -// - AWS S3. -// - Google Cloud Storage. -// - Backblaze B2. -// - Add your favorite backend here. -func newGatewayLayer(backendType gatewayBackend, arg string) (GatewayLayer, error) { - switch backendType { - case azureBackend: - return newAzureLayer(arg) - case s3Backend: - return newS3Gateway(arg) - case gcsBackend: - // FIXME: The following print command is temporary and - // will be removed when gcs is ready for production use. - log.Println(colorYellow("\n *** Warning: Not Ready for Production ***")) - return newGCSGateway(arg) - case b2Backend: - // FIXME: The following print command is temporary and - // will be removed when B2 is ready for production use. - log.Println(colorYellow("\n *** Warning: Not Ready for Production ***")) - return newB2Gateway() +// RegisterGatewayCommand registers a new command for gateway. +func RegisterGatewayCommand(cmd cli.Command) error { + // We should not have multiple subcommands with same name. + for _, c := range gatewayCmd.Subcommands { + if c.Name == cmd.Name { + return fmt.Errorf("duplicate gateway: %s", cmd.Name) + } } - return nil, fmt.Errorf("Unrecognized backend type %s", backendType) + gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd) + return nil +} + +// MustRegisterGatewayCommand is like RegisterGatewayCommand but panics instead of returning error. +func MustRegisterGatewayCommand(cmd cli.Command) { + if err := RegisterGatewayCommand(cmd); err != nil { + panic(err) + } } // Return endpoint. @@ -292,62 +121,8 @@ func validateGatewayArguments(serverAddr, endpointAddr string) error { return nil } -// Handler for 'minio gateway azure' command line. -func azureGatewayMain(ctx *cli.Context) { - if ctx.Args().Present() && ctx.Args().First() == "help" { - cli.ShowCommandHelpAndExit(ctx, "azure", 1) - } - - // Validate gateway arguments. - fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument") - - gatewayMain(ctx, azureBackend) -} - -// Handler for 'minio gateway s3' command line. -func s3GatewayMain(ctx *cli.Context) { - if ctx.Args().Present() && ctx.Args().First() == "help" { - cli.ShowCommandHelpAndExit(ctx, "s3", 1) - } - - // Validate gateway arguments. - fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument") - - gatewayMain(ctx, s3Backend) -} - -// Handler for 'minio gateway gcs' command line -func gcsGatewayMain(ctx *cli.Context) { - if ctx.Args().Present() && ctx.Args().First() == "help" { - cli.ShowCommandHelpAndExit(ctx, "gcs", 1) - } - - projectID := ctx.Args().First() - if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { - errorIf(errGCSProjectIDNotFound, "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json") - cli.ShowCommandHelpAndExit(ctx, "gcs", 1) - } - if projectID != "" && !isValidGCSProjectIDFormat(projectID) { - errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First()) - cli.ShowCommandHelpAndExit(ctx, "gcs", 1) - } - - gatewayMain(ctx, gcsBackend) -} - -func b2GatewayMain(ctx *cli.Context) { - if ctx.Args().Present() && ctx.Args().First() == "help" { - cli.ShowCommandHelpAndExit(ctx, "b2", 1) - } - - // Validate gateway arguments. - fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument") - - gatewayMain(ctx, b2Backend) -} - -// Handler for 'minio gateway'. -func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { +// Handler for 'minio gateway '. +func startGateway(ctx *cli.Context, gw Gateway) { // Get quiet flag from command line argument. quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet") if quietFlag { @@ -367,8 +142,9 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { handleCommonEnvVars() // Validate if we have access, secret set through environment. + gatewayName := gw.Name() if !globalIsEnvCreds { - fatalIf(fmt.Errorf("Access and Secret keys should be set through ENVs for backend [%s]", backendType), "") + fatalIf(fmt.Errorf("Access and Secret keys should be set through ENVs for backend [%s]", gatewayName), "") } // Create certs path. @@ -390,7 +166,11 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { initNSLock(false) // Enable local namespace lock. - newObject, err := newGatewayLayer(backendType, ctx.Args().First()) + if ctx.Args().First() == "help" { + cli.ShowCommandHelpAndExit(ctx, gatewayName, 1) + } + + newObject, err := gw.NewGatewayLayer() fatalIf(err, "Unable to initialize gateway layer") router := mux.NewRouter().SkipClean(true) @@ -445,23 +225,12 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) { // Prints the formatted startup message once object layer is initialized. if !quietFlag { - mode := "" - switch gatewayBackend(backendType) { - case azureBackend: - mode = globalMinioModeGatewayAzure - case gcsBackend: - mode = globalMinioModeGatewayGCS - case s3Backend: - mode = globalMinioModeGatewayS3 - case b2Backend: - mode = globalMinioModeGatewayB2 - } - + mode := globalMinioModeGatewayPrefix + gatewayName // Check update mode. checkUpdate(mode) // Print gateway startup message. - printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), backendType) + printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName) } handleSignals() diff --git a/cmd/gateway-main_test.go b/cmd/gateway-main_test.go index fdab9580f..a13cbd913 100644 --- a/cmd/gateway-main_test.go +++ b/cmd/gateway-main_test.go @@ -19,8 +19,31 @@ package cmd import ( "strings" "testing" + + "github.com/minio/cli" ) +// Test RegisterGatewayCommand +func TestRegisterGatewayCommand(t *testing.T) { + var err error + + cmd := cli.Command{Name: "test"} + err = RegisterGatewayCommand(cmd) + if err != nil { + t.Errorf("RegisterGatewayCommand got unexpected error: %s", err) + } + + // Should returns 'duplicated' error + err = RegisterGatewayCommand(cmd) + if err == nil { + t.Errorf("RegisterGatewayCommand twice with same name should return error") + } else { + if err.Error() != "duplicate gateway: test" { + t.Errorf("RegisterGatewayCommand got unexpected error: %s", err) + } + } +} + // Test parseGatewayEndpoint func TestParseGatewayEndpoint(t *testing.T) { testCases := []struct { diff --git a/cmd/gateway-s3.go b/cmd/gateway-s3.go index d5c4d658c..845aa5ac2 100644 --- a/cmd/gateway-s3.go +++ b/cmd/gateway-s3.go @@ -20,12 +20,85 @@ import ( "io" "net/http" + "github.com/minio/cli" minio "github.com/minio/minio-go" "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/s3utils" "github.com/minio/minio/pkg/hash" ) +const ( + s3Backend = "s3" +) + +func init() { + const s3GatewayTemplate = `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT] +{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +ENDPOINT: + S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com + +ENVIRONMENT VARIABLES: + ACCESS: + MINIO_ACCESS_KEY: Username or access key of S3 storage. + MINIO_SECRET_KEY: Password or secret key of S3 storage. + + BROWSER: + MINIO_BROWSER: To disable web browser access, set this value to "off". + +EXAMPLES: + 1. Start minio gateway server for AWS S3 backend. + $ export MINIO_ACCESS_KEY=accesskey + $ export MINIO_SECRET_KEY=secretkey + $ {{.HelpName}} + + 2. Start minio gateway server for S3 backend on custom endpoint. + $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F + $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG + $ {{.HelpName}} https://play.minio.io:9000 +` + + MustRegisterGatewayCommand(cli.Command{ + Name: s3Backend, + Usage: "Amazon Simple Storage Service (S3).", + Action: s3GatewayMain, + CustomHelpTemplate: s3GatewayTemplate, + Flags: append(serverFlags, globalFlags...), + HideHelpCommand: true, + }) +} + +// Handler for 'minio gateway s3' command line. +func s3GatewayMain(ctx *cli.Context) { + // Validate gateway arguments. + host := ctx.Args().First() + // Validate gateway arguments. + fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument") + + startGateway(ctx, &S3Gateway{host}) +} + +// S3Gateway implements Gateway. +type S3Gateway struct { + host string +} + +// Name implements Gateway interface. +func (g *S3Gateway) Name() string { + return s3Backend +} + +// NewGatewayLayer returns s3 gatewaylayer. +func (g *S3Gateway) NewGatewayLayer() (GatewayLayer, error) { + return newS3GatewayLayer(g.host) +} + // s3ToObjectError converts Minio errors to minio object layer errors. func s3ToObjectError(err error, params ...string) error { if err == nil { @@ -101,8 +174,8 @@ type s3Objects struct { anonClient *minio.Core } -// newS3Gateway returns s3 gatewaylayer -func newS3Gateway(host string) (GatewayLayer, error) { +// newS3GatewayLayer returns s3 gatewaylayer +func newS3GatewayLayer(host string) (GatewayLayer, error) { var err error var endpoint string var secure = true diff --git a/cmd/gateway-startup-msg.go b/cmd/gateway-startup-msg.go index e17a3e4d5..4ade58d32 100644 --- a/cmd/gateway-startup-msg.go +++ b/cmd/gateway-startup-msg.go @@ -22,7 +22,7 @@ import ( ) // Prints the formatted startup message. -func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBackend) { +func printGatewayStartupMessage(apiEndPoints []string, backendType string) { strippedAPIEndpoints := stripStandardPorts(apiEndPoints) // Prints credential. diff --git a/cmd/globals.go b/cmd/globals.go index af544fe88..08a8d8316 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -50,10 +50,7 @@ const ( globalMinioModeFS = "mode-server-fs" globalMinioModeXL = "mode-server-xl" globalMinioModeDistXL = "mode-server-distributed-xl" - globalMinioModeGatewayAzure = "mode-gateway-azure" - globalMinioModeGatewayS3 = "mode-gateway-s3" - globalMinioModeGatewayGCS = "mode-gateway-gcs" - globalMinioModeGatewayB2 = "mode-gateway-b2" + globalMinioModeGatewayPrefix = "mode-gateway-" // globalMinioSysTmp prefix is used in Azure/GCS gateway for save metadata sent by Initialize Multipart Upload API. globalMinioSysTmp = "minio.sys.tmp/" diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index c6d654e77..c4e7d987b 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -728,20 +728,10 @@ func readBucketAccessPolicy(objAPI ObjectLayer, bucketName string) (policy.Bucke func getBucketAccessPolicy(objAPI ObjectLayer, bucketName string) (policy.BucketAccessPolicy, error) { // FIXME: remove this code when S3 layer for gateway and server is unified. - var policyInfo policy.BucketAccessPolicy - var err error - - switch layer := objAPI.(type) { - case *s3Objects: - policyInfo, err = layer.GetBucketPolicies(bucketName) - case *azureObjects: - policyInfo, err = layer.GetBucketPolicies(bucketName) - case *gcsGateway: - policyInfo, err = layer.GetBucketPolicies(bucketName) - default: - policyInfo, err = readBucketAccessPolicy(objAPI, bucketName) + if layer, ok := objAPI.(GatewayLayer); ok { + return layer.GetBucketPolicies(bucketName) } - return policyInfo, err + return readBucketAccessPolicy(objAPI, bucketName) } // GetBucketPolicy - get bucket policy for the requested prefix.