From 716316f71192a8955db7045bfd7724cbbf09d615 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 11 Nov 2016 16:36:07 -0800 Subject: [PATCH] Reduce number of envs and options from command line. (#3230) Ref #3229 After review with @abperiasamy we decided to remove all the unnecessary options - MINIO_BROWSER (Implemented as a security feature but now deemed obsolete since even if blocking access to MINIO_BROWSER, s3 API port is open) - MINIO_CACHE_EXPIRY (Defaults to 72h) - MINIO_MAXCONN (No one used this option and we don't test this) - MINIO_ENABLE_FSMETA (Enable FSMETA all the time) Remove --ignore-disks option - this option was implemented when XL layer would initialize the backend disks and heal them automatically to disallow XL accidentally using the root partition itself this option was introduced. This behavior has been changed XL no longer automatically initializes `format.json` a HEAL is controlled activity, so ignore-disks is not useful anymore. This change also addresses the problems of our documentation going forward and keeps things simple. This patch brings in reduction of options and defaulting them to a valid known inputs. This patch also serves as a guideline of limiting many ways to do the same thing. --- cmd/api-router.go | 15 +- cmd/benchmark-utils_test.go | 2 +- cmd/erasure-readfile_test.go | 2 +- cmd/event-notifier_test.go | 8 +- cmd/format-config-v1_test.go | 38 +- cmd/fs-v1-metadata.go | 31 -- cmd/fs-v1-metadata_test.go | 46 +- cmd/fs-v1-multipart.go | 26 +- cmd/fs-v1.go | 18 +- cmd/fs-v1_test.go | 18 +- cmd/generic-handlers.go | 51 +- cmd/globals.go | 5 +- cmd/lock-rpc-server.go | 5 - cmd/lock-rpc-server_test.go | 9 +- cmd/main.go | 4 +- cmd/object-api-listobjects_test.go | 3 +- cmd/prepare-storage-msg.go | 6 +- cmd/prepare-storage.go | 15 +- cmd/rate-limit-handler.go | 100 ---- cmd/rate-limit-handler_test.go | 83 ---- cmd/routers.go | 22 +- cmd/server-main.go | 77 +-- cmd/server-main_test.go | 5 +- cmd/server-startup-msg.go | 19 +- cmd/storage-rpc-server.go | 4 - cmd/test-utils_test.go | 18 +- cmd/utils.go | 11 +- cmd/utils_test.go | 7 + cmd/xl-v1.go | 7 +- cmd/xl-v1_test.go | 15 +- docs/caching/README.md | 36 +- docs/minio-env-variables.md | 47 -- vendor/github.com/urfave/negroni/CHANGELOG.md | 35 -- vendor/github.com/urfave/negroni/LICENSE | 21 - vendor/github.com/urfave/negroni/README.md | 443 ------------------ vendor/github.com/urfave/negroni/doc.go | 25 - vendor/github.com/urfave/negroni/logger.go | 35 -- vendor/github.com/urfave/negroni/negroni.go | 133 ------ vendor/github.com/urfave/negroni/recovery.go | 65 --- .../urfave/negroni/response_writer.go | 99 ---- vendor/github.com/urfave/negroni/static.go | 88 ---- vendor/vendor.json | 6 - 42 files changed, 166 insertions(+), 1537 deletions(-) delete mode 100644 cmd/rate-limit-handler.go delete mode 100644 cmd/rate-limit-handler_test.go delete mode 100644 docs/minio-env-variables.md delete mode 100644 vendor/github.com/urfave/negroni/CHANGELOG.md delete mode 100644 vendor/github.com/urfave/negroni/LICENSE delete mode 100644 vendor/github.com/urfave/negroni/README.md delete mode 100644 vendor/github.com/urfave/negroni/doc.go delete mode 100644 vendor/github.com/urfave/negroni/logger.go delete mode 100644 vendor/github.com/urfave/negroni/negroni.go delete mode 100644 vendor/github.com/urfave/negroni/recovery.go delete mode 100644 vendor/github.com/urfave/negroni/response_writer.go delete mode 100644 vendor/github.com/urfave/negroni/static.go diff --git a/cmd/api-router.go b/cmd/api-router.go index df4ee0110..d92aaaa06 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -16,10 +16,7 @@ package cmd -import ( - router "github.com/gorilla/mux" - "github.com/urfave/negroni" -) +import router "github.com/gorilla/mux" // objectAPIHandler implements and provides http handlers for S3 API. type objectAPIHandlers struct { @@ -34,7 +31,7 @@ func registerAPIRouter(mux *router.Router) { } // API Router - apiRouter := mux.NewRoute().PathPrefix("").Subrouter() + apiRouter := mux.NewRoute().PathPrefix("/").Subrouter() // Bucket router bucket := apiRouter.PathPrefix("/{bucket}").Subrouter() @@ -99,12 +96,4 @@ func registerAPIRouter(mux *router.Router) { // ListBuckets apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler) - - mux.PathPrefix("/").Handler(negroni.New( - // Validates all incoming requests to have a valid date header. - negroni.Wrap(timeValidityHandler{}), - // Route requests - negroni.Wrap(apiRouter), - )) - } diff --git a/cmd/benchmark-utils_test.go b/cmd/benchmark-utils_test.go index bf05a2489..c2b071504 100644 --- a/cmd/benchmark-utils_test.go +++ b/cmd/benchmark-utils_test.go @@ -51,7 +51,7 @@ func prepareBenchmarkBackend(instanceType string) (ObjectLayer, []string, error) return nil, nil, err } // initialize object layer. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { return nil, nil, err } diff --git a/cmd/erasure-readfile_test.go b/cmd/erasure-readfile_test.go index 8a2e4e3cd..7de1a99af 100644 --- a/cmd/erasure-readfile_test.go +++ b/cmd/erasure-readfile_test.go @@ -226,7 +226,7 @@ func TestErasureReadUtils(t *testing.T) { if err != nil { t.Fatal(err) } - objLayer, _, err := initObjectLayer(endpoints, nil) + objLayer, _, err := initObjectLayer(endpoints) if err != nil { removeRoots(disks) t.Fatal(err) diff --git a/cmd/event-notifier_test.go b/cmd/event-notifier_test.go index 0a65e105c..a662e439e 100644 --- a/cmd/event-notifier_test.go +++ b/cmd/event-notifier_test.go @@ -43,7 +43,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) { if err != nil { t.Fatal(err) } - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } @@ -97,7 +97,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) { if err != nil { t.Fatal(err) } - fs, _, err := initObjectLayer(endpoints, nil) + fs, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } @@ -128,7 +128,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) { if err != nil { t.Fatal(err) } - fs, _, err := initObjectLayer(endpoints, nil) + fs, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } @@ -159,7 +159,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) { if err != nil { t.Fatal(err) } - fs, _, err := initObjectLayer(endpoints, nil) + fs, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } diff --git a/cmd/format-config-v1_test.go b/cmd/format-config-v1_test.go index c5689354c..0790e1cee 100644 --- a/cmd/format-config-v1_test.go +++ b/cmd/format-config-v1_test.go @@ -280,7 +280,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Error(err) } @@ -316,7 +316,7 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Error(err) } @@ -605,7 +605,7 @@ func TestInitFormatXLErrors(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -711,7 +711,7 @@ func TestLoadFormatXLErrs(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -741,7 +741,7 @@ func TestLoadFormatXLErrs(t *testing.T) { if err != nil { t.Fatal(err) } - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -769,7 +769,7 @@ func TestLoadFormatXLErrs(t *testing.T) { if err != nil { t.Fatal(err) } - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -795,7 +795,7 @@ func TestLoadFormatXLErrs(t *testing.T) { if err != nil { t.Fatal(err) } - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -824,7 +824,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Everything is fine, should return nil - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -846,7 +846,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Disks 0..15 are nil - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -870,7 +870,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // One disk returns Faulty Disk - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -896,7 +896,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // One disk is not found, heal corrupted disks should return nil - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -918,7 +918,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -944,7 +944,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Corrupted format json in one disk - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -974,7 +974,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Everything is fine, should return nil - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -995,7 +995,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Disks 0..15 are nil - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -1019,7 +1019,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // One disk returns Faulty Disk - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -1045,7 +1045,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // One disk is not found, heal corrupted disks should return nil - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -1067,7 +1067,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -1093,7 +1093,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index a7503a0c8..cd64e4249 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -18,10 +18,8 @@ package cmd import ( "encoding/json" - "os" "path" "sort" - "strings" ) const ( @@ -164,32 +162,3 @@ func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) b } return true } - -var extendedHeaders = []string{ - "X-Amz-Meta-", - "X-Minio-Meta-", - // Add new extended headers. -} - -// isExtendedHeader validates if input string matches extended headers. -func isExtendedHeader(header string) bool { - for _, extendedHeader := range extendedHeaders { - if strings.HasPrefix(header, extendedHeader) { - return true - } - } - return false -} - -// Return true if extended HTTP headers are set, false otherwise. -func hasExtendedHeader(metadata map[string]string) bool { - if os.Getenv("MINIO_ENABLE_FSMETA") == "1" { - return true - } - for k := range metadata { - if isExtendedHeader(k) { - return true - } - } - return false -} diff --git a/cmd/fs-v1-metadata_test.go b/cmd/fs-v1-metadata_test.go index df404746a..e7d71c91c 100644 --- a/cmd/fs-v1-metadata_test.go +++ b/cmd/fs-v1-metadata_test.go @@ -23,56 +23,12 @@ import ( "testing" ) -// Tests scenarios which can occur for hasExtendedHeader function. -func TestHasExtendedHeader(t *testing.T) { - // All test cases concerning hasExtendedHeader function. - testCases := []struct { - metadata map[string]string - has bool - }{ - // Verifies if X-Amz-Meta is present. - { - metadata: map[string]string{ - "X-Amz-Meta-1": "value", - }, - has: true || os.Getenv("MINIO_ENABLE_FSMETA") == "1", - }, - // Verifies if X-Minio-Meta is present. - { - metadata: map[string]string{ - "X-Minio-Meta-1": "value", - }, - has: true || os.Getenv("MINIO_ENABLE_FSMETA") == "1", - }, - // Verifies if extended header is not present. - { - metadata: map[string]string{ - "md5Sum": "value", - }, - has: false || os.Getenv("MINIO_ENABLE_FSMETA") == "1", - }, - // Verifies if extended header is not present, but with an empty input. - { - metadata: nil, - has: false || os.Getenv("MINIO_ENABLE_FSMETA") == "1", - }, - } - - // Validate all test cases. - for i, testCase := range testCases { - has := hasExtendedHeader(testCase.metadata) - if has != testCase.has { - t.Fatalf("Test case %d: Expected \"%#v\", but got \"%#v\"", i+1, testCase.has, has) - } - } -} - func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) { endpoints, err := parseStorageEndpoints([]string{disk}) if err != nil { t.Fatal(err) } - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal("Unexpected err: ", err) } diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index 5523b3159..114b9b745 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -227,10 +227,8 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st // Initialize `fs.json` values. fsMeta := newFSMetaV1() - // Save additional metadata only if extended headers such as "X-Amz-Meta-" are set. - if hasExtendedHeader(meta) { - fsMeta.Meta = meta - } + // Save additional metadata. + fsMeta.Meta = meta // This lock needs to be held for any changes to the directory // contents of ".minio.sys/multipart/object/" @@ -765,18 +763,16 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload // No need to save part info, since we have concatenated all parts. fsMeta.Parts = nil - // Save additional metadata only if extended headers such as "X-Amz-Meta-" are set. - if hasExtendedHeader(fsMeta.Meta) { - if len(fsMeta.Meta) == 0 { - fsMeta.Meta = make(map[string]string) - } - fsMeta.Meta["md5Sum"] = s3MD5 + // Save additional metadata. + if len(fsMeta.Meta) == 0 { + fsMeta.Meta = make(map[string]string) + } + fsMeta.Meta["md5Sum"] = s3MD5 - fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) - // Write the metadata to a temp file and rename it to the actual location. - if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { - return "", toObjectErr(err, bucket, object) - } + fsMetaPath = path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) + // Write the metadata to a temp file and rename it to the actual location. + if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { + return "", toObjectErr(err, bucket, object) } // Cleanup all the parts if everything else has been safely committed. diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 9b1fba85d..369a5d8c3 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -455,17 +455,15 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } - // Save additional metadata only if extended headers such as "X-Amz-Meta-" are set. - if hasExtendedHeader(metadata) { - // Initialize `fs.json` values. - fsMeta := newFSMetaV1() - fsMeta.Meta = metadata - - fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) - if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { - return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) - } + // Save additional metadata. Initialize `fs.json` values. + fsMeta := newFSMetaV1() + fsMeta.Meta = metadata + + fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) + if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { + return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } + objInfo, err = fs.getObjectInfo(bucket, object) if err == nil { // If MINIO_ENABLE_FSMETA is not enabled objInfo.MD5Sum will be empty. diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 296ec4dc3..727110678 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -45,7 +45,7 @@ func TestNewFS(t *testing.T) { t.Fatal("Uexpected error: ", err) } - fsStorageDisks, err := initStorageDisks(endpoints, nil) + fsStorageDisks, err := initStorageDisks(endpoints) if err != nil { t.Fatal("Uexpected error: ", err) } @@ -55,7 +55,7 @@ func TestNewFS(t *testing.T) { t.Fatal("Uexpected error: ", err) } - xlStorageDisks, err := initStorageDisks(endpoints, nil) + xlStorageDisks, err := initStorageDisks(endpoints) if err != nil { t.Fatal("Uexpected error: ", err) } @@ -96,17 +96,22 @@ func TestNewFS(t *testing.T) { } } -// TestFSShutdown - initialize a new FS object layer then calls Shutdown -// to check returned results +// TestFSShutdown - initialize a new FS object layer then calls +// Shutdown to check returned results func TestFSShutdown(t *testing.T) { + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatal(err) + } + defer removeAll(rootPath) + bucketName := "testbucket" + objectName := "object" // Create and return an fsObject with its path in the disk prepareTest := func() (fsObjects, string) { disk := filepath.Join(os.TempDir(), "minio-"+nextSuffix()) obj := initFSObjects(disk, t) fs := obj.(fsObjects) - bucketName := "testbucket" - objectName := "object" objectContent := "12345" obj.MakeBucket(bucketName) sha256sum := "" @@ -124,6 +129,7 @@ func TestFSShutdown(t *testing.T) { // Test Shutdown with faulty disk for i := 1; i <= 5; i++ { fs, disk := prepareTest() + fs.DeleteObject(bucketName, objectName) fsStorage := fs.storage.(*posix) fs.storage = newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil) if err := fs.Shutdown(); errorCause(err) != errFaultyDisk { diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 81f3dc704..1ae9a8c80 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -18,7 +18,6 @@ package cmd import ( "net/http" - "os" "path" "regexp" "strings" @@ -77,27 +76,24 @@ func setBrowserRedirectHandler(h http.Handler) http.Handler { } func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off") { - // Re-direction handled specifically for browsers. - if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && !isRequestSignatureV4(r) { - switch r.URL.Path { - case "/", "/webrpc", "/login", "/favicon.ico": - // '/' is redirected to 'locationPrefix/' - // '/webrpc' is redirected to 'locationPrefix/webrpc' - // '/login' is redirected to 'locationPrefix/login' - location := h.locationPrefix + r.URL.Path - // Redirect to new location. - http.Redirect(w, r, location, http.StatusTemporaryRedirect) - return - case h.locationPrefix: - // locationPrefix is redirected to 'locationPrefix/' - location := h.locationPrefix + "/" - http.Redirect(w, r, location, http.StatusTemporaryRedirect) - return - } + // Re-direction handled specifically for browsers. + if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && !isRequestSignatureV4(r) { + switch r.URL.Path { + case "/", "/webrpc", "/login", "/favicon.ico": + // '/' is redirected to 'locationPrefix/' + // '/webrpc' is redirected to 'locationPrefix/webrpc' + // '/login' is redirected to 'locationPrefix/login' + location := h.locationPrefix + r.URL.Path + // Redirect to new location. + http.Redirect(w, r, location, http.StatusTemporaryRedirect) + return + case h.locationPrefix: + // locationPrefix is redirected to 'locationPrefix/' + location := h.locationPrefix + "/" + http.Redirect(w, r, location, http.StatusTemporaryRedirect) + return } } - h.handler.ServeHTTP(w, r) } @@ -187,11 +183,19 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) { return time.Time{}, ErrMissingDateHeader } -type timeValidityHandler struct{} +type timeValidityHandler struct { + handler http.Handler +} + +// setTimeValidityHandler to validate parsable time over http header +func setTimeValidityHandler(h http.Handler) http.Handler { + return timeValidityHandler{h} +} func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Verify if date headers are set, if not reject the request - if _, ok := r.Header["Authorization"]; ok { + aType := getRequestAuthType(r) + if aType != authTypeAnonymous && aType != authTypeJWT { + // Verify if date headers are set, if not reject the request amzDate, apiErr := parseAmzDateHeader(r) if apiErr != ErrNone { // All our internal APIs are sensitive towards Date @@ -208,6 +212,7 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } } + h.handler.ServeHTTP(w, r) } type resourceHandler struct { diff --git a/cmd/globals.go b/cmd/globals.go index c307b2eaa..36702d1c1 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -41,14 +41,11 @@ const ( ) var ( - globalQuiet = false // Quiet flag set via command line + globalQuiet = false // Quiet flag set via command line. globalIsDistXL = false // "Is Distributed?" flag. // Add new global flags here. - // Maximum connections handled per - // server, defaults to 0 (unlimited). - globalMaxConn = 0 // Maximum cache size. globalMaxCacheSize = uint64(maxCacheSize) // Cache expiry. diff --git a/cmd/lock-rpc-server.go b/cmd/lock-rpc-server.go index df25d0f6d..d3a8f370b 100644 --- a/cmd/lock-rpc-server.go +++ b/cmd/lock-rpc-server.go @@ -83,11 +83,6 @@ func registerDistNSLockRouter(mux *router.Router, serverConfig serverCmdConfig) // Create one lock server for every local storage rpc server. func newLockServers(srvConfig serverCmdConfig) (lockServers []*lockServer) { for _, ep := range srvConfig.endpoints { - if containsEndpoint(srvConfig.ignoredEndpoints, ep) { - // Skip initializing ignored endpoint. - continue - } - // Not local storage move to the next node. if !isLocalStorage(ep) { continue diff --git a/cmd/lock-rpc-server_test.go b/cmd/lock-rpc-server_test.go index 230f6bb02..5cc55c840 100644 --- a/cmd/lock-rpc-server_test.go +++ b/cmd/lock-rpc-server_test.go @@ -475,7 +475,7 @@ func TestLockServers(t *testing.T) { }, totalLockServers: 1, }, - // Test - 2 two servers possible, 1 ignored. + // Test - 2 two servers possible. { isDistXL: true, srvCmdConfig: serverCmdConfig{ @@ -496,13 +496,8 @@ func TestLockServers(t *testing.T) { Host: "1.1.2.2:9000", Path: "/mnt/disk4", }}, - ignoredEndpoints: []*url.URL{{ - Scheme: "http", - Host: "localhost:9000", - Path: "/mnt/disk2", - }}, }, - totalLockServers: 1, + totalLockServers: 2, }, } diff --git a/cmd/main.go b/cmd/main.go index b832363ad..65e58894d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,7 +42,7 @@ var ( }, cli.BoolFlag{ Name: "quiet", - Usage: "Suppress chatty output.", + Usage: "Disable startup information.", }, } ) @@ -218,7 +218,7 @@ func Main() { } // Start profiler if env is set. - if profiler := os.Getenv("MINIO_PROFILER"); profiler != "" { + if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" { globalProfiler = startProfiler(profiler) } diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 52b8d1fc4..84c7578e5 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -566,13 +566,12 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { // Initialize FS backend for the benchmark. func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) { - endPoints, err := parseStorageEndpoints([]string{disk}) if err != nil { t.Fatal("Unexpected err: ", err) } - obj, _, err = initObjectLayer(endPoints, nil) + obj, _, err = initObjectLayer(endPoints) if err != nil { t.Fatal("Unexpected err: ", err) } diff --git a/cmd/prepare-storage-msg.go b/cmd/prepare-storage-msg.go index 51d672913..e3bd03236 100644 --- a/cmd/prepare-storage-msg.go +++ b/cmd/prepare-storage-msg.go @@ -45,7 +45,11 @@ type printOnceFunc func(msg string) func printOnceFn() printOnceFunc { var once sync.Once return func(msg string) { - once.Do(func() { console.Println(msg) }) + once.Do(func() { + if !globalQuiet { + console.Println(msg) + } + }) } } diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 266cbdca3..d1c26891b 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -266,7 +266,7 @@ func retryFormattingDisks(firstDisk bool, endpoints []*url.URL, storageDisks []S } // Initialize storage disks based on input arguments. -func initStorageDisks(endpoints, ignoredEndpoints []*url.URL) ([]StorageAPI, error) { +func initStorageDisks(endpoints []*url.URL) ([]StorageAPI, error) { // Single disk means we will use FS backend. if len(endpoints) == 1 { if endpoints[0] == nil { @@ -284,19 +284,6 @@ func initStorageDisks(endpoints, ignoredEndpoints []*url.URL) ([]StorageAPI, err if ep == nil { return nil, errInvalidArgument } - // Check if disk is ignored. - ignored := false - for _, iep := range ignoredEndpoints { - if *ep == *iep { - ignored = true - break - } - } - if ignored { - // Set this situation as disk not found. - storageDisks[index] = nil - continue - } // Intentionally ignore disk not found errors. XL is designed // to handle these errors internally. storage, err := newStorageAPI(ep) diff --git a/cmd/rate-limit-handler.go b/cmd/rate-limit-handler.go deleted file mode 100644 index 0a159825d..000000000 --- a/cmd/rate-limit-handler.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "errors" - "net/http" -) - -var errTooManyRequests = errors.New("Too many clients in the waiting list") - -// rateLimit - represents datatype of the functionality implemented to -// limit the number of concurrent http requests. -type rateLimit struct { - handler http.Handler - workQueue chan struct{} - waitQueue chan struct{} -} - -// acquire and release implement a way to send and receive from the -// channel this is in-turn used to rate limit incoming connections in -// ServeHTTP() http.Handler method. -func (c *rateLimit) acquire() error { - // attempt to enter the waitQueue. If no slot is immediately - // available return error. - select { - case c.waitQueue <- struct{}{}: - // entered wait queue - break - default: - // no slot available for waiting - return errTooManyRequests - } - - // block attempting to enter the workQueue. If the workQueue - // is full, there can be at most cap(waitQueue) == - // 4*globalMaxConn goroutines waiting here because of the - // select above. - select { - case c.workQueue <- struct{}{}: - // entered workQueue - so remove one waiter. This step - // does not block as the waitQueue cannot be empty. - <-c.waitQueue - } - - return nil -} - -// Release one element from workQueue to serve a new client in the -// waiting list -func (c *rateLimit) release() { - <-c.workQueue -} - -// ServeHTTP is an http.Handler ServeHTTP method, implemented to rate -// limit incoming HTTP requests. -func (c *rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Acquire the connection if queue is not full, otherwise - // code path waits here until the previous case is true. - if err := c.acquire(); err != nil { - w.WriteHeader(http.StatusTooManyRequests) - return - } - - // Serves the request. - c.handler.ServeHTTP(w, r) - - // Release - c.release() -} - -// setRateLimitHandler limits the number of concurrent http requests -// based on MINIO_MAXCONN. -func setRateLimitHandler(handler http.Handler) http.Handler { - if globalMaxConn == 0 { - return handler - } // else proceed to rate limiting. - - // For max connection limit of > '0' we initialize rate limit - // handler. - return &rateLimit{ - handler: handler, - workQueue: make(chan struct{}, globalMaxConn), - waitQueue: make(chan struct{}, globalMaxConn*4), - } -} diff --git a/cmd/rate-limit-handler_test.go b/cmd/rate-limit-handler_test.go deleted file mode 100644 index feccc7a02..000000000 --- a/cmd/rate-limit-handler_test.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" -) - -// This test sets globalMaxConn to 1 and starts 6 connections in -// parallel on a server with the rate limit handler configured. This -// should allow one request to execute at a time, and at most 4 to -// wait to execute and the 6th request should get a 429 status code -// error. -func TestRateLimitHandler(t *testing.T) { - // save the global Max connections - saveGlobalMaxConn := globalMaxConn - - globalMaxConn = 1 - testHandler := func(w http.ResponseWriter, r *http.Request) { - time.Sleep(100 * time.Millisecond) - fmt.Fprintln(w, "Hello client!") - } - rlh := setRateLimitHandler(http.HandlerFunc(testHandler)) - ts := httptest.NewServer(rlh) - respCh := make(chan int) - startTime := time.Now() - for i := 0; i < 6; i++ { - go func(ch chan<- int) { - resp, err := http.Get(ts.URL) - if err != nil { - t.Errorf( - "Got error requesting test server - %v\n", - err, - ) - } - respCh <- resp.StatusCode - }(respCh) - } - - tooManyReqErrCount := 0 - for i := 0; i < 6; i++ { - code := <-respCh - if code == 429 { - tooManyReqErrCount++ - } else if code != 200 { - t.Errorf("Got non-200 resp code - %d\n", code) - } - } - duration := time.Since(startTime) - if duration < time.Duration(500*time.Millisecond) { - // as globalMaxConn is 1, only 1 request will execute - // at a time, and the five allowed requested will take - // at least 500 ms. - t.Errorf("Expected all requests to take at least 500ms, but it was done in %v\n", - duration) - } - if tooManyReqErrCount != 1 { - t.Errorf("Expected to get 1 error, but got %d", - tooManyReqErrCount) - } - ts.Close() - - // restore the global Max connections - globalMaxConn = saveGlobalMaxConn -} diff --git a/cmd/routers.go b/cmd/routers.go index 5ce1c5fed..14209c028 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -18,8 +18,6 @@ package cmd import ( "net/http" - "os" - "strings" router "github.com/gorilla/mux" ) @@ -109,17 +107,13 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) return nil, err } - // set environmental variable MINIO_BROWSER=off to disable minio web browser. - // By default minio web browser is enabled. - if !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off") { - // Register RPC router for web related calls. - if err = registerBrowserPeerRPCRouter(mux); err != nil { - return nil, err - } + // Register RPC router for web related calls. + if err = registerBrowserPeerRPCRouter(mux); err != nil { + return nil, err + } - if err = registerWebRouter(mux); err != nil { - return nil, err - } + if err = registerWebRouter(mux); err != nil { + return nil, err } // Add API router. @@ -127,8 +121,6 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) // List of some generic handlers which are applied for all incoming requests. var handlerFns = []HandlerFunc{ - // Limits the number of concurrent http requests. - setRateLimitHandler, // Limits all requests size to a maximum fixed limit setRequestSizeLimitHandler, // Adds 'crossdomain.xml' policy handler to serve legacy flash clients. @@ -139,6 +131,8 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) setPrivateBucketHandler, // Adds cache control for all browser requests. setBrowserCacheControlHandler, + // Validates all incoming requests to have a valid date header. + setTimeValidityHandler, // CORS setting for all browser API requests. setCorsHandler, // Validates all incoming URL resources, for invalid/unsupported diff --git a/cmd/server-main.go b/cmd/server-main.go index 453eae681..593708044 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -22,9 +22,7 @@ import ( "net/http" "net/url" "os" - "strconv" "strings" - "time" "regexp" "runtime" @@ -38,10 +36,6 @@ var serverFlags = []cli.Flag{ Value: ":9000", Usage: `Bind to a specific IP:PORT. Defaults to ":9000".`, }, - cli.StringFlag{ - Name: "ignore-disks", - Usage: `Comma separated list of faulty drives to ignore at startup.`, - }, } var serverCmd = cli.Command{ @@ -63,13 +57,6 @@ ENVIRONMENT VARIABLES: MINIO_ACCESS_KEY: Username or access key of 5 to 20 characters in length. MINIO_SECRET_KEY: Password or secret key of 8 to 40 characters in length. - CACHING: - MINIO_CACHE_SIZE: Limit maximum cache size. Allowed units are [GB|MB|KB]. Defaults to 8GB. - MINIO_CACHE_EXPIRY: Automatically expire cached objects. Allowed units are [h|m|s]. Defaults to 72h. - - SECURITY: - MINIO_SECURE_CONSOLE: Set secure console to 'no' to disable printing secret key. Defaults to 'yes'. - EXAMPLES: 1. Start minio server on "/home/shared" directory. $ minio {{.Name}} /home/shared @@ -92,10 +79,9 @@ EXAMPLES: } type serverCmdConfig struct { - serverAddr string - endpoints []*url.URL - ignoredEndpoints []*url.URL - storageDisks []StorageAPI + serverAddr string + endpoints []*url.URL + storageDisks []StorageAPI } // Parse an array of end-points (from the command line) @@ -190,27 +176,6 @@ func initServerConfig(c *cli.Context) { err := createCertsPath() fatalIf(err, "Unable to create \"certs\" directory.") - // Fetch max conn limit from environment variable. - if maxConnStr := os.Getenv("MINIO_MAXCONN"); maxConnStr != "" { - // We need to parse to its integer value. - globalMaxConn, err = strconv.Atoi(maxConnStr) - fatalIf(err, "Unable to convert MINIO_MAXCONN=%s environment variable into its integer value.", maxConnStr) - } - - // Fetch max cache size from environment variable. - if maxCacheSizeStr := os.Getenv("MINIO_CACHE_SIZE"); maxCacheSizeStr != "" { - // We need to parse cache size to its integer value. - globalMaxCacheSize, err = strconvBytes(maxCacheSizeStr) - fatalIf(err, "Unable to convert MINIO_CACHE_SIZE=%s environment variable into its integer value.", maxCacheSizeStr) - } - - // Fetch cache expiry from environment variable. - if cacheExpiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); cacheExpiryStr != "" { - // We need to parse cache expiry to its time.Duration value. - globalCacheExpiry, err = time.ParseDuration(cacheExpiryStr) - fatalIf(err, "Unable to convert MINIO_CACHE_EXPIRY=%s environment variable into its time.Duration value.", cacheExpiryStr) - } - // When credentials inherited from the env, server cmd has to save them in the disk if os.Getenv("MINIO_ACCESS_KEY") != "" && os.Getenv("MINIO_SECRET_KEY") != "" { // Env credentials are already loaded in serverConfig, just save in the disk @@ -332,26 +297,6 @@ func checkServerSyntax(c *cli.Context) { fatalIf(err, "Storage endpoint error.") } - // Verify syntax for all the ignored disks. - var ignoredEndpoints []*url.URL - ignoredDisksStr := c.String("ignore-disks") - if ignoredDisksStr != "" { - ignoredDisks := strings.Split(ignoredDisksStr, ",") - if len(endpoints) == 1 { - fatalIf(errInvalidArgument, "--ignore-disks is valid only for XL setup.") - } - ignoredEndpoints, err = parseStorageEndpoints(ignoredDisks) - fatalIf(err, "Unable to parse ignored storage endpoints %s", ignoredDisks) - checkEndpointsSyntax(ignoredEndpoints, ignoredDisks) - - for i, ep := range ignoredEndpoints { - // An ignored disk should be present in the XL disks list. - if !containsEndpoint(endpoints, ep) { - fatalIf(errInvalidArgument, "Ignored storage %s not available in the list of erasure storages list.", disks[i]) - } - } - } - if !isDistributedSetup(endpoints) { // for FS and singlenode-XL validation is done, return. return @@ -412,18 +357,11 @@ func serverMain(c *cli.Context) { // depends on it. checkServerSyntax(c) - // Disks to be ignored in server init, to skip format healing. - var ignoredEndpoints []*url.URL - if len(c.String("ignore-disks")) > 0 { - ignoredEndpoints, err = parseStorageEndpoints(strings.Split(c.String("ignore-disks"), ",")) - fatalIf(err, "Unable to parse storage endpoints %s", strings.Split(c.String("ignore-disks"), ",")) - } - // Disks to be used in server init. endpoints, err := parseStorageEndpoints(c.Args()) fatalIf(err, "Unable to parse storage endpoints %s", c.Args()) - storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints) + storageDisks, err := initStorageDisks(endpoints) fatalIf(err, "Unable to initialize storage disks.") // Cleanup objects that weren't successfully written into the namespace. @@ -440,10 +378,9 @@ func serverMain(c *cli.Context) { // Configure server. srvConfig := serverCmdConfig{ - serverAddr: serverAddr, - endpoints: endpoints, - ignoredEndpoints: ignoredEndpoints, - storageDisks: storageDisks, + serverAddr: serverAddr, + endpoints: endpoints, + storageDisks: storageDisks, } // Configure server. diff --git a/cmd/server-main_test.go b/cmd/server-main_test.go index 3ba1402ef..4d0463971 100644 --- a/cmd/server-main_test.go +++ b/cmd/server-main_test.go @@ -258,7 +258,7 @@ func TestCheckServerSyntax(t *testing.T) { t.Fatalf("Test %d : Unexpected error %s", i+1, err) } checkEndpointsSyntax(endpoints, disks) - _, err = initStorageDisks(endpoints, nil) + _, err = initStorageDisks(endpoints) if err != nil { t.Errorf("Test %d : disk init failed : %s", i+1, err) } @@ -336,9 +336,6 @@ func TestInitServerConfig(t *testing.T) { envVar string val string }{ - {"MINIO_MAXCONN", "10"}, - {"MINIO_CACHE_SIZE", "42MB"}, - {"MINIO_CACHE_EXPIRY", "2h45m"}, {"MINIO_ACCESS_KEY", "abcd1"}, {"MINIO_SECRET_KEY", "abcd12345"}, } diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index ccb3b9180..8e0e96742 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -19,7 +19,6 @@ package cmd import ( "crypto/x509" "fmt" - "os" "runtime" "strings" "time" @@ -45,6 +44,10 @@ func getFormatStr(strLen int, padding int) string { // Prints the formatted startup message. func printStartupMessage(endPoints []string) { + // If quiet flag is set do not print startup message. + if globalQuiet { + return + } printServerCommonMsg(endPoints) printCLIAccessMsg(endPoints[0]) printObjectAPIMsg() @@ -73,11 +76,7 @@ func printServerCommonMsg(endPoints []string) { // Colorize the message and print. console.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(endPointStr), 1), endPointStr))) console.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKeyID))) - secretKey := cred.SecretAccessKey - if strings.EqualFold(os.Getenv("MINIO_SECURE_CONSOLE"), "no") { - secretKey = "*REDACTED*" - } - console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", secretKey))) + console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretAccessKey))) console.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region))) printEventNotifiers() @@ -109,15 +108,11 @@ func printCLIAccessMsg(endPoint string) { // Configure 'mc', following block prints platform specific information for minio client. console.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide) - secretKey := cred.SecretAccessKey - if os.Getenv("MINIO_SECURE_CONSOLE") == "0" { - secretKey = "*REDACTED*" - } if runtime.GOOS == "windows" { - mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKeyID, secretKey) + mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKeyID, cred.SecretAccessKey) console.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } else { - mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKeyID, secretKey) + mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKeyID, cred.SecretAccessKey) console.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } } diff --git a/cmd/storage-rpc-server.go b/cmd/storage-rpc-server.go index 6c81b7dd2..8f4eec28b 100644 --- a/cmd/storage-rpc-server.go +++ b/cmd/storage-rpc-server.go @@ -227,10 +227,6 @@ func (s *storageServer) TryInitHandler(args *GenericArgs, reply *GenericReply) e // Initialize new storage rpc. func newRPCServer(srvConfig serverCmdConfig) (servers []*storageServer, err error) { for _, ep := range srvConfig.endpoints { - if containsEndpoint(srvConfig.ignoredEndpoints, ep) { - // Do not init disk RPC for ignored end point. - continue - } // e.g server:/mnt/disk1 if isLocalStorage(ep) { // Get the posix path. diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index d1a22277f..edc649cd9 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -66,7 +66,7 @@ func prepareFS() (ObjectLayer, string, error) { if err != nil { return nil, "", err } - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { removeRoots(fsDirs) return nil, "", err @@ -84,7 +84,7 @@ func prepareXL() (ObjectLayer, []string, error) { if err != nil { return nil, nil, err } - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { removeRoots(fsDirs) return nil, nil, err @@ -189,7 +189,7 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer { testServer.AccessKey = credentials.AccessKeyID testServer.SecretKey = credentials.SecretAccessKey - objLayer, storageDisks, err := initObjectLayer(testServer.Disks, nil) + objLayer, storageDisks, err := initObjectLayer(testServer.Disks) if err != nil { t.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -385,7 +385,7 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer { testRPCServer.SecretKey = credentials.SecretAccessKey // create temporary backend for the test server. - objLayer, storageDisks, err := initObjectLayer(endpoints, nil) + objLayer, storageDisks, err := initObjectLayer(endpoints) if err != nil { t.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -459,7 +459,7 @@ func StartTestControlRPCServer(t TestErrHandler, instanceType string) TestServer testRPCServer.SecretKey = credentials.SecretAccessKey // create temporary backend for the test server. - objLayer, storageDisks, err := initObjectLayer(endpoints, nil) + objLayer, storageDisks, err := initObjectLayer(endpoints) if err != nil { t.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -1595,8 +1595,8 @@ func getRandomDisks(N int) ([]string, error) { } // initObjectLayer - Instantiates object layer and returns it. -func initObjectLayer(endpoints, ignoredEndpoints []*url.URL) (ObjectLayer, []StorageAPI, error) { - storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints) +func initObjectLayer(endpoints []*url.URL) (ObjectLayer, []StorageAPI, error) { + storageDisks, err := initStorageDisks(endpoints) if err != nil { return nil, nil, err } @@ -1677,7 +1677,7 @@ func prepareXLStorageDisks(t *testing.T) ([]StorageAPI, []string) { t.Fatal("Unexpected error: ", err) } - _, storageDisks, err := initObjectLayer(endpoints, nil) + _, storageDisks, err := initObjectLayer(endpoints) if err != nil { removeRoots(fsDirs) t.Fatal("Unable to initialize storage disks", err) @@ -1972,7 +1972,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType) if err != nil { t.Fatalf("Initialization of disks for XL setup: %s", err) } - objLayer, _, err := initObjectLayer(endpoints, nil) + objLayer, _, err := initObjectLayer(endpoints) if err != nil { t.Fatalf("Initialization of object layer failed for XL setup: %s", err) } diff --git a/cmd/utils.go b/cmd/utils.go index d6a26be14..94ab1dc29 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -23,7 +23,6 @@ import ( "io" "net/http" "net/url" - "os" "strings" "encoding/json" @@ -171,16 +170,14 @@ func urlPathSplit(urlPath string) (bucketName, prefixName string) { func startProfiler(profiler string) interface { Stop() } { - // Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted - profileDir := os.Getenv("MINIO_PROFILE_DIR") - // Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block]. + // Enable profiler if ``_MINIO_PROFILER`` is set. Supported options are [cpu, mem, block]. switch profiler { case "cpu": - return profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir)) + return profile.Start(profile.CPUProfile, profile.NoShutdownHook) case "mem": - return profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir)) + return profile.Start(profile.MemProfile, profile.NoShutdownHook) case "block": - return profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir)) + return profile.Start(profile.BlockProfile, profile.NoShutdownHook) default: return nil } diff --git a/cmd/utils_test.go b/cmd/utils_test.go index 5fb6be69c..80efbf61b 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -220,6 +220,13 @@ func TestMaxPartID(t *testing.T) { } } +// Add tests for starting and stopping different profilers. +func TestStartProfiler(t *testing.T) { + if startProfiler("") != nil { + t.Fatal("Expected nil, but non-nil value returned for invalid profiler.") + } +} + // Tests fetch local address. func TestLocalAddress(t *testing.T) { if runtime.GOOS == "windows" { diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index 4a9f2d580..e22e53b57 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -18,7 +18,9 @@ package cmd import ( "fmt" + "os" "sort" + "strings" "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/objcache" @@ -129,6 +131,9 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) { // Initialize list pool. listPool := newTreeWalkPool(globalLookupTimeout) + // Check if object cache is disabled. + objCacheDisabled := strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off") + // Initialize xl objects. xl := xlObjects{ storageDisks: newStorageDisks, @@ -136,7 +141,7 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) { parityBlocks: parityBlocks, listPool: listPool, objCache: objCache, - objCacheEnabled: globalMaxCacheSize > 0, + objCacheEnabled: !objCacheDisabled, } // Figure out read and write quorum based on number of storage disks. diff --git a/cmd/xl-v1_test.go b/cmd/xl-v1_test.go index 126ff59d2..011a237f7 100644 --- a/cmd/xl-v1_test.go +++ b/cmd/xl-v1_test.go @@ -56,12 +56,7 @@ func TestStorageInfo(t *testing.T) { t.Fatalf("Unexpected error %s", err) } - ignoredEndpoints, err := parseStorageEndpoints(fsDirs[:4]) - if err != nil { - t.Fatalf("Unexpected error %s", err) - } - - storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints) + storageDisks, err := initStorageDisks(endpoints) if err != nil { t.Fatal("Unexpected error: ", err) } @@ -156,7 +151,7 @@ func TestNewXL(t *testing.T) { t.Fatalf("Unable to initialize erasure, %s", err) } - storageDisks, err := initStorageDisks(endpoints, nil) + storageDisks, err := initStorageDisks(endpoints) if err != nil { t.Fatal("Unexpected error: ", err) } @@ -186,11 +181,7 @@ func TestNewXL(t *testing.T) { t.Fatalf("Unable to initialize erasure, %s", err) } - ignoredEndpoints, err := parseStorageEndpoints(erasureDisks[:2]) - if err != nil { - t.Fatalf("Unable to initialize erasure, %s", err) - } - storageDisks, err = initStorageDisks(endpoints, ignoredEndpoints) + storageDisks, err = initStorageDisks(endpoints) if err != nil { t.Fatal("Unexpected error: ", err) } diff --git a/docs/caching/README.md b/docs/caching/README.md index fddd3fbee..4c52a5054 100644 --- a/docs/caching/README.md +++ b/docs/caching/README.md @@ -2,36 +2,30 @@ Object caching by turned on by default with following settings - - Default cache size 8GB, can be changed from environment variable - ``MINIO_CACHE_SIZE`` supports both SI and ISO IEC standard forms - for input size parameters. + - Default cache size 8GB. Cache size also automatically picks + a lower value if your local memory size is lower than 8GB. - - Default expiration of entries is 72 hours, can be changed from - environment variable ``MINIO_CACHE_EXPIRY`` supportings Go - ``time.Duration`` with valid units "ns", "us" (or "µs"), - "ms", "s", "m", "h". + - Default expiration of entries happensat 72 hours, + this option cannot be changed. - - Default expiry interval is 1/4th of the expiration hours, so - expiration sweep happens across the cache every 1/4th the time - duration of the set entry expiration duration. - -### Tricks - -Setting MINIO_CACHE_SIZE=0 will turn off caching entirely. -Setting MINIO_CACHE_EXPIRY=0s will turn off cache garbage collections, -all cached objects will never expire. + - Default expiry interval is 1/4th of the expiration hours, so + expiration sweep happens across the cache every 1/4th the time + duration of the set entry expiration duration. ### Behavior -Caching happens for both GET and PUT. +Caching happens on both GET and PUT operations. -- GET caches new objects for entries not found in cache, -otherwise serves from the cache. +- GET caches new objects for entries not found in cache. - PUT/POST caches all successfully uploaded objects. -NOTE: Cache is not populated if there are any errors - while reading from the disk. +In all other cases if objects are served from cache. + +NOTE: + +Cache is always populated upon object is successfully +read from the disk. Expiration happens automatically based on the configured interval as explained above, frequently accessed objects diff --git a/docs/minio-env-variables.md b/docs/minio-env-variables.md deleted file mode 100644 index a40ccf635..000000000 --- a/docs/minio-env-variables.md +++ /dev/null @@ -1,47 +0,0 @@ -# Minio Environmental variables - -#### MINIO_ENABLE_FSMETA -When enabled, minio-FS saves the HTTP headers that start with `X-Amz-Meta-` and `X-Minio-Meta`. These header meta data can be retrieved on HEAD and GET requests on the object. - -#### MINIO_PROFILER -Used for Go profiling. Supported values are: - -`cpu` - for CPU profiling - -`mem` - for memory profiling - -`block` - for block profiling - -#### MINIO_PROFILE_DIR - -Path where cpu/mem/block profiling files are dumped - -#### MINIO_BROWSER - -setting this to `off` disables the minio browser. - -#### MINIO_ACCESS_KEY - -Minio access key. - -#### MINIO_SECRET_KEY - -Minio secret key. - -#### MINIO_CACHE_SIZE - -Set total cache size in NN[GB|MB|KB]. Defaults to 8GB - -Ex: MINIO_CACHE_SIZE=2GB - -#### MINIO_CACHE_EXPIRY - -Set the object cache expiration duration in NN[h|m|s]. Defaults to 72 hours. - -Ex. MINIO_CACHE_EXPIRY=24h - -#### MINIO_MAXCONN - -Limit of the number of concurrent http requests. - -Ex. MINIO_MAXCONN=500 diff --git a/vendor/github.com/urfave/negroni/CHANGELOG.md b/vendor/github.com/urfave/negroni/CHANGELOG.md deleted file mode 100644 index 5e77c3fb6..000000000 --- a/vendor/github.com/urfave/negroni/CHANGELOG.md +++ /dev/null @@ -1,35 +0,0 @@ -# Change Log - -**ATTN**: This project uses [semantic versioning](http://semver.org/). - -## [Unreleased] -### Added -- `Recovery.ErrorHandlerFunc` for custom error handling during recovery - -### Fixed -- `Written()` correct returns `false` if no response header has been written - -### Changed -- Set default status to `0` in the case that no handler writes status -- was - previously `200` (in 0.2.0, before that it was `0` so this reestablishes that - behavior) -- Catch `panic`s thrown by callbacks provided to the `Recovery` handler - -## [0.2.0] - 2016-05-10 -### Added -- Support for variadic handlers in `New()` -- Added `Negroni.Handlers()` to fetch all of the handlers for a given chain -- Allowed size in `Recovery` handler was bumped to 8k -- `Negroni.UseFunc` to push another handler onto the chain - -### Changed -- Set the status before calling `beforeFuncs` so the information is available to them -- Set default status to `200` in the case that no handler writes status -- was previously `0` -- Panic if `nil` handler is given to `negroni.Use` - -## 0.1.0 - 2013-07-22 -### Added -- Initial implementation. - -[Unreleased]: https://github.com/urfave/negroni/compare/v0.2.0...HEAD -[0.2.0]: https://github.com/urfave/negroni/compare/v0.1.0...v0.2.0 diff --git a/vendor/github.com/urfave/negroni/LICENSE b/vendor/github.com/urfave/negroni/LICENSE deleted file mode 100644 index 08b5e20ac..000000000 --- a/vendor/github.com/urfave/negroni/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Jeremy Saenz - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/urfave/negroni/README.md b/vendor/github.com/urfave/negroni/README.md deleted file mode 100644 index 6fa836410..000000000 --- a/vendor/github.com/urfave/negroni/README.md +++ /dev/null @@ -1,443 +0,0 @@ -# Negroni -[![GoDoc](https://godoc.org/github.com/urfave/negroni?status.svg)](http://godoc.org/github.com/urfave/negroni) -[![Build Status](https://travis-ci.org/urfave/negroni.svg?branch=master)](https://travis-ci.org/urfave/negroni) -[![codebeat](https://codebeat.co/badges/47d320b1-209e-45e8-bd99-9094bc5111e2)](https://codebeat.co/projects/github-com-urfave-negroni) - -**Notice:** This is the library formerly known as -`github.com/codegangsta/negroni` -- Github will automatically redirect requests -to this repository, but we recommend updating your references for clarity. - -Negroni is an idiomatic approach to web middleware in Go. It is tiny, -non-intrusive, and encourages use of `net/http` Handlers. - -If you like the idea of [Martini](https://github.com/go-martini/martini), but -you think it contains too much magic, then Negroni is a great fit. - -Language Translations: -* [German (de_DE)](translations/README_de_de.md) -* [Português Brasileiro (pt_BR)](translations/README_pt_br.md) -* [简体中文 (zh_cn)](translations/README_zh_cn.md) -* [繁體中文 (zh_tw)](translations/README_zh_tw.md) -* [日本語 (ja_JP)](translations/README_ja_JP.md) - -## Getting Started - -After installing Go and setting up your -[GOPATH](http://golang.org/doc/code.html#GOPATH), create your first `.go` file. -We'll call it `server.go`. - - -``` go -package main - -import ( - "fmt" - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome to the home page!") - }) - - n := negroni.Classic() // Includes some default middlewares - n.UseHandler(mux) - - http.ListenAndServe(":3000", n) -} -``` - -Then install the Negroni package (**NOTE**: >= **go 1.1** is required): - -``` -go get github.com/urfave/negroni -``` - -Then run your server: - -``` -go run server.go -``` - -You will now have a Go `net/http` webserver running on `localhost:3000`. - -## Is Negroni a Framework? - -Negroni is **not** a framework. It is a middleware-focused library that is -designed to work directly with `net/http`. - -## Routing? - -Negroni is BYOR (Bring your own Router). The Go community already has a number -of great http routers available, and Negroni tries to play well with all of them -by fully supporting `net/http`. For instance, integrating with [Gorilla Mux] -looks like so: - -``` go -router := mux.NewRouter() -router.HandleFunc("/", HomeHandler) - -n := negroni.New(Middleware1, Middleware2) -// Or use a middleware with the Use() function -n.Use(Middleware3) -// router goes last -n.UseHandler(router) - -http.ListenAndServe(":3001", n) -``` - -## `negroni.Classic()` - -`negroni.Classic()` provides some default middleware that is useful for most -applications: - -* [`negroni.Recovery`](#recovery) - Panic Recovery Middleware. -* [`negroni.Logger`](#logger) - Request/Response Logger Middleware. -* [`negroni.Static`](#static) - Static File serving under the "public" - directory. - -This makes it really easy to get started with some useful features from Negroni. - -## Handlers - -Negroni provides a bidirectional middleware flow. This is done through the -`negroni.Handler` interface: - -``` go -type Handler interface { - ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) -} -``` - -If a middleware hasn't already written to the `ResponseWriter`, it should call -the next `http.HandlerFunc` in the chain to yield to the next middleware -handler. This can be used for great good: - -``` go -func MyMiddleware(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - // do some stuff before - next(rw, r) - // do some stuff after -} -``` - -And you can map it to the handler chain with the `Use` function: - -``` go -n := negroni.New() -n.Use(negroni.HandlerFunc(MyMiddleware)) -``` - -You can also map plain old `http.Handler`s: - -``` go -n := negroni.New() - -mux := http.NewServeMux() -// map your routes - -n.UseHandler(mux) - -http.ListenAndServe(":3000", n) -``` - -## `Run()` - -Negroni has a convenience function called `Run`. `Run` takes an addr string -identical to [`http.ListenAndServe`](https://godoc.org/net/http#ListenAndServe). - - -``` go -package main - -import ( - "github.com/urfave/negroni" -) - -func main() { - n := negroni.Classic() - n.Run(":8080") -} -``` - -In general, you will want to use `net/http` methods and pass `negroni` as a -`Handler`, as this is more flexible, e.g.: - - -``` go -package main - -import ( - "fmt" - "log" - "net/http" - "time" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome to the home page!") - }) - - n := negroni.Classic() // Includes some default middlewares - n.UseHandler(mux) - - s := &http.Server{ - Addr: ":8080", - Handler: n, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - } - log.Fatal(s.ListenAndServe()) -} -``` - -## Route Specific Middleware - -If you have a route group of routes that need specific middleware to be -executed, you can simply create a new Negroni instance and use it as your route -handler. - -``` go -router := mux.NewRouter() -adminRoutes := mux.NewRouter() -// add admin routes here - -// Create a new negroni for the admin middleware -router.PathPrefix("/admin").Handler(negroni.New( - Middleware1, - Middleware2, - negroni.Wrap(adminRoutes), -)) -``` - -If you are using [Gorilla Mux], here is an example using a subrouter: - -``` go -router := mux.NewRouter() -subRouter := mux.NewRouter().PathPrefix("/subpath").Subrouter().StrictSlash(true) -subRouter.HandleFunc("/", someSubpathHandler) // "/subpath/" -subRouter.HandleFunc("/:id", someSubpathHandler) // "/subpath/:id" - -// "/subpath" is necessary to ensure the subRouter and main router linkup -router.PathPrefix("/subpath").Handler(negroni.New( - Middleware1, - Middleware2, - negroni.Wrap(subRouter), -)) -``` - -## Bundled Middleware - -### Static - -This middleware will serve files on the filesystem. If the files do not exist, -it proxies the request to the next middleware. If you want the requests for -non-existent files to return a `404 File Not Found` to the user you should look -at using [http.FileServer](https://golang.org/pkg/net/http/#FileServer) as -a handler. - -Example: - - -``` go -package main - -import ( - "fmt" - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome to the home page!") - }) - - // Example of using a http.FileServer if you want "server-like" rather than "middleware" behavior - // mux.Handle("/public", http.FileServer(http.Dir("/home/public"))) - - n := negroni.New() - n.Use(negroni.NewStatic(http.Dir("/tmp"))) - n.UseHandler(mux) - - http.ListenAndServe(":3002", n) -} -``` - -Will serve files from the `/tmp` directory first, but proxy calls to the next -handler if the request does not match a file on the filesystem. - -### Recovery - -This middleware catches `panic`s and responds with a `500` response code. If -any other middleware has written a response code or body, this middleware will -fail to properly send a 500 to the client, as the client has already received -the HTTP response code. Additionally, an `ErrorHandlerFunc` can be attached -to report 500's to an error reporting service such as Sentry or Airbrake. - -Example: - - -``` go -package main - -import ( - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - panic("oh no") - }) - - n := negroni.New() - n.Use(negroni.NewRecovery()) - n.UseHandler(mux) - - http.ListenAndServe(":3003", n) -} -``` - -Will return a `500 Internal Server Error` to each request. It will also log the -stack traces as well as print the stack trace to the requester if `PrintStack` -is set to `true` (the default). - -Example with error handler: - -``` go -package main - -import ( - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - panic("oh no") - }) - - n := negroni.New() - recovery := negroni.NewRecovery() - recovery.ErrorHandlerFunc = reportToSentry - n.Use(recovery) - n.UseHandler(mux) - - http.ListenAndServe(":3003", n) -} - -func reportToSentry(error interface{}) { - // write code here to report error to Sentry -} -``` - - -## Logger - -This middleware logs each incoming request and response. - -Example: - - -``` go -package main - -import ( - "fmt" - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome to the home page!") - }) - - n := negroni.New() - n.Use(negroni.NewLogger()) - n.UseHandler(mux) - - http.ListenAndServe(":3004", n) -} -``` - -Will print a log similar to: - -``` -[negroni] Started GET / -[negroni] Completed 200 OK in 145.446µs -``` - -on each request. - -## Third Party Middleware - -Here is a current list of Negroni compatible middlware. Feel free to put up a PR -linking your middleware if you have built one: - -| Middleware | Author | Description | -| -----------|--------|-------------| -| [binding](https://github.com/mholt/binding) | [Matt Holt](https://github.com/mholt) | Data binding from HTTP requests into structs | -| [cloudwatch](https://github.com/cvillecsteele/negroni-cloudwatch) | [Colin Steele](https://github.com/cvillecsteele) | AWS cloudwatch metrics middleware | -| [cors](https://github.com/rs/cors) | [Olivier Poitrey](https://github.com/rs) | [Cross Origin Resource Sharing](http://www.w3.org/TR/cors/) (CORS) support | -| [csp](https://github.com/awakenetworks/csp) | [Awake Networks](https://github.com/awakenetworks) | [Content Security Policy](https://www.w3.org/TR/CSP2/) (CSP) support | -| [delay](https://github.com/jeffbmartinez/delay) | [Jeff Martinez](https://github.com/jeffbmartinez) | Add delays/latency to endpoints. Useful when testing effects of high latency | -| [New Relic Go Agent](https://github.com/yadvendar/negroni-newrelic-go-agent) | [Yadvendar Champawat](https://github.com/yadvendar) | Official [New Relic Go Agent](https://github.com/newrelic/go-agent) (currently in beta) | -| [gorelic](https://github.com/jingweno/negroni-gorelic) | [Jingwen Owen Ou](https://github.com/jingweno) | New Relic agent for Go runtime | -| [Graceful](https://github.com/tylerb/graceful) | [Tyler Bunnell](https://github.com/tylerb) | Graceful HTTP Shutdown | -| [gzip](https://github.com/phyber/negroni-gzip) | [phyber](https://github.com/phyber) | GZIP response compression | -| [JWT Middleware](https://github.com/auth0/go-jwt-middleware) | [Auth0](https://github.com/auth0) | Middleware checks for a JWT on the `Authorization` header on incoming requests and decodes it| -| [logrus](https://github.com/meatballhat/negroni-logrus) | [Dan Buch](https://github.com/meatballhat) | Logrus-based logger | -| [oauth2](https://github.com/goincremental/negroni-oauth2) | [David Bochenski](https://github.com/bochenski) | oAuth2 middleware | -| [onthefly](https://github.com/xyproto/onthefly) | [Alexander Rødseth](https://github.com/xyproto) | Generate TinySVG, HTML and CSS on the fly | -| [permissions2](https://github.com/xyproto/permissions2) | [Alexander Rødseth](https://github.com/xyproto) | Cookies, users and permissions | -| [prometheus](https://github.com/zbindenren/negroni-prometheus) | [Rene Zbinden](https://github.com/zbindenren) | Easily create metrics endpoint for the [prometheus](http://prometheus.io) instrumentation tool | -| [render](https://github.com/unrolled/render) | [Cory Jacobsen](https://github.com/unrolled) | Render JSON, XML and HTML templates | -| [RestGate](https://github.com/pjebs/restgate) | [Prasanga Siripala](https://github.com/pjebs) | Secure authentication for REST API endpoints | -| [secure](https://github.com/unrolled/secure) | [Cory Jacobsen](https://github.com/unrolled) | Middleware that implements a few quick security wins | -| [sessions](https://github.com/goincremental/negroni-sessions) | [David Bochenski](https://github.com/bochenski) | Session Management | -| [stats](https://github.com/thoas/stats) | [Florent Messa](https://github.com/thoas) | Store information about your web application (response time, etc.) | -| [VanGoH](https://github.com/auroratechnologies/vangoh) | [Taylor Wrobel](https://github.com/twrobel3) | Configurable [AWS-Style](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html) HMAC authentication middleware | -| [xrequestid](https://github.com/pilu/xrequestid) | [Andrea Franz](https://github.com/pilu) | Middleware that assigns a random X-Request-Id header to each request | -| [mgo session](https://github.com/joeljames/nigroni-mgo-session) | [Joel James](https://github.com/joeljames) | Middleware that handles creating and closing mgo sessions per request | - -## Examples - -[Alexander Rødseth](https://github.com/xyproto) created -[mooseware](https://github.com/xyproto/mooseware), a skeleton for writing a -Negroni middleware handler. - -## Live code reload? - -[gin](https://github.com/codegangsta/gin) and -[fresh](https://github.com/pilu/fresh) both live reload negroni apps. - -## Essential Reading for Beginners of Go & Negroni - -* [Using a Context to pass information from middleware to end handler](http://elithrar.github.io/article/map-string-interface/) -* [Understanding middleware](https://mattstauffer.co/blog/laravel-5.0-middleware-filter-style) - -## About - -Negroni is obsessively designed by none other than the [Code -Gangsta](https://codegangsta.io/) - -[Gorilla Mux]: https://github.com/gorilla/mux -[`http.FileSystem`]: https://godoc.org/net/http#FileSystem diff --git a/vendor/github.com/urfave/negroni/doc.go b/vendor/github.com/urfave/negroni/doc.go deleted file mode 100644 index add1ed9f7..000000000 --- a/vendor/github.com/urfave/negroni/doc.go +++ /dev/null @@ -1,25 +0,0 @@ -// Package negroni is an idiomatic approach to web middleware in Go. It is tiny, non-intrusive, and encourages use of net/http Handlers. -// -// If you like the idea of Martini, but you think it contains too much magic, then Negroni is a great fit. -// -// For a full guide visit http://github.com/urfave/negroni -// -// package main -// -// import ( -// "github.com/urfave/negroni" -// "net/http" -// "fmt" -// ) -// -// func main() { -// mux := http.NewServeMux() -// mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { -// fmt.Fprintf(w, "Welcome to the home page!") -// }) -// -// n := negroni.Classic() -// n.UseHandler(mux) -// n.Run(":3000") -// } -package negroni diff --git a/vendor/github.com/urfave/negroni/logger.go b/vendor/github.com/urfave/negroni/logger.go deleted file mode 100644 index 04cd53b56..000000000 --- a/vendor/github.com/urfave/negroni/logger.go +++ /dev/null @@ -1,35 +0,0 @@ -package negroni - -import ( - "log" - "net/http" - "os" - "time" -) - -// ALogger interface -type ALogger interface { - Println(v ...interface{}) - Printf(format string, v ...interface{}) -} - -// Logger is a middleware handler that logs the request as it goes in and the response as it goes out. -type Logger struct { - // ALogger implements just enough log.Logger interface to be compatible with other implementations - ALogger -} - -// NewLogger returns a new Logger instance -func NewLogger() *Logger { - return &Logger{log.New(os.Stdout, "[negroni] ", 0)} -} - -func (l *Logger) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - start := time.Now() - l.Printf("Started %s %s", r.Method, r.URL.Path) - - next(rw, r) - - res := rw.(ResponseWriter) - l.Printf("Completed %v %s in %v", res.Status(), http.StatusText(res.Status()), time.Since(start)) -} diff --git a/vendor/github.com/urfave/negroni/negroni.go b/vendor/github.com/urfave/negroni/negroni.go deleted file mode 100644 index 9c7c187e7..000000000 --- a/vendor/github.com/urfave/negroni/negroni.go +++ /dev/null @@ -1,133 +0,0 @@ -package negroni - -import ( - "log" - "net/http" - "os" -) - -// Handler handler is an interface that objects can implement to be registered to serve as middleware -// in the Negroni middleware stack. -// ServeHTTP should yield to the next middleware in the chain by invoking the next http.HandlerFunc -// passed in. -// -// If the Handler writes to the ResponseWriter, the next http.HandlerFunc should not be invoked. -type Handler interface { - ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) -} - -// HandlerFunc is an adapter to allow the use of ordinary functions as Negroni handlers. -// If f is a function with the appropriate signature, HandlerFunc(f) is a Handler object that calls f. -type HandlerFunc func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) - -func (h HandlerFunc) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - h(rw, r, next) -} - -type middleware struct { - handler Handler - next *middleware -} - -func (m middleware) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - m.handler.ServeHTTP(rw, r, m.next.ServeHTTP) -} - -// Wrap converts a http.Handler into a negroni.Handler so it can be used as a Negroni -// middleware. The next http.HandlerFunc is automatically called after the Handler -// is executed. -func Wrap(handler http.Handler) Handler { - return HandlerFunc(func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - handler.ServeHTTP(rw, r) - next(rw, r) - }) -} - -// Negroni is a stack of Middleware Handlers that can be invoked as an http.Handler. -// Negroni middleware is evaluated in the order that they are added to the stack using -// the Use and UseHandler methods. -type Negroni struct { - middleware middleware - handlers []Handler -} - -// New returns a new Negroni instance with no middleware preconfigured. -func New(handlers ...Handler) *Negroni { - return &Negroni{ - handlers: handlers, - middleware: build(handlers), - } -} - -// Classic returns a new Negroni instance with the default middleware already -// in the stack. -// -// Recovery - Panic Recovery Middleware -// Logger - Request/Response Logging -// Static - Static File Serving -func Classic() *Negroni { - return New(NewRecovery(), NewLogger(), NewStatic(http.Dir("public"))) -} - -func (n *Negroni) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - n.middleware.ServeHTTP(NewResponseWriter(rw), r) -} - -// Use adds a Handler onto the middleware stack. Handlers are invoked in the order they are added to a Negroni. -func (n *Negroni) Use(handler Handler) { - if handler == nil { - panic("handler cannot be nil") - } - - n.handlers = append(n.handlers, handler) - n.middleware = build(n.handlers) -} - -// UseFunc adds a Negroni-style handler function onto the middleware stack. -func (n *Negroni) UseFunc(handlerFunc func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc)) { - n.Use(HandlerFunc(handlerFunc)) -} - -// UseHandler adds a http.Handler onto the middleware stack. Handlers are invoked in the order they are added to a Negroni. -func (n *Negroni) UseHandler(handler http.Handler) { - n.Use(Wrap(handler)) -} - -// UseHandler adds a http.HandlerFunc-style handler function onto the middleware stack. -func (n *Negroni) UseHandlerFunc(handlerFunc func(rw http.ResponseWriter, r *http.Request)) { - n.UseHandler(http.HandlerFunc(handlerFunc)) -} - -// Run is a convenience function that runs the negroni stack as an HTTP -// server. The addr string takes the same format as http.ListenAndServe. -func (n *Negroni) Run(addr string) { - l := log.New(os.Stdout, "[negroni] ", 0) - l.Printf("listening on %s", addr) - l.Fatal(http.ListenAndServe(addr, n)) -} - -// Returns a list of all the handlers in the current Negroni middleware chain. -func (n *Negroni) Handlers() []Handler { - return n.handlers -} - -func build(handlers []Handler) middleware { - var next middleware - - if len(handlers) == 0 { - return voidMiddleware() - } else if len(handlers) > 1 { - next = build(handlers[1:]) - } else { - next = voidMiddleware() - } - - return middleware{handlers[0], &next} -} - -func voidMiddleware() middleware { - return middleware{ - HandlerFunc(func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {}), - &middleware{}, - } -} diff --git a/vendor/github.com/urfave/negroni/recovery.go b/vendor/github.com/urfave/negroni/recovery.go deleted file mode 100644 index 8396cb1ed..000000000 --- a/vendor/github.com/urfave/negroni/recovery.go +++ /dev/null @@ -1,65 +0,0 @@ -package negroni - -import ( - "fmt" - "log" - "net/http" - "os" - "runtime" - "runtime/debug" -) - -// Recovery is a Negroni middleware that recovers from any panics and writes a 500 if there was one. -type Recovery struct { - Logger ALogger - PrintStack bool - ErrorHandlerFunc func(interface{}) - StackAll bool - StackSize int -} - -// NewRecovery returns a new instance of Recovery -func NewRecovery() *Recovery { - return &Recovery{ - Logger: log.New(os.Stdout, "[negroni] ", 0), - PrintStack: true, - StackAll: false, - StackSize: 1024 * 8, - } -} - -func (rec *Recovery) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - defer func() { - if err := recover(); err != nil { - if rw.Header().Get("Content-Type") == "" { - rw.Header().Set("Content-Type", "text/plain; charset=utf-8") - } - - rw.WriteHeader(http.StatusInternalServerError) - - stack := make([]byte, rec.StackSize) - stack = stack[:runtime.Stack(stack, rec.StackAll)] - - f := "PANIC: %s\n%s" - rec.Logger.Printf(f, err, stack) - - if rec.PrintStack { - fmt.Fprintf(rw, f, err, stack) - } - - if rec.ErrorHandlerFunc != nil { - func() { - defer func() { - if err := recover(); err != nil { - rec.Logger.Printf("provided ErrorHandlerFunc panic'd: %s, trace:\n%s", err, debug.Stack()) - rec.Logger.Printf("%s\n", debug.Stack()) - } - }() - rec.ErrorHandlerFunc(err) - }() - } - } - }() - - next(rw, r) -} diff --git a/vendor/github.com/urfave/negroni/response_writer.go b/vendor/github.com/urfave/negroni/response_writer.go deleted file mode 100644 index f805825aa..000000000 --- a/vendor/github.com/urfave/negroni/response_writer.go +++ /dev/null @@ -1,99 +0,0 @@ -package negroni - -import ( - "bufio" - "fmt" - "net" - "net/http" -) - -// ResponseWriter is a wrapper around http.ResponseWriter that provides extra information about -// the response. It is recommended that middleware handlers use this construct to wrap a responsewriter -// if the functionality calls for it. -type ResponseWriter interface { - http.ResponseWriter - http.Flusher - // Status returns the status code of the response or 200 if the response has - // not been written (as this is the default response code in net/http) - Status() int - // Written returns whether or not the ResponseWriter has been written. - Written() bool - // Size returns the size of the response body. - Size() int - // Before allows for a function to be called before the ResponseWriter has been written to. This is - // useful for setting headers or any other operations that must happen before a response has been written. - Before(func(ResponseWriter)) -} - -type beforeFunc func(ResponseWriter) - -// NewResponseWriter creates a ResponseWriter that wraps an http.ResponseWriter -func NewResponseWriter(rw http.ResponseWriter) ResponseWriter { - return &responseWriter{ - ResponseWriter: rw, - } -} - -type responseWriter struct { - http.ResponseWriter - status int - size int - beforeFuncs []beforeFunc -} - -func (rw *responseWriter) WriteHeader(s int) { - rw.status = s - rw.callBefore() - rw.ResponseWriter.WriteHeader(s) -} - -func (rw *responseWriter) Write(b []byte) (int, error) { - if !rw.Written() { - // The status will be StatusOK if WriteHeader has not been called yet - rw.WriteHeader(http.StatusOK) - } - size, err := rw.ResponseWriter.Write(b) - rw.size += size - return size, err -} - -func (rw *responseWriter) Status() int { - return rw.status -} - -func (rw *responseWriter) Size() int { - return rw.size -} - -func (rw *responseWriter) Written() bool { - return rw.status != 0 -} - -func (rw *responseWriter) Before(before func(ResponseWriter)) { - rw.beforeFuncs = append(rw.beforeFuncs, before) -} - -func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { - hijacker, ok := rw.ResponseWriter.(http.Hijacker) - if !ok { - return nil, nil, fmt.Errorf("the ResponseWriter doesn't support the Hijacker interface") - } - return hijacker.Hijack() -} - -func (rw *responseWriter) CloseNotify() <-chan bool { - return rw.ResponseWriter.(http.CloseNotifier).CloseNotify() -} - -func (rw *responseWriter) callBefore() { - for i := len(rw.beforeFuncs) - 1; i >= 0; i-- { - rw.beforeFuncs[i](rw) - } -} - -func (rw *responseWriter) Flush() { - flusher, ok := rw.ResponseWriter.(http.Flusher) - if ok { - flusher.Flush() - } -} diff --git a/vendor/github.com/urfave/negroni/static.go b/vendor/github.com/urfave/negroni/static.go deleted file mode 100644 index 34be967c0..000000000 --- a/vendor/github.com/urfave/negroni/static.go +++ /dev/null @@ -1,88 +0,0 @@ -package negroni - -import ( - "net/http" - "path" - "strings" -) - -// Static is a middleware handler that serves static files in the given -// directory/filesystem. If the file does not exist on the filesystem, it -// passes along to the next middleware in the chain. If you desire "fileserver" -// type behavior where it returns a 404 for unfound files, you should consider -// using http.FileServer from the Go stdlib. -type Static struct { - // Dir is the directory to serve static files from - Dir http.FileSystem - // Prefix is the optional prefix used to serve the static directory content - Prefix string - // IndexFile defines which file to serve as index if it exists. - IndexFile string -} - -// NewStatic returns a new instance of Static -func NewStatic(directory http.FileSystem) *Static { - return &Static{ - Dir: directory, - Prefix: "", - IndexFile: "index.html", - } -} - -func (s *Static) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - if r.Method != "GET" && r.Method != "HEAD" { - next(rw, r) - return - } - file := r.URL.Path - // if we have a prefix, filter requests by stripping the prefix - if s.Prefix != "" { - if !strings.HasPrefix(file, s.Prefix) { - next(rw, r) - return - } - file = file[len(s.Prefix):] - if file != "" && file[0] != '/' { - next(rw, r) - return - } - } - f, err := s.Dir.Open(file) - if err != nil { - // discard the error? - next(rw, r) - return - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - next(rw, r) - return - } - - // try to serve index file - if fi.IsDir() { - // redirect if missing trailing slash - if !strings.HasSuffix(r.URL.Path, "/") { - http.Redirect(rw, r, r.URL.Path+"/", http.StatusFound) - return - } - - file = path.Join(file, s.IndexFile) - f, err = s.Dir.Open(file) - if err != nil { - next(rw, r) - return - } - defer f.Close() - - fi, err = f.Stat() - if err != nil || fi.IsDir() { - next(rw, r) - return - } - } - - http.ServeContent(rw, r, file, fi.ModTime(), f) -} diff --git a/vendor/vendor.json b/vendor/vendor.json index f325b800c..4f2dd6e89 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -213,12 +213,6 @@ "revision": "173748da739a410c5b0b813b956f89ff94730b4c", "revisionTime": "2016-08-30T17:39:30Z" }, - { - "checksumSHA1": "hMj8vjbjGCAE368E3xAjZGFON+E=", - "path": "github.com/urfave/negroni", - "revision": "3f7ce7b928e14ff890b067e5bbbc80af73690a9c", - "revisionTime": "2016-09-09T03:51:52Z" - }, { "path": "golang.org/x/crypto/bcrypt", "revision": "7b85b097bf7527677d54d3220065e966a0e3b613",