diff --git a/cmd/api-router.go b/cmd/api-router.go index df4ee0110..d92aaaa06 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -16,10 +16,7 @@ package cmd -import ( - router "github.com/gorilla/mux" - "github.com/urfave/negroni" -) +import router "github.com/gorilla/mux" // objectAPIHandler implements and provides http handlers for S3 API. type objectAPIHandlers struct { @@ -34,7 +31,7 @@ func registerAPIRouter(mux *router.Router) { } // API Router - apiRouter := mux.NewRoute().PathPrefix("").Subrouter() + apiRouter := mux.NewRoute().PathPrefix("/").Subrouter() // Bucket router bucket := apiRouter.PathPrefix("/{bucket}").Subrouter() @@ -99,12 +96,4 @@ func registerAPIRouter(mux *router.Router) { // ListBuckets apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler) - - mux.PathPrefix("/").Handler(negroni.New( - // Validates all incoming requests to have a valid date header. - negroni.Wrap(timeValidityHandler{}), - // Route requests - negroni.Wrap(apiRouter), - )) - } diff --git a/cmd/benchmark-utils_test.go b/cmd/benchmark-utils_test.go index bf05a2489..c2b071504 100644 --- a/cmd/benchmark-utils_test.go +++ b/cmd/benchmark-utils_test.go @@ -51,7 +51,7 @@ func prepareBenchmarkBackend(instanceType string) (ObjectLayer, []string, error) return nil, nil, err } // initialize object layer. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { return nil, nil, err } diff --git a/cmd/erasure-readfile_test.go b/cmd/erasure-readfile_test.go index 8a2e4e3cd..7de1a99af 100644 --- a/cmd/erasure-readfile_test.go +++ b/cmd/erasure-readfile_test.go @@ -226,7 +226,7 @@ func TestErasureReadUtils(t *testing.T) { if err != nil { t.Fatal(err) } - objLayer, _, err := initObjectLayer(endpoints, nil) + objLayer, _, err := initObjectLayer(endpoints) if err != nil { removeRoots(disks) t.Fatal(err) diff --git a/cmd/event-notifier_test.go b/cmd/event-notifier_test.go index 0a65e105c..a662e439e 100644 --- a/cmd/event-notifier_test.go +++ b/cmd/event-notifier_test.go @@ -43,7 +43,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) { if err != nil { t.Fatal(err) } - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } @@ -97,7 +97,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) { if err != nil { t.Fatal(err) } - fs, _, err := initObjectLayer(endpoints, nil) + fs, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } @@ -128,7 +128,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) { if err != nil { t.Fatal(err) } - fs, _, err := initObjectLayer(endpoints, nil) + fs, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } @@ -159,7 +159,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) { if err != nil { t.Fatal(err) } - fs, _, err := initObjectLayer(endpoints, nil) + fs, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal("Unable to initialize FS backend.", err) } diff --git a/cmd/format-config-v1_test.go b/cmd/format-config-v1_test.go index c5689354c..0790e1cee 100644 --- a/cmd/format-config-v1_test.go +++ b/cmd/format-config-v1_test.go @@ -280,7 +280,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Error(err) } @@ -316,7 +316,7 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Error(err) } @@ -605,7 +605,7 @@ func TestInitFormatXLErrors(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -711,7 +711,7 @@ func TestLoadFormatXLErrs(t *testing.T) { t.Fatal(err) } // Create an instance of xl backend. - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -741,7 +741,7 @@ func TestLoadFormatXLErrs(t *testing.T) { if err != nil { t.Fatal(err) } - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -769,7 +769,7 @@ func TestLoadFormatXLErrs(t *testing.T) { if err != nil { t.Fatal(err) } - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -795,7 +795,7 @@ func TestLoadFormatXLErrs(t *testing.T) { if err != nil { t.Fatal(err) } - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -824,7 +824,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Everything is fine, should return nil - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -846,7 +846,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Disks 0..15 are nil - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -870,7 +870,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // One disk returns Faulty Disk - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -896,7 +896,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // One disk is not found, heal corrupted disks should return nil - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -918,7 +918,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -944,7 +944,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) { } // Corrupted format json in one disk - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -974,7 +974,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Everything is fine, should return nil - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -995,7 +995,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Disks 0..15 are nil - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -1019,7 +1019,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // One disk returns Faulty Disk - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -1045,7 +1045,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // One disk is not found, heal corrupted disks should return nil - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -1067,7 +1067,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } @@ -1093,7 +1093,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) { } // Remove format.json of all disks - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal(err) } diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index a7503a0c8..cd64e4249 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -18,10 +18,8 @@ package cmd import ( "encoding/json" - "os" "path" "sort" - "strings" ) const ( @@ -164,32 +162,3 @@ func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) b } return true } - -var extendedHeaders = []string{ - "X-Amz-Meta-", - "X-Minio-Meta-", - // Add new extended headers. -} - -// isExtendedHeader validates if input string matches extended headers. -func isExtendedHeader(header string) bool { - for _, extendedHeader := range extendedHeaders { - if strings.HasPrefix(header, extendedHeader) { - return true - } - } - return false -} - -// Return true if extended HTTP headers are set, false otherwise. -func hasExtendedHeader(metadata map[string]string) bool { - if os.Getenv("MINIO_ENABLE_FSMETA") == "1" { - return true - } - for k := range metadata { - if isExtendedHeader(k) { - return true - } - } - return false -} diff --git a/cmd/fs-v1-metadata_test.go b/cmd/fs-v1-metadata_test.go index df404746a..e7d71c91c 100644 --- a/cmd/fs-v1-metadata_test.go +++ b/cmd/fs-v1-metadata_test.go @@ -23,56 +23,12 @@ import ( "testing" ) -// Tests scenarios which can occur for hasExtendedHeader function. -func TestHasExtendedHeader(t *testing.T) { - // All test cases concerning hasExtendedHeader function. - testCases := []struct { - metadata map[string]string - has bool - }{ - // Verifies if X-Amz-Meta is present. - { - metadata: map[string]string{ - "X-Amz-Meta-1": "value", - }, - has: true || os.Getenv("MINIO_ENABLE_FSMETA") == "1", - }, - // Verifies if X-Minio-Meta is present. - { - metadata: map[string]string{ - "X-Minio-Meta-1": "value", - }, - has: true || os.Getenv("MINIO_ENABLE_FSMETA") == "1", - }, - // Verifies if extended header is not present. - { - metadata: map[string]string{ - "md5Sum": "value", - }, - has: false || os.Getenv("MINIO_ENABLE_FSMETA") == "1", - }, - // Verifies if extended header is not present, but with an empty input. - { - metadata: nil, - has: false || os.Getenv("MINIO_ENABLE_FSMETA") == "1", - }, - } - - // Validate all test cases. - for i, testCase := range testCases { - has := hasExtendedHeader(testCase.metadata) - if has != testCase.has { - t.Fatalf("Test case %d: Expected \"%#v\", but got \"%#v\"", i+1, testCase.has, has) - } - } -} - func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) { endpoints, err := parseStorageEndpoints([]string{disk}) if err != nil { t.Fatal(err) } - obj, _, err = initObjectLayer(endpoints, nil) + obj, _, err = initObjectLayer(endpoints) if err != nil { t.Fatal("Unexpected err: ", err) } diff --git a/cmd/fs-v1-multipart.go b/cmd/fs-v1-multipart.go index 5523b3159..114b9b745 100644 --- a/cmd/fs-v1-multipart.go +++ b/cmd/fs-v1-multipart.go @@ -227,10 +227,8 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st // Initialize `fs.json` values. fsMeta := newFSMetaV1() - // Save additional metadata only if extended headers such as "X-Amz-Meta-" are set. - if hasExtendedHeader(meta) { - fsMeta.Meta = meta - } + // Save additional metadata. + fsMeta.Meta = meta // This lock needs to be held for any changes to the directory // contents of ".minio.sys/multipart/object/" @@ -765,18 +763,16 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload // No need to save part info, since we have concatenated all parts. fsMeta.Parts = nil - // Save additional metadata only if extended headers such as "X-Amz-Meta-" are set. - if hasExtendedHeader(fsMeta.Meta) { - if len(fsMeta.Meta) == 0 { - fsMeta.Meta = make(map[string]string) - } - fsMeta.Meta["md5Sum"] = s3MD5 + // Save additional metadata. + if len(fsMeta.Meta) == 0 { + fsMeta.Meta = make(map[string]string) + } + fsMeta.Meta["md5Sum"] = s3MD5 - fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) - // Write the metadata to a temp file and rename it to the actual location. - if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { - return "", toObjectErr(err, bucket, object) - } + fsMetaPath = path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) + // Write the metadata to a temp file and rename it to the actual location. + if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { + return "", toObjectErr(err, bucket, object) } // Cleanup all the parts if everything else has been safely committed. diff --git a/cmd/fs-v1.go b/cmd/fs-v1.go index 9b1fba85d..369a5d8c3 100644 --- a/cmd/fs-v1.go +++ b/cmd/fs-v1.go @@ -455,17 +455,15 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io. return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } - // Save additional metadata only if extended headers such as "X-Amz-Meta-" are set. - if hasExtendedHeader(metadata) { - // Initialize `fs.json` values. - fsMeta := newFSMetaV1() - fsMeta.Meta = metadata - - fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) - if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { - return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) - } + // Save additional metadata. Initialize `fs.json` values. + fsMeta := newFSMetaV1() + fsMeta.Meta = metadata + + fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile) + if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil { + return ObjectInfo{}, toObjectErr(traceError(err), bucket, object) } + objInfo, err = fs.getObjectInfo(bucket, object) if err == nil { // If MINIO_ENABLE_FSMETA is not enabled objInfo.MD5Sum will be empty. diff --git a/cmd/fs-v1_test.go b/cmd/fs-v1_test.go index 296ec4dc3..727110678 100644 --- a/cmd/fs-v1_test.go +++ b/cmd/fs-v1_test.go @@ -45,7 +45,7 @@ func TestNewFS(t *testing.T) { t.Fatal("Uexpected error: ", err) } - fsStorageDisks, err := initStorageDisks(endpoints, nil) + fsStorageDisks, err := initStorageDisks(endpoints) if err != nil { t.Fatal("Uexpected error: ", err) } @@ -55,7 +55,7 @@ func TestNewFS(t *testing.T) { t.Fatal("Uexpected error: ", err) } - xlStorageDisks, err := initStorageDisks(endpoints, nil) + xlStorageDisks, err := initStorageDisks(endpoints) if err != nil { t.Fatal("Uexpected error: ", err) } @@ -96,17 +96,22 @@ func TestNewFS(t *testing.T) { } } -// TestFSShutdown - initialize a new FS object layer then calls Shutdown -// to check returned results +// TestFSShutdown - initialize a new FS object layer then calls +// Shutdown to check returned results func TestFSShutdown(t *testing.T) { + rootPath, err := newTestConfig("us-east-1") + if err != nil { + t.Fatal(err) + } + defer removeAll(rootPath) + bucketName := "testbucket" + objectName := "object" // Create and return an fsObject with its path in the disk prepareTest := func() (fsObjects, string) { disk := filepath.Join(os.TempDir(), "minio-"+nextSuffix()) obj := initFSObjects(disk, t) fs := obj.(fsObjects) - bucketName := "testbucket" - objectName := "object" objectContent := "12345" obj.MakeBucket(bucketName) sha256sum := "" @@ -124,6 +129,7 @@ func TestFSShutdown(t *testing.T) { // Test Shutdown with faulty disk for i := 1; i <= 5; i++ { fs, disk := prepareTest() + fs.DeleteObject(bucketName, objectName) fsStorage := fs.storage.(*posix) fs.storage = newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil) if err := fs.Shutdown(); errorCause(err) != errFaultyDisk { diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 81f3dc704..1ae9a8c80 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -18,7 +18,6 @@ package cmd import ( "net/http" - "os" "path" "regexp" "strings" @@ -77,27 +76,24 @@ func setBrowserRedirectHandler(h http.Handler) http.Handler { } func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off") { - // Re-direction handled specifically for browsers. - if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && !isRequestSignatureV4(r) { - switch r.URL.Path { - case "/", "/webrpc", "/login", "/favicon.ico": - // '/' is redirected to 'locationPrefix/' - // '/webrpc' is redirected to 'locationPrefix/webrpc' - // '/login' is redirected to 'locationPrefix/login' - location := h.locationPrefix + r.URL.Path - // Redirect to new location. - http.Redirect(w, r, location, http.StatusTemporaryRedirect) - return - case h.locationPrefix: - // locationPrefix is redirected to 'locationPrefix/' - location := h.locationPrefix + "/" - http.Redirect(w, r, location, http.StatusTemporaryRedirect) - return - } + // Re-direction handled specifically for browsers. + if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && !isRequestSignatureV4(r) { + switch r.URL.Path { + case "/", "/webrpc", "/login", "/favicon.ico": + // '/' is redirected to 'locationPrefix/' + // '/webrpc' is redirected to 'locationPrefix/webrpc' + // '/login' is redirected to 'locationPrefix/login' + location := h.locationPrefix + r.URL.Path + // Redirect to new location. + http.Redirect(w, r, location, http.StatusTemporaryRedirect) + return + case h.locationPrefix: + // locationPrefix is redirected to 'locationPrefix/' + location := h.locationPrefix + "/" + http.Redirect(w, r, location, http.StatusTemporaryRedirect) + return } } - h.handler.ServeHTTP(w, r) } @@ -187,11 +183,19 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) { return time.Time{}, ErrMissingDateHeader } -type timeValidityHandler struct{} +type timeValidityHandler struct { + handler http.Handler +} + +// setTimeValidityHandler to validate parsable time over http header +func setTimeValidityHandler(h http.Handler) http.Handler { + return timeValidityHandler{h} +} func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Verify if date headers are set, if not reject the request - if _, ok := r.Header["Authorization"]; ok { + aType := getRequestAuthType(r) + if aType != authTypeAnonymous && aType != authTypeJWT { + // Verify if date headers are set, if not reject the request amzDate, apiErr := parseAmzDateHeader(r) if apiErr != ErrNone { // All our internal APIs are sensitive towards Date @@ -208,6 +212,7 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } } + h.handler.ServeHTTP(w, r) } type resourceHandler struct { diff --git a/cmd/globals.go b/cmd/globals.go index c307b2eaa..36702d1c1 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -41,14 +41,11 @@ const ( ) var ( - globalQuiet = false // Quiet flag set via command line + globalQuiet = false // Quiet flag set via command line. globalIsDistXL = false // "Is Distributed?" flag. // Add new global flags here. - // Maximum connections handled per - // server, defaults to 0 (unlimited). - globalMaxConn = 0 // Maximum cache size. globalMaxCacheSize = uint64(maxCacheSize) // Cache expiry. diff --git a/cmd/lock-rpc-server.go b/cmd/lock-rpc-server.go index df25d0f6d..d3a8f370b 100644 --- a/cmd/lock-rpc-server.go +++ b/cmd/lock-rpc-server.go @@ -83,11 +83,6 @@ func registerDistNSLockRouter(mux *router.Router, serverConfig serverCmdConfig) // Create one lock server for every local storage rpc server. func newLockServers(srvConfig serverCmdConfig) (lockServers []*lockServer) { for _, ep := range srvConfig.endpoints { - if containsEndpoint(srvConfig.ignoredEndpoints, ep) { - // Skip initializing ignored endpoint. - continue - } - // Not local storage move to the next node. if !isLocalStorage(ep) { continue diff --git a/cmd/lock-rpc-server_test.go b/cmd/lock-rpc-server_test.go index 230f6bb02..5cc55c840 100644 --- a/cmd/lock-rpc-server_test.go +++ b/cmd/lock-rpc-server_test.go @@ -475,7 +475,7 @@ func TestLockServers(t *testing.T) { }, totalLockServers: 1, }, - // Test - 2 two servers possible, 1 ignored. + // Test - 2 two servers possible. { isDistXL: true, srvCmdConfig: serverCmdConfig{ @@ -496,13 +496,8 @@ func TestLockServers(t *testing.T) { Host: "1.1.2.2:9000", Path: "/mnt/disk4", }}, - ignoredEndpoints: []*url.URL{{ - Scheme: "http", - Host: "localhost:9000", - Path: "/mnt/disk2", - }}, }, - totalLockServers: 1, + totalLockServers: 2, }, } diff --git a/cmd/main.go b/cmd/main.go index b832363ad..65e58894d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,7 +42,7 @@ var ( }, cli.BoolFlag{ Name: "quiet", - Usage: "Suppress chatty output.", + Usage: "Disable startup information.", }, } ) @@ -218,7 +218,7 @@ func Main() { } // Start profiler if env is set. - if profiler := os.Getenv("MINIO_PROFILER"); profiler != "" { + if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" { globalProfiler = startProfiler(profiler) } diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go index 52b8d1fc4..84c7578e5 100644 --- a/cmd/object-api-listobjects_test.go +++ b/cmd/object-api-listobjects_test.go @@ -566,13 +566,12 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) { // Initialize FS backend for the benchmark. func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) { - endPoints, err := parseStorageEndpoints([]string{disk}) if err != nil { t.Fatal("Unexpected err: ", err) } - obj, _, err = initObjectLayer(endPoints, nil) + obj, _, err = initObjectLayer(endPoints) if err != nil { t.Fatal("Unexpected err: ", err) } diff --git a/cmd/prepare-storage-msg.go b/cmd/prepare-storage-msg.go index 51d672913..e3bd03236 100644 --- a/cmd/prepare-storage-msg.go +++ b/cmd/prepare-storage-msg.go @@ -45,7 +45,11 @@ type printOnceFunc func(msg string) func printOnceFn() printOnceFunc { var once sync.Once return func(msg string) { - once.Do(func() { console.Println(msg) }) + once.Do(func() { + if !globalQuiet { + console.Println(msg) + } + }) } } diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index 266cbdca3..d1c26891b 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -266,7 +266,7 @@ func retryFormattingDisks(firstDisk bool, endpoints []*url.URL, storageDisks []S } // Initialize storage disks based on input arguments. -func initStorageDisks(endpoints, ignoredEndpoints []*url.URL) ([]StorageAPI, error) { +func initStorageDisks(endpoints []*url.URL) ([]StorageAPI, error) { // Single disk means we will use FS backend. if len(endpoints) == 1 { if endpoints[0] == nil { @@ -284,19 +284,6 @@ func initStorageDisks(endpoints, ignoredEndpoints []*url.URL) ([]StorageAPI, err if ep == nil { return nil, errInvalidArgument } - // Check if disk is ignored. - ignored := false - for _, iep := range ignoredEndpoints { - if *ep == *iep { - ignored = true - break - } - } - if ignored { - // Set this situation as disk not found. - storageDisks[index] = nil - continue - } // Intentionally ignore disk not found errors. XL is designed // to handle these errors internally. storage, err := newStorageAPI(ep) diff --git a/cmd/rate-limit-handler.go b/cmd/rate-limit-handler.go deleted file mode 100644 index 0a159825d..000000000 --- a/cmd/rate-limit-handler.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "errors" - "net/http" -) - -var errTooManyRequests = errors.New("Too many clients in the waiting list") - -// rateLimit - represents datatype of the functionality implemented to -// limit the number of concurrent http requests. -type rateLimit struct { - handler http.Handler - workQueue chan struct{} - waitQueue chan struct{} -} - -// acquire and release implement a way to send and receive from the -// channel this is in-turn used to rate limit incoming connections in -// ServeHTTP() http.Handler method. -func (c *rateLimit) acquire() error { - // attempt to enter the waitQueue. If no slot is immediately - // available return error. - select { - case c.waitQueue <- struct{}{}: - // entered wait queue - break - default: - // no slot available for waiting - return errTooManyRequests - } - - // block attempting to enter the workQueue. If the workQueue - // is full, there can be at most cap(waitQueue) == - // 4*globalMaxConn goroutines waiting here because of the - // select above. - select { - case c.workQueue <- struct{}{}: - // entered workQueue - so remove one waiter. This step - // does not block as the waitQueue cannot be empty. - <-c.waitQueue - } - - return nil -} - -// Release one element from workQueue to serve a new client in the -// waiting list -func (c *rateLimit) release() { - <-c.workQueue -} - -// ServeHTTP is an http.Handler ServeHTTP method, implemented to rate -// limit incoming HTTP requests. -func (c *rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // Acquire the connection if queue is not full, otherwise - // code path waits here until the previous case is true. - if err := c.acquire(); err != nil { - w.WriteHeader(http.StatusTooManyRequests) - return - } - - // Serves the request. - c.handler.ServeHTTP(w, r) - - // Release - c.release() -} - -// setRateLimitHandler limits the number of concurrent http requests -// based on MINIO_MAXCONN. -func setRateLimitHandler(handler http.Handler) http.Handler { - if globalMaxConn == 0 { - return handler - } // else proceed to rate limiting. - - // For max connection limit of > '0' we initialize rate limit - // handler. - return &rateLimit{ - handler: handler, - workQueue: make(chan struct{}, globalMaxConn), - waitQueue: make(chan struct{}, globalMaxConn*4), - } -} diff --git a/cmd/rate-limit-handler_test.go b/cmd/rate-limit-handler_test.go deleted file mode 100644 index feccc7a02..000000000 --- a/cmd/rate-limit-handler_test.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Minio Cloud Storage, (C) 2016 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cmd - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" -) - -// This test sets globalMaxConn to 1 and starts 6 connections in -// parallel on a server with the rate limit handler configured. This -// should allow one request to execute at a time, and at most 4 to -// wait to execute and the 6th request should get a 429 status code -// error. -func TestRateLimitHandler(t *testing.T) { - // save the global Max connections - saveGlobalMaxConn := globalMaxConn - - globalMaxConn = 1 - testHandler := func(w http.ResponseWriter, r *http.Request) { - time.Sleep(100 * time.Millisecond) - fmt.Fprintln(w, "Hello client!") - } - rlh := setRateLimitHandler(http.HandlerFunc(testHandler)) - ts := httptest.NewServer(rlh) - respCh := make(chan int) - startTime := time.Now() - for i := 0; i < 6; i++ { - go func(ch chan<- int) { - resp, err := http.Get(ts.URL) - if err != nil { - t.Errorf( - "Got error requesting test server - %v\n", - err, - ) - } - respCh <- resp.StatusCode - }(respCh) - } - - tooManyReqErrCount := 0 - for i := 0; i < 6; i++ { - code := <-respCh - if code == 429 { - tooManyReqErrCount++ - } else if code != 200 { - t.Errorf("Got non-200 resp code - %d\n", code) - } - } - duration := time.Since(startTime) - if duration < time.Duration(500*time.Millisecond) { - // as globalMaxConn is 1, only 1 request will execute - // at a time, and the five allowed requested will take - // at least 500 ms. - t.Errorf("Expected all requests to take at least 500ms, but it was done in %v\n", - duration) - } - if tooManyReqErrCount != 1 { - t.Errorf("Expected to get 1 error, but got %d", - tooManyReqErrCount) - } - ts.Close() - - // restore the global Max connections - globalMaxConn = saveGlobalMaxConn -} diff --git a/cmd/routers.go b/cmd/routers.go index 5ce1c5fed..14209c028 100644 --- a/cmd/routers.go +++ b/cmd/routers.go @@ -18,8 +18,6 @@ package cmd import ( "net/http" - "os" - "strings" router "github.com/gorilla/mux" ) @@ -109,17 +107,13 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) return nil, err } - // set environmental variable MINIO_BROWSER=off to disable minio web browser. - // By default minio web browser is enabled. - if !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off") { - // Register RPC router for web related calls. - if err = registerBrowserPeerRPCRouter(mux); err != nil { - return nil, err - } + // Register RPC router for web related calls. + if err = registerBrowserPeerRPCRouter(mux); err != nil { + return nil, err + } - if err = registerWebRouter(mux); err != nil { - return nil, err - } + if err = registerWebRouter(mux); err != nil { + return nil, err } // Add API router. @@ -127,8 +121,6 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) // List of some generic handlers which are applied for all incoming requests. var handlerFns = []HandlerFunc{ - // Limits the number of concurrent http requests. - setRateLimitHandler, // Limits all requests size to a maximum fixed limit setRequestSizeLimitHandler, // Adds 'crossdomain.xml' policy handler to serve legacy flash clients. @@ -139,6 +131,8 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error) setPrivateBucketHandler, // Adds cache control for all browser requests. setBrowserCacheControlHandler, + // Validates all incoming requests to have a valid date header. + setTimeValidityHandler, // CORS setting for all browser API requests. setCorsHandler, // Validates all incoming URL resources, for invalid/unsupported diff --git a/cmd/server-main.go b/cmd/server-main.go index 453eae681..593708044 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -22,9 +22,7 @@ import ( "net/http" "net/url" "os" - "strconv" "strings" - "time" "regexp" "runtime" @@ -38,10 +36,6 @@ var serverFlags = []cli.Flag{ Value: ":9000", Usage: `Bind to a specific IP:PORT. Defaults to ":9000".`, }, - cli.StringFlag{ - Name: "ignore-disks", - Usage: `Comma separated list of faulty drives to ignore at startup.`, - }, } var serverCmd = cli.Command{ @@ -63,13 +57,6 @@ ENVIRONMENT VARIABLES: MINIO_ACCESS_KEY: Username or access key of 5 to 20 characters in length. MINIO_SECRET_KEY: Password or secret key of 8 to 40 characters in length. - CACHING: - MINIO_CACHE_SIZE: Limit maximum cache size. Allowed units are [GB|MB|KB]. Defaults to 8GB. - MINIO_CACHE_EXPIRY: Automatically expire cached objects. Allowed units are [h|m|s]. Defaults to 72h. - - SECURITY: - MINIO_SECURE_CONSOLE: Set secure console to 'no' to disable printing secret key. Defaults to 'yes'. - EXAMPLES: 1. Start minio server on "/home/shared" directory. $ minio {{.Name}} /home/shared @@ -92,10 +79,9 @@ EXAMPLES: } type serverCmdConfig struct { - serverAddr string - endpoints []*url.URL - ignoredEndpoints []*url.URL - storageDisks []StorageAPI + serverAddr string + endpoints []*url.URL + storageDisks []StorageAPI } // Parse an array of end-points (from the command line) @@ -190,27 +176,6 @@ func initServerConfig(c *cli.Context) { err := createCertsPath() fatalIf(err, "Unable to create \"certs\" directory.") - // Fetch max conn limit from environment variable. - if maxConnStr := os.Getenv("MINIO_MAXCONN"); maxConnStr != "" { - // We need to parse to its integer value. - globalMaxConn, err = strconv.Atoi(maxConnStr) - fatalIf(err, "Unable to convert MINIO_MAXCONN=%s environment variable into its integer value.", maxConnStr) - } - - // Fetch max cache size from environment variable. - if maxCacheSizeStr := os.Getenv("MINIO_CACHE_SIZE"); maxCacheSizeStr != "" { - // We need to parse cache size to its integer value. - globalMaxCacheSize, err = strconvBytes(maxCacheSizeStr) - fatalIf(err, "Unable to convert MINIO_CACHE_SIZE=%s environment variable into its integer value.", maxCacheSizeStr) - } - - // Fetch cache expiry from environment variable. - if cacheExpiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); cacheExpiryStr != "" { - // We need to parse cache expiry to its time.Duration value. - globalCacheExpiry, err = time.ParseDuration(cacheExpiryStr) - fatalIf(err, "Unable to convert MINIO_CACHE_EXPIRY=%s environment variable into its time.Duration value.", cacheExpiryStr) - } - // When credentials inherited from the env, server cmd has to save them in the disk if os.Getenv("MINIO_ACCESS_KEY") != "" && os.Getenv("MINIO_SECRET_KEY") != "" { // Env credentials are already loaded in serverConfig, just save in the disk @@ -332,26 +297,6 @@ func checkServerSyntax(c *cli.Context) { fatalIf(err, "Storage endpoint error.") } - // Verify syntax for all the ignored disks. - var ignoredEndpoints []*url.URL - ignoredDisksStr := c.String("ignore-disks") - if ignoredDisksStr != "" { - ignoredDisks := strings.Split(ignoredDisksStr, ",") - if len(endpoints) == 1 { - fatalIf(errInvalidArgument, "--ignore-disks is valid only for XL setup.") - } - ignoredEndpoints, err = parseStorageEndpoints(ignoredDisks) - fatalIf(err, "Unable to parse ignored storage endpoints %s", ignoredDisks) - checkEndpointsSyntax(ignoredEndpoints, ignoredDisks) - - for i, ep := range ignoredEndpoints { - // An ignored disk should be present in the XL disks list. - if !containsEndpoint(endpoints, ep) { - fatalIf(errInvalidArgument, "Ignored storage %s not available in the list of erasure storages list.", disks[i]) - } - } - } - if !isDistributedSetup(endpoints) { // for FS and singlenode-XL validation is done, return. return @@ -412,18 +357,11 @@ func serverMain(c *cli.Context) { // depends on it. checkServerSyntax(c) - // Disks to be ignored in server init, to skip format healing. - var ignoredEndpoints []*url.URL - if len(c.String("ignore-disks")) > 0 { - ignoredEndpoints, err = parseStorageEndpoints(strings.Split(c.String("ignore-disks"), ",")) - fatalIf(err, "Unable to parse storage endpoints %s", strings.Split(c.String("ignore-disks"), ",")) - } - // Disks to be used in server init. endpoints, err := parseStorageEndpoints(c.Args()) fatalIf(err, "Unable to parse storage endpoints %s", c.Args()) - storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints) + storageDisks, err := initStorageDisks(endpoints) fatalIf(err, "Unable to initialize storage disks.") // Cleanup objects that weren't successfully written into the namespace. @@ -440,10 +378,9 @@ func serverMain(c *cli.Context) { // Configure server. srvConfig := serverCmdConfig{ - serverAddr: serverAddr, - endpoints: endpoints, - ignoredEndpoints: ignoredEndpoints, - storageDisks: storageDisks, + serverAddr: serverAddr, + endpoints: endpoints, + storageDisks: storageDisks, } // Configure server. diff --git a/cmd/server-main_test.go b/cmd/server-main_test.go index 3ba1402ef..4d0463971 100644 --- a/cmd/server-main_test.go +++ b/cmd/server-main_test.go @@ -258,7 +258,7 @@ func TestCheckServerSyntax(t *testing.T) { t.Fatalf("Test %d : Unexpected error %s", i+1, err) } checkEndpointsSyntax(endpoints, disks) - _, err = initStorageDisks(endpoints, nil) + _, err = initStorageDisks(endpoints) if err != nil { t.Errorf("Test %d : disk init failed : %s", i+1, err) } @@ -336,9 +336,6 @@ func TestInitServerConfig(t *testing.T) { envVar string val string }{ - {"MINIO_MAXCONN", "10"}, - {"MINIO_CACHE_SIZE", "42MB"}, - {"MINIO_CACHE_EXPIRY", "2h45m"}, {"MINIO_ACCESS_KEY", "abcd1"}, {"MINIO_SECRET_KEY", "abcd12345"}, } diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go index ccb3b9180..8e0e96742 100644 --- a/cmd/server-startup-msg.go +++ b/cmd/server-startup-msg.go @@ -19,7 +19,6 @@ package cmd import ( "crypto/x509" "fmt" - "os" "runtime" "strings" "time" @@ -45,6 +44,10 @@ func getFormatStr(strLen int, padding int) string { // Prints the formatted startup message. func printStartupMessage(endPoints []string) { + // If quiet flag is set do not print startup message. + if globalQuiet { + return + } printServerCommonMsg(endPoints) printCLIAccessMsg(endPoints[0]) printObjectAPIMsg() @@ -73,11 +76,7 @@ func printServerCommonMsg(endPoints []string) { // Colorize the message and print. console.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(endPointStr), 1), endPointStr))) console.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKeyID))) - secretKey := cred.SecretAccessKey - if strings.EqualFold(os.Getenv("MINIO_SECURE_CONSOLE"), "no") { - secretKey = "*REDACTED*" - } - console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", secretKey))) + console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretAccessKey))) console.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region))) printEventNotifiers() @@ -109,15 +108,11 @@ func printCLIAccessMsg(endPoint string) { // Configure 'mc', following block prints platform specific information for minio client. console.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide) - secretKey := cred.SecretAccessKey - if os.Getenv("MINIO_SECURE_CONSOLE") == "0" { - secretKey = "*REDACTED*" - } if runtime.GOOS == "windows" { - mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKeyID, secretKey) + mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKeyID, cred.SecretAccessKey) console.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } else { - mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKeyID, secretKey) + mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKeyID, cred.SecretAccessKey) console.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) } } diff --git a/cmd/storage-rpc-server.go b/cmd/storage-rpc-server.go index 6c81b7dd2..8f4eec28b 100644 --- a/cmd/storage-rpc-server.go +++ b/cmd/storage-rpc-server.go @@ -227,10 +227,6 @@ func (s *storageServer) TryInitHandler(args *GenericArgs, reply *GenericReply) e // Initialize new storage rpc. func newRPCServer(srvConfig serverCmdConfig) (servers []*storageServer, err error) { for _, ep := range srvConfig.endpoints { - if containsEndpoint(srvConfig.ignoredEndpoints, ep) { - // Do not init disk RPC for ignored end point. - continue - } // e.g server:/mnt/disk1 if isLocalStorage(ep) { // Get the posix path. diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go index d1a22277f..edc649cd9 100644 --- a/cmd/test-utils_test.go +++ b/cmd/test-utils_test.go @@ -66,7 +66,7 @@ func prepareFS() (ObjectLayer, string, error) { if err != nil { return nil, "", err } - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { removeRoots(fsDirs) return nil, "", err @@ -84,7 +84,7 @@ func prepareXL() (ObjectLayer, []string, error) { if err != nil { return nil, nil, err } - obj, _, err := initObjectLayer(endpoints, nil) + obj, _, err := initObjectLayer(endpoints) if err != nil { removeRoots(fsDirs) return nil, nil, err @@ -189,7 +189,7 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer { testServer.AccessKey = credentials.AccessKeyID testServer.SecretKey = credentials.SecretAccessKey - objLayer, storageDisks, err := initObjectLayer(testServer.Disks, nil) + objLayer, storageDisks, err := initObjectLayer(testServer.Disks) if err != nil { t.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -385,7 +385,7 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer { testRPCServer.SecretKey = credentials.SecretAccessKey // create temporary backend for the test server. - objLayer, storageDisks, err := initObjectLayer(endpoints, nil) + objLayer, storageDisks, err := initObjectLayer(endpoints) if err != nil { t.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -459,7 +459,7 @@ func StartTestControlRPCServer(t TestErrHandler, instanceType string) TestServer testRPCServer.SecretKey = credentials.SecretAccessKey // create temporary backend for the test server. - objLayer, storageDisks, err := initObjectLayer(endpoints, nil) + objLayer, storageDisks, err := initObjectLayer(endpoints) if err != nil { t.Fatalf("Failed obtaining Temp Backend: %s", err) } @@ -1595,8 +1595,8 @@ func getRandomDisks(N int) ([]string, error) { } // initObjectLayer - Instantiates object layer and returns it. -func initObjectLayer(endpoints, ignoredEndpoints []*url.URL) (ObjectLayer, []StorageAPI, error) { - storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints) +func initObjectLayer(endpoints []*url.URL) (ObjectLayer, []StorageAPI, error) { + storageDisks, err := initStorageDisks(endpoints) if err != nil { return nil, nil, err } @@ -1677,7 +1677,7 @@ func prepareXLStorageDisks(t *testing.T) ([]StorageAPI, []string) { t.Fatal("Unexpected error: ", err) } - _, storageDisks, err := initObjectLayer(endpoints, nil) + _, storageDisks, err := initObjectLayer(endpoints) if err != nil { removeRoots(fsDirs) t.Fatal("Unable to initialize storage disks", err) @@ -1972,7 +1972,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType) if err != nil { t.Fatalf("Initialization of disks for XL setup: %s", err) } - objLayer, _, err := initObjectLayer(endpoints, nil) + objLayer, _, err := initObjectLayer(endpoints) if err != nil { t.Fatalf("Initialization of object layer failed for XL setup: %s", err) } diff --git a/cmd/utils.go b/cmd/utils.go index d6a26be14..94ab1dc29 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -23,7 +23,6 @@ import ( "io" "net/http" "net/url" - "os" "strings" "encoding/json" @@ -171,16 +170,14 @@ func urlPathSplit(urlPath string) (bucketName, prefixName string) { func startProfiler(profiler string) interface { Stop() } { - // Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted - profileDir := os.Getenv("MINIO_PROFILE_DIR") - // Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block]. + // Enable profiler if ``_MINIO_PROFILER`` is set. Supported options are [cpu, mem, block]. switch profiler { case "cpu": - return profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir)) + return profile.Start(profile.CPUProfile, profile.NoShutdownHook) case "mem": - return profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir)) + return profile.Start(profile.MemProfile, profile.NoShutdownHook) case "block": - return profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir)) + return profile.Start(profile.BlockProfile, profile.NoShutdownHook) default: return nil } diff --git a/cmd/utils_test.go b/cmd/utils_test.go index 5fb6be69c..80efbf61b 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -220,6 +220,13 @@ func TestMaxPartID(t *testing.T) { } } +// Add tests for starting and stopping different profilers. +func TestStartProfiler(t *testing.T) { + if startProfiler("") != nil { + t.Fatal("Expected nil, but non-nil value returned for invalid profiler.") + } +} + // Tests fetch local address. func TestLocalAddress(t *testing.T) { if runtime.GOOS == "windows" { diff --git a/cmd/xl-v1.go b/cmd/xl-v1.go index 4a9f2d580..e22e53b57 100644 --- a/cmd/xl-v1.go +++ b/cmd/xl-v1.go @@ -18,7 +18,9 @@ package cmd import ( "fmt" + "os" "sort" + "strings" "github.com/minio/minio/pkg/disk" "github.com/minio/minio/pkg/objcache" @@ -129,6 +131,9 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) { // Initialize list pool. listPool := newTreeWalkPool(globalLookupTimeout) + // Check if object cache is disabled. + objCacheDisabled := strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off") + // Initialize xl objects. xl := xlObjects{ storageDisks: newStorageDisks, @@ -136,7 +141,7 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) { parityBlocks: parityBlocks, listPool: listPool, objCache: objCache, - objCacheEnabled: globalMaxCacheSize > 0, + objCacheEnabled: !objCacheDisabled, } // Figure out read and write quorum based on number of storage disks. diff --git a/cmd/xl-v1_test.go b/cmd/xl-v1_test.go index 126ff59d2..011a237f7 100644 --- a/cmd/xl-v1_test.go +++ b/cmd/xl-v1_test.go @@ -56,12 +56,7 @@ func TestStorageInfo(t *testing.T) { t.Fatalf("Unexpected error %s", err) } - ignoredEndpoints, err := parseStorageEndpoints(fsDirs[:4]) - if err != nil { - t.Fatalf("Unexpected error %s", err) - } - - storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints) + storageDisks, err := initStorageDisks(endpoints) if err != nil { t.Fatal("Unexpected error: ", err) } @@ -156,7 +151,7 @@ func TestNewXL(t *testing.T) { t.Fatalf("Unable to initialize erasure, %s", err) } - storageDisks, err := initStorageDisks(endpoints, nil) + storageDisks, err := initStorageDisks(endpoints) if err != nil { t.Fatal("Unexpected error: ", err) } @@ -186,11 +181,7 @@ func TestNewXL(t *testing.T) { t.Fatalf("Unable to initialize erasure, %s", err) } - ignoredEndpoints, err := parseStorageEndpoints(erasureDisks[:2]) - if err != nil { - t.Fatalf("Unable to initialize erasure, %s", err) - } - storageDisks, err = initStorageDisks(endpoints, ignoredEndpoints) + storageDisks, err = initStorageDisks(endpoints) if err != nil { t.Fatal("Unexpected error: ", err) } diff --git a/docs/caching/README.md b/docs/caching/README.md index fddd3fbee..4c52a5054 100644 --- a/docs/caching/README.md +++ b/docs/caching/README.md @@ -2,36 +2,30 @@ Object caching by turned on by default with following settings - - Default cache size 8GB, can be changed from environment variable - ``MINIO_CACHE_SIZE`` supports both SI and ISO IEC standard forms - for input size parameters. + - Default cache size 8GB. Cache size also automatically picks + a lower value if your local memory size is lower than 8GB. - - Default expiration of entries is 72 hours, can be changed from - environment variable ``MINIO_CACHE_EXPIRY`` supportings Go - ``time.Duration`` with valid units "ns", "us" (or "µs"), - "ms", "s", "m", "h". + - Default expiration of entries happensat 72 hours, + this option cannot be changed. - - Default expiry interval is 1/4th of the expiration hours, so - expiration sweep happens across the cache every 1/4th the time - duration of the set entry expiration duration. - -### Tricks - -Setting MINIO_CACHE_SIZE=0 will turn off caching entirely. -Setting MINIO_CACHE_EXPIRY=0s will turn off cache garbage collections, -all cached objects will never expire. + - Default expiry interval is 1/4th of the expiration hours, so + expiration sweep happens across the cache every 1/4th the time + duration of the set entry expiration duration. ### Behavior -Caching happens for both GET and PUT. +Caching happens on both GET and PUT operations. -- GET caches new objects for entries not found in cache, -otherwise serves from the cache. +- GET caches new objects for entries not found in cache. - PUT/POST caches all successfully uploaded objects. -NOTE: Cache is not populated if there are any errors - while reading from the disk. +In all other cases if objects are served from cache. + +NOTE: + +Cache is always populated upon object is successfully +read from the disk. Expiration happens automatically based on the configured interval as explained above, frequently accessed objects diff --git a/docs/minio-env-variables.md b/docs/minio-env-variables.md deleted file mode 100644 index a40ccf635..000000000 --- a/docs/minio-env-variables.md +++ /dev/null @@ -1,47 +0,0 @@ -# Minio Environmental variables - -#### MINIO_ENABLE_FSMETA -When enabled, minio-FS saves the HTTP headers that start with `X-Amz-Meta-` and `X-Minio-Meta`. These header meta data can be retrieved on HEAD and GET requests on the object. - -#### MINIO_PROFILER -Used for Go profiling. Supported values are: - -`cpu` - for CPU profiling - -`mem` - for memory profiling - -`block` - for block profiling - -#### MINIO_PROFILE_DIR - -Path where cpu/mem/block profiling files are dumped - -#### MINIO_BROWSER - -setting this to `off` disables the minio browser. - -#### MINIO_ACCESS_KEY - -Minio access key. - -#### MINIO_SECRET_KEY - -Minio secret key. - -#### MINIO_CACHE_SIZE - -Set total cache size in NN[GB|MB|KB]. Defaults to 8GB - -Ex: MINIO_CACHE_SIZE=2GB - -#### MINIO_CACHE_EXPIRY - -Set the object cache expiration duration in NN[h|m|s]. Defaults to 72 hours. - -Ex. MINIO_CACHE_EXPIRY=24h - -#### MINIO_MAXCONN - -Limit of the number of concurrent http requests. - -Ex. MINIO_MAXCONN=500 diff --git a/vendor/github.com/urfave/negroni/CHANGELOG.md b/vendor/github.com/urfave/negroni/CHANGELOG.md deleted file mode 100644 index 5e77c3fb6..000000000 --- a/vendor/github.com/urfave/negroni/CHANGELOG.md +++ /dev/null @@ -1,35 +0,0 @@ -# Change Log - -**ATTN**: This project uses [semantic versioning](http://semver.org/). - -## [Unreleased] -### Added -- `Recovery.ErrorHandlerFunc` for custom error handling during recovery - -### Fixed -- `Written()` correct returns `false` if no response header has been written - -### Changed -- Set default status to `0` in the case that no handler writes status -- was - previously `200` (in 0.2.0, before that it was `0` so this reestablishes that - behavior) -- Catch `panic`s thrown by callbacks provided to the `Recovery` handler - -## [0.2.0] - 2016-05-10 -### Added -- Support for variadic handlers in `New()` -- Added `Negroni.Handlers()` to fetch all of the handlers for a given chain -- Allowed size in `Recovery` handler was bumped to 8k -- `Negroni.UseFunc` to push another handler onto the chain - -### Changed -- Set the status before calling `beforeFuncs` so the information is available to them -- Set default status to `200` in the case that no handler writes status -- was previously `0` -- Panic if `nil` handler is given to `negroni.Use` - -## 0.1.0 - 2013-07-22 -### Added -- Initial implementation. - -[Unreleased]: https://github.com/urfave/negroni/compare/v0.2.0...HEAD -[0.2.0]: https://github.com/urfave/negroni/compare/v0.1.0...v0.2.0 diff --git a/vendor/github.com/urfave/negroni/LICENSE b/vendor/github.com/urfave/negroni/LICENSE deleted file mode 100644 index 08b5e20ac..000000000 --- a/vendor/github.com/urfave/negroni/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Jeremy Saenz - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/urfave/negroni/README.md b/vendor/github.com/urfave/negroni/README.md deleted file mode 100644 index 6fa836410..000000000 --- a/vendor/github.com/urfave/negroni/README.md +++ /dev/null @@ -1,443 +0,0 @@ -# Negroni -[![GoDoc](https://godoc.org/github.com/urfave/negroni?status.svg)](http://godoc.org/github.com/urfave/negroni) -[![Build Status](https://travis-ci.org/urfave/negroni.svg?branch=master)](https://travis-ci.org/urfave/negroni) -[![codebeat](https://codebeat.co/badges/47d320b1-209e-45e8-bd99-9094bc5111e2)](https://codebeat.co/projects/github-com-urfave-negroni) - -**Notice:** This is the library formerly known as -`github.com/codegangsta/negroni` -- Github will automatically redirect requests -to this repository, but we recommend updating your references for clarity. - -Negroni is an idiomatic approach to web middleware in Go. It is tiny, -non-intrusive, and encourages use of `net/http` Handlers. - -If you like the idea of [Martini](https://github.com/go-martini/martini), but -you think it contains too much magic, then Negroni is a great fit. - -Language Translations: -* [German (de_DE)](translations/README_de_de.md) -* [Português Brasileiro (pt_BR)](translations/README_pt_br.md) -* [简体中文 (zh_cn)](translations/README_zh_cn.md) -* [繁體中文 (zh_tw)](translations/README_zh_tw.md) -* [日本語 (ja_JP)](translations/README_ja_JP.md) - -## Getting Started - -After installing Go and setting up your -[GOPATH](http://golang.org/doc/code.html#GOPATH), create your first `.go` file. -We'll call it `server.go`. - - -``` go -package main - -import ( - "fmt" - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome to the home page!") - }) - - n := negroni.Classic() // Includes some default middlewares - n.UseHandler(mux) - - http.ListenAndServe(":3000", n) -} -``` - -Then install the Negroni package (**NOTE**: >= **go 1.1** is required): - -``` -go get github.com/urfave/negroni -``` - -Then run your server: - -``` -go run server.go -``` - -You will now have a Go `net/http` webserver running on `localhost:3000`. - -## Is Negroni a Framework? - -Negroni is **not** a framework. It is a middleware-focused library that is -designed to work directly with `net/http`. - -## Routing? - -Negroni is BYOR (Bring your own Router). The Go community already has a number -of great http routers available, and Negroni tries to play well with all of them -by fully supporting `net/http`. For instance, integrating with [Gorilla Mux] -looks like so: - -``` go -router := mux.NewRouter() -router.HandleFunc("/", HomeHandler) - -n := negroni.New(Middleware1, Middleware2) -// Or use a middleware with the Use() function -n.Use(Middleware3) -// router goes last -n.UseHandler(router) - -http.ListenAndServe(":3001", n) -``` - -## `negroni.Classic()` - -`negroni.Classic()` provides some default middleware that is useful for most -applications: - -* [`negroni.Recovery`](#recovery) - Panic Recovery Middleware. -* [`negroni.Logger`](#logger) - Request/Response Logger Middleware. -* [`negroni.Static`](#static) - Static File serving under the "public" - directory. - -This makes it really easy to get started with some useful features from Negroni. - -## Handlers - -Negroni provides a bidirectional middleware flow. This is done through the -`negroni.Handler` interface: - -``` go -type Handler interface { - ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) -} -``` - -If a middleware hasn't already written to the `ResponseWriter`, it should call -the next `http.HandlerFunc` in the chain to yield to the next middleware -handler. This can be used for great good: - -``` go -func MyMiddleware(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - // do some stuff before - next(rw, r) - // do some stuff after -} -``` - -And you can map it to the handler chain with the `Use` function: - -``` go -n := negroni.New() -n.Use(negroni.HandlerFunc(MyMiddleware)) -``` - -You can also map plain old `http.Handler`s: - -``` go -n := negroni.New() - -mux := http.NewServeMux() -// map your routes - -n.UseHandler(mux) - -http.ListenAndServe(":3000", n) -``` - -## `Run()` - -Negroni has a convenience function called `Run`. `Run` takes an addr string -identical to [`http.ListenAndServe`](https://godoc.org/net/http#ListenAndServe). - - -``` go -package main - -import ( - "github.com/urfave/negroni" -) - -func main() { - n := negroni.Classic() - n.Run(":8080") -} -``` - -In general, you will want to use `net/http` methods and pass `negroni` as a -`Handler`, as this is more flexible, e.g.: - - -``` go -package main - -import ( - "fmt" - "log" - "net/http" - "time" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome to the home page!") - }) - - n := negroni.Classic() // Includes some default middlewares - n.UseHandler(mux) - - s := &http.Server{ - Addr: ":8080", - Handler: n, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - } - log.Fatal(s.ListenAndServe()) -} -``` - -## Route Specific Middleware - -If you have a route group of routes that need specific middleware to be -executed, you can simply create a new Negroni instance and use it as your route -handler. - -``` go -router := mux.NewRouter() -adminRoutes := mux.NewRouter() -// add admin routes here - -// Create a new negroni for the admin middleware -router.PathPrefix("/admin").Handler(negroni.New( - Middleware1, - Middleware2, - negroni.Wrap(adminRoutes), -)) -``` - -If you are using [Gorilla Mux], here is an example using a subrouter: - -``` go -router := mux.NewRouter() -subRouter := mux.NewRouter().PathPrefix("/subpath").Subrouter().StrictSlash(true) -subRouter.HandleFunc("/", someSubpathHandler) // "/subpath/" -subRouter.HandleFunc("/:id", someSubpathHandler) // "/subpath/:id" - -// "/subpath" is necessary to ensure the subRouter and main router linkup -router.PathPrefix("/subpath").Handler(negroni.New( - Middleware1, - Middleware2, - negroni.Wrap(subRouter), -)) -``` - -## Bundled Middleware - -### Static - -This middleware will serve files on the filesystem. If the files do not exist, -it proxies the request to the next middleware. If you want the requests for -non-existent files to return a `404 File Not Found` to the user you should look -at using [http.FileServer](https://golang.org/pkg/net/http/#FileServer) as -a handler. - -Example: - - -``` go -package main - -import ( - "fmt" - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome to the home page!") - }) - - // Example of using a http.FileServer if you want "server-like" rather than "middleware" behavior - // mux.Handle("/public", http.FileServer(http.Dir("/home/public"))) - - n := negroni.New() - n.Use(negroni.NewStatic(http.Dir("/tmp"))) - n.UseHandler(mux) - - http.ListenAndServe(":3002", n) -} -``` - -Will serve files from the `/tmp` directory first, but proxy calls to the next -handler if the request does not match a file on the filesystem. - -### Recovery - -This middleware catches `panic`s and responds with a `500` response code. If -any other middleware has written a response code or body, this middleware will -fail to properly send a 500 to the client, as the client has already received -the HTTP response code. Additionally, an `ErrorHandlerFunc` can be attached -to report 500's to an error reporting service such as Sentry or Airbrake. - -Example: - - -``` go -package main - -import ( - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - panic("oh no") - }) - - n := negroni.New() - n.Use(negroni.NewRecovery()) - n.UseHandler(mux) - - http.ListenAndServe(":3003", n) -} -``` - -Will return a `500 Internal Server Error` to each request. It will also log the -stack traces as well as print the stack trace to the requester if `PrintStack` -is set to `true` (the default). - -Example with error handler: - -``` go -package main - -import ( - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - panic("oh no") - }) - - n := negroni.New() - recovery := negroni.NewRecovery() - recovery.ErrorHandlerFunc = reportToSentry - n.Use(recovery) - n.UseHandler(mux) - - http.ListenAndServe(":3003", n) -} - -func reportToSentry(error interface{}) { - // write code here to report error to Sentry -} -``` - - -## Logger - -This middleware logs each incoming request and response. - -Example: - - -``` go -package main - -import ( - "fmt" - "net/http" - - "github.com/urfave/negroni" -) - -func main() { - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "Welcome to the home page!") - }) - - n := negroni.New() - n.Use(negroni.NewLogger()) - n.UseHandler(mux) - - http.ListenAndServe(":3004", n) -} -``` - -Will print a log similar to: - -``` -[negroni] Started GET / -[negroni] Completed 200 OK in 145.446µs -``` - -on each request. - -## Third Party Middleware - -Here is a current list of Negroni compatible middlware. Feel free to put up a PR -linking your middleware if you have built one: - -| Middleware | Author | Description | -| -----------|--------|-------------| -| [binding](https://github.com/mholt/binding) | [Matt Holt](https://github.com/mholt) | Data binding from HTTP requests into structs | -| [cloudwatch](https://github.com/cvillecsteele/negroni-cloudwatch) | [Colin Steele](https://github.com/cvillecsteele) | AWS cloudwatch metrics middleware | -| [cors](https://github.com/rs/cors) | [Olivier Poitrey](https://github.com/rs) | [Cross Origin Resource Sharing](http://www.w3.org/TR/cors/) (CORS) support | -| [csp](https://github.com/awakenetworks/csp) | [Awake Networks](https://github.com/awakenetworks) | [Content Security Policy](https://www.w3.org/TR/CSP2/) (CSP) support | -| [delay](https://github.com/jeffbmartinez/delay) | [Jeff Martinez](https://github.com/jeffbmartinez) | Add delays/latency to endpoints. Useful when testing effects of high latency | -| [New Relic Go Agent](https://github.com/yadvendar/negroni-newrelic-go-agent) | [Yadvendar Champawat](https://github.com/yadvendar) | Official [New Relic Go Agent](https://github.com/newrelic/go-agent) (currently in beta) | -| [gorelic](https://github.com/jingweno/negroni-gorelic) | [Jingwen Owen Ou](https://github.com/jingweno) | New Relic agent for Go runtime | -| [Graceful](https://github.com/tylerb/graceful) | [Tyler Bunnell](https://github.com/tylerb) | Graceful HTTP Shutdown | -| [gzip](https://github.com/phyber/negroni-gzip) | [phyber](https://github.com/phyber) | GZIP response compression | -| [JWT Middleware](https://github.com/auth0/go-jwt-middleware) | [Auth0](https://github.com/auth0) | Middleware checks for a JWT on the `Authorization` header on incoming requests and decodes it| -| [logrus](https://github.com/meatballhat/negroni-logrus) | [Dan Buch](https://github.com/meatballhat) | Logrus-based logger | -| [oauth2](https://github.com/goincremental/negroni-oauth2) | [David Bochenski](https://github.com/bochenski) | oAuth2 middleware | -| [onthefly](https://github.com/xyproto/onthefly) | [Alexander Rødseth](https://github.com/xyproto) | Generate TinySVG, HTML and CSS on the fly | -| [permissions2](https://github.com/xyproto/permissions2) | [Alexander Rødseth](https://github.com/xyproto) | Cookies, users and permissions | -| [prometheus](https://github.com/zbindenren/negroni-prometheus) | [Rene Zbinden](https://github.com/zbindenren) | Easily create metrics endpoint for the [prometheus](http://prometheus.io) instrumentation tool | -| [render](https://github.com/unrolled/render) | [Cory Jacobsen](https://github.com/unrolled) | Render JSON, XML and HTML templates | -| [RestGate](https://github.com/pjebs/restgate) | [Prasanga Siripala](https://github.com/pjebs) | Secure authentication for REST API endpoints | -| [secure](https://github.com/unrolled/secure) | [Cory Jacobsen](https://github.com/unrolled) | Middleware that implements a few quick security wins | -| [sessions](https://github.com/goincremental/negroni-sessions) | [David Bochenski](https://github.com/bochenski) | Session Management | -| [stats](https://github.com/thoas/stats) | [Florent Messa](https://github.com/thoas) | Store information about your web application (response time, etc.) | -| [VanGoH](https://github.com/auroratechnologies/vangoh) | [Taylor Wrobel](https://github.com/twrobel3) | Configurable [AWS-Style](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html) HMAC authentication middleware | -| [xrequestid](https://github.com/pilu/xrequestid) | [Andrea Franz](https://github.com/pilu) | Middleware that assigns a random X-Request-Id header to each request | -| [mgo session](https://github.com/joeljames/nigroni-mgo-session) | [Joel James](https://github.com/joeljames) | Middleware that handles creating and closing mgo sessions per request | - -## Examples - -[Alexander Rødseth](https://github.com/xyproto) created -[mooseware](https://github.com/xyproto/mooseware), a skeleton for writing a -Negroni middleware handler. - -## Live code reload? - -[gin](https://github.com/codegangsta/gin) and -[fresh](https://github.com/pilu/fresh) both live reload negroni apps. - -## Essential Reading for Beginners of Go & Negroni - -* [Using a Context to pass information from middleware to end handler](http://elithrar.github.io/article/map-string-interface/) -* [Understanding middleware](https://mattstauffer.co/blog/laravel-5.0-middleware-filter-style) - -## About - -Negroni is obsessively designed by none other than the [Code -Gangsta](https://codegangsta.io/) - -[Gorilla Mux]: https://github.com/gorilla/mux -[`http.FileSystem`]: https://godoc.org/net/http#FileSystem diff --git a/vendor/github.com/urfave/negroni/doc.go b/vendor/github.com/urfave/negroni/doc.go deleted file mode 100644 index add1ed9f7..000000000 --- a/vendor/github.com/urfave/negroni/doc.go +++ /dev/null @@ -1,25 +0,0 @@ -// Package negroni is an idiomatic approach to web middleware in Go. It is tiny, non-intrusive, and encourages use of net/http Handlers. -// -// If you like the idea of Martini, but you think it contains too much magic, then Negroni is a great fit. -// -// For a full guide visit http://github.com/urfave/negroni -// -// package main -// -// import ( -// "github.com/urfave/negroni" -// "net/http" -// "fmt" -// ) -// -// func main() { -// mux := http.NewServeMux() -// mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { -// fmt.Fprintf(w, "Welcome to the home page!") -// }) -// -// n := negroni.Classic() -// n.UseHandler(mux) -// n.Run(":3000") -// } -package negroni diff --git a/vendor/github.com/urfave/negroni/logger.go b/vendor/github.com/urfave/negroni/logger.go deleted file mode 100644 index 04cd53b56..000000000 --- a/vendor/github.com/urfave/negroni/logger.go +++ /dev/null @@ -1,35 +0,0 @@ -package negroni - -import ( - "log" - "net/http" - "os" - "time" -) - -// ALogger interface -type ALogger interface { - Println(v ...interface{}) - Printf(format string, v ...interface{}) -} - -// Logger is a middleware handler that logs the request as it goes in and the response as it goes out. -type Logger struct { - // ALogger implements just enough log.Logger interface to be compatible with other implementations - ALogger -} - -// NewLogger returns a new Logger instance -func NewLogger() *Logger { - return &Logger{log.New(os.Stdout, "[negroni] ", 0)} -} - -func (l *Logger) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - start := time.Now() - l.Printf("Started %s %s", r.Method, r.URL.Path) - - next(rw, r) - - res := rw.(ResponseWriter) - l.Printf("Completed %v %s in %v", res.Status(), http.StatusText(res.Status()), time.Since(start)) -} diff --git a/vendor/github.com/urfave/negroni/negroni.go b/vendor/github.com/urfave/negroni/negroni.go deleted file mode 100644 index 9c7c187e7..000000000 --- a/vendor/github.com/urfave/negroni/negroni.go +++ /dev/null @@ -1,133 +0,0 @@ -package negroni - -import ( - "log" - "net/http" - "os" -) - -// Handler handler is an interface that objects can implement to be registered to serve as middleware -// in the Negroni middleware stack. -// ServeHTTP should yield to the next middleware in the chain by invoking the next http.HandlerFunc -// passed in. -// -// If the Handler writes to the ResponseWriter, the next http.HandlerFunc should not be invoked. -type Handler interface { - ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) -} - -// HandlerFunc is an adapter to allow the use of ordinary functions as Negroni handlers. -// If f is a function with the appropriate signature, HandlerFunc(f) is a Handler object that calls f. -type HandlerFunc func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) - -func (h HandlerFunc) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - h(rw, r, next) -} - -type middleware struct { - handler Handler - next *middleware -} - -func (m middleware) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - m.handler.ServeHTTP(rw, r, m.next.ServeHTTP) -} - -// Wrap converts a http.Handler into a negroni.Handler so it can be used as a Negroni -// middleware. The next http.HandlerFunc is automatically called after the Handler -// is executed. -func Wrap(handler http.Handler) Handler { - return HandlerFunc(func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - handler.ServeHTTP(rw, r) - next(rw, r) - }) -} - -// Negroni is a stack of Middleware Handlers that can be invoked as an http.Handler. -// Negroni middleware is evaluated in the order that they are added to the stack using -// the Use and UseHandler methods. -type Negroni struct { - middleware middleware - handlers []Handler -} - -// New returns a new Negroni instance with no middleware preconfigured. -func New(handlers ...Handler) *Negroni { - return &Negroni{ - handlers: handlers, - middleware: build(handlers), - } -} - -// Classic returns a new Negroni instance with the default middleware already -// in the stack. -// -// Recovery - Panic Recovery Middleware -// Logger - Request/Response Logging -// Static - Static File Serving -func Classic() *Negroni { - return New(NewRecovery(), NewLogger(), NewStatic(http.Dir("public"))) -} - -func (n *Negroni) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - n.middleware.ServeHTTP(NewResponseWriter(rw), r) -} - -// Use adds a Handler onto the middleware stack. Handlers are invoked in the order they are added to a Negroni. -func (n *Negroni) Use(handler Handler) { - if handler == nil { - panic("handler cannot be nil") - } - - n.handlers = append(n.handlers, handler) - n.middleware = build(n.handlers) -} - -// UseFunc adds a Negroni-style handler function onto the middleware stack. -func (n *Negroni) UseFunc(handlerFunc func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc)) { - n.Use(HandlerFunc(handlerFunc)) -} - -// UseHandler adds a http.Handler onto the middleware stack. Handlers are invoked in the order they are added to a Negroni. -func (n *Negroni) UseHandler(handler http.Handler) { - n.Use(Wrap(handler)) -} - -// UseHandler adds a http.HandlerFunc-style handler function onto the middleware stack. -func (n *Negroni) UseHandlerFunc(handlerFunc func(rw http.ResponseWriter, r *http.Request)) { - n.UseHandler(http.HandlerFunc(handlerFunc)) -} - -// Run is a convenience function that runs the negroni stack as an HTTP -// server. The addr string takes the same format as http.ListenAndServe. -func (n *Negroni) Run(addr string) { - l := log.New(os.Stdout, "[negroni] ", 0) - l.Printf("listening on %s", addr) - l.Fatal(http.ListenAndServe(addr, n)) -} - -// Returns a list of all the handlers in the current Negroni middleware chain. -func (n *Negroni) Handlers() []Handler { - return n.handlers -} - -func build(handlers []Handler) middleware { - var next middleware - - if len(handlers) == 0 { - return voidMiddleware() - } else if len(handlers) > 1 { - next = build(handlers[1:]) - } else { - next = voidMiddleware() - } - - return middleware{handlers[0], &next} -} - -func voidMiddleware() middleware { - return middleware{ - HandlerFunc(func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {}), - &middleware{}, - } -} diff --git a/vendor/github.com/urfave/negroni/recovery.go b/vendor/github.com/urfave/negroni/recovery.go deleted file mode 100644 index 8396cb1ed..000000000 --- a/vendor/github.com/urfave/negroni/recovery.go +++ /dev/null @@ -1,65 +0,0 @@ -package negroni - -import ( - "fmt" - "log" - "net/http" - "os" - "runtime" - "runtime/debug" -) - -// Recovery is a Negroni middleware that recovers from any panics and writes a 500 if there was one. -type Recovery struct { - Logger ALogger - PrintStack bool - ErrorHandlerFunc func(interface{}) - StackAll bool - StackSize int -} - -// NewRecovery returns a new instance of Recovery -func NewRecovery() *Recovery { - return &Recovery{ - Logger: log.New(os.Stdout, "[negroni] ", 0), - PrintStack: true, - StackAll: false, - StackSize: 1024 * 8, - } -} - -func (rec *Recovery) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - defer func() { - if err := recover(); err != nil { - if rw.Header().Get("Content-Type") == "" { - rw.Header().Set("Content-Type", "text/plain; charset=utf-8") - } - - rw.WriteHeader(http.StatusInternalServerError) - - stack := make([]byte, rec.StackSize) - stack = stack[:runtime.Stack(stack, rec.StackAll)] - - f := "PANIC: %s\n%s" - rec.Logger.Printf(f, err, stack) - - if rec.PrintStack { - fmt.Fprintf(rw, f, err, stack) - } - - if rec.ErrorHandlerFunc != nil { - func() { - defer func() { - if err := recover(); err != nil { - rec.Logger.Printf("provided ErrorHandlerFunc panic'd: %s, trace:\n%s", err, debug.Stack()) - rec.Logger.Printf("%s\n", debug.Stack()) - } - }() - rec.ErrorHandlerFunc(err) - }() - } - } - }() - - next(rw, r) -} diff --git a/vendor/github.com/urfave/negroni/response_writer.go b/vendor/github.com/urfave/negroni/response_writer.go deleted file mode 100644 index f805825aa..000000000 --- a/vendor/github.com/urfave/negroni/response_writer.go +++ /dev/null @@ -1,99 +0,0 @@ -package negroni - -import ( - "bufio" - "fmt" - "net" - "net/http" -) - -// ResponseWriter is a wrapper around http.ResponseWriter that provides extra information about -// the response. It is recommended that middleware handlers use this construct to wrap a responsewriter -// if the functionality calls for it. -type ResponseWriter interface { - http.ResponseWriter - http.Flusher - // Status returns the status code of the response or 200 if the response has - // not been written (as this is the default response code in net/http) - Status() int - // Written returns whether or not the ResponseWriter has been written. - Written() bool - // Size returns the size of the response body. - Size() int - // Before allows for a function to be called before the ResponseWriter has been written to. This is - // useful for setting headers or any other operations that must happen before a response has been written. - Before(func(ResponseWriter)) -} - -type beforeFunc func(ResponseWriter) - -// NewResponseWriter creates a ResponseWriter that wraps an http.ResponseWriter -func NewResponseWriter(rw http.ResponseWriter) ResponseWriter { - return &responseWriter{ - ResponseWriter: rw, - } -} - -type responseWriter struct { - http.ResponseWriter - status int - size int - beforeFuncs []beforeFunc -} - -func (rw *responseWriter) WriteHeader(s int) { - rw.status = s - rw.callBefore() - rw.ResponseWriter.WriteHeader(s) -} - -func (rw *responseWriter) Write(b []byte) (int, error) { - if !rw.Written() { - // The status will be StatusOK if WriteHeader has not been called yet - rw.WriteHeader(http.StatusOK) - } - size, err := rw.ResponseWriter.Write(b) - rw.size += size - return size, err -} - -func (rw *responseWriter) Status() int { - return rw.status -} - -func (rw *responseWriter) Size() int { - return rw.size -} - -func (rw *responseWriter) Written() bool { - return rw.status != 0 -} - -func (rw *responseWriter) Before(before func(ResponseWriter)) { - rw.beforeFuncs = append(rw.beforeFuncs, before) -} - -func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { - hijacker, ok := rw.ResponseWriter.(http.Hijacker) - if !ok { - return nil, nil, fmt.Errorf("the ResponseWriter doesn't support the Hijacker interface") - } - return hijacker.Hijack() -} - -func (rw *responseWriter) CloseNotify() <-chan bool { - return rw.ResponseWriter.(http.CloseNotifier).CloseNotify() -} - -func (rw *responseWriter) callBefore() { - for i := len(rw.beforeFuncs) - 1; i >= 0; i-- { - rw.beforeFuncs[i](rw) - } -} - -func (rw *responseWriter) Flush() { - flusher, ok := rw.ResponseWriter.(http.Flusher) - if ok { - flusher.Flush() - } -} diff --git a/vendor/github.com/urfave/negroni/static.go b/vendor/github.com/urfave/negroni/static.go deleted file mode 100644 index 34be967c0..000000000 --- a/vendor/github.com/urfave/negroni/static.go +++ /dev/null @@ -1,88 +0,0 @@ -package negroni - -import ( - "net/http" - "path" - "strings" -) - -// Static is a middleware handler that serves static files in the given -// directory/filesystem. If the file does not exist on the filesystem, it -// passes along to the next middleware in the chain. If you desire "fileserver" -// type behavior where it returns a 404 for unfound files, you should consider -// using http.FileServer from the Go stdlib. -type Static struct { - // Dir is the directory to serve static files from - Dir http.FileSystem - // Prefix is the optional prefix used to serve the static directory content - Prefix string - // IndexFile defines which file to serve as index if it exists. - IndexFile string -} - -// NewStatic returns a new instance of Static -func NewStatic(directory http.FileSystem) *Static { - return &Static{ - Dir: directory, - Prefix: "", - IndexFile: "index.html", - } -} - -func (s *Static) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - if r.Method != "GET" && r.Method != "HEAD" { - next(rw, r) - return - } - file := r.URL.Path - // if we have a prefix, filter requests by stripping the prefix - if s.Prefix != "" { - if !strings.HasPrefix(file, s.Prefix) { - next(rw, r) - return - } - file = file[len(s.Prefix):] - if file != "" && file[0] != '/' { - next(rw, r) - return - } - } - f, err := s.Dir.Open(file) - if err != nil { - // discard the error? - next(rw, r) - return - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - next(rw, r) - return - } - - // try to serve index file - if fi.IsDir() { - // redirect if missing trailing slash - if !strings.HasSuffix(r.URL.Path, "/") { - http.Redirect(rw, r, r.URL.Path+"/", http.StatusFound) - return - } - - file = path.Join(file, s.IndexFile) - f, err = s.Dir.Open(file) - if err != nil { - next(rw, r) - return - } - defer f.Close() - - fi, err = f.Stat() - if err != nil || fi.IsDir() { - next(rw, r) - return - } - } - - http.ServeContent(rw, r, file, fi.ModTime(), f) -} diff --git a/vendor/vendor.json b/vendor/vendor.json index f325b800c..4f2dd6e89 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -213,12 +213,6 @@ "revision": "173748da739a410c5b0b813b956f89ff94730b4c", "revisionTime": "2016-08-30T17:39:30Z" }, - { - "checksumSHA1": "hMj8vjbjGCAE368E3xAjZGFON+E=", - "path": "github.com/urfave/negroni", - "revision": "3f7ce7b928e14ff890b067e5bbbc80af73690a9c", - "revisionTime": "2016-09-09T03:51:52Z" - }, { "path": "golang.org/x/crypto/bcrypt", "revision": "7b85b097bf7527677d54d3220065e966a0e3b613",