Merge pull request #1113 from harshavardhana/list-objects-optimization

listObjects: list objects minor optimization.

Makefile

@@ -26,6 +26,7 @@ getdeps: checkdeps checkgopath
 	@go get -u golang.org/x/tools/cmd/vet && echo "Installed vet:"
 	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
 	@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
+	@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:"
 
 $(UI_ASSETS):
 	@curl -s https://dl.minio.io/assets/server/$(UI_ASSETS_ARMOR) 2>&1 > $(UI_ASSETS_ARMOR) && echo "Downloading signature file $(UI_ASSETS_ARMOR) for verification:"
@@ -33,7 +34,7 @@ $(UI_ASSETS):
 	@curl -s https://dl.minio.io/assets/server/$@ 2>&1 > $@ && echo "Downloading UI assets file $@:"
 	@gpg --batch --no-tty --verify $(UI_ASSETS_ARMOR) $@ 2>&1 > /dev/null && echo "Verifying signature of downloaded assets."
 
-verifiers: getdeps vet fmt lint cyclo
+verifiers: getdeps vet fmt lint cyclo spelling
 
 vet:
 	@echo "Running $@:"
@@ -61,7 +62,11 @@ build: getdeps verifiers $(UI_ASSETS)
 	@echo "Installing minio:"
 
 deadcode:
-	@GO15VENDOREXPERIMENT=1 deadcode
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
+
+spelling:
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell pkg/**/*
 
 test: build
 	@echo "Running all testing:"
@@ -72,13 +77,13 @@ gomake-all: build
 	@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
 
 pkg-add:
-	@GO15VENDOREXPERIMENT=1 govendor add $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
 
 pkg-update:
-	@GO15VENDOREXPERIMENT=1 govendor update $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor update $(PKG)
 
 pkg-remove:
-	@GO15VENDOREXPERIMENT=1 govendor remove $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor remove $(PKG)
 
 install: gomake-all
 

appveyor.yml

@@ -24,7 +24,7 @@ build_script:
   - curl -o ui-assets.go -L https://dl.minio.io/assets/server/%UI_ASSETS%
   - curl -o ui-assets.asc -L https://dl.minio.io/assets/server/%UI_ASSETS_ARMOR%
   - gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728
-  - gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%  
+  - gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%
   - go test .
   - go test -race .
   - go test github.com/minio/minio/pkg...

buildscripts/checkdeps.sh

@@ -21,6 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
+    CURL_VERSION="7.12.0"
     GPG_VERSION="1.3"
     GO_VERSION="1.5.1"
     OSX_VERSION="10.8"
@@ -186,6 +187,11 @@ check_deps() {
     if [ $? -ge 2 ]; then
         MISSING="${MISSING} gpg"
     fi
+
+    check_version "$(env curl --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g' | head -1)" "${CURL_VERSION}"
+    if [ $? -ge 2 ]; then
+        MISSING="${MISSING} curl"
+    fi
 }
 
 main() {

generic-handlers.go

@@ -176,7 +176,7 @@ func setIgnoreResourcesHandler(h http.Handler) http.Handler {
 
 // Resource handler ServeHTTP() wrapper
 func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// Skip the first element which is usally '/' and split the rest.
+	// Skip the first element which is usually '/' and split the rest.
 	splits := strings.SplitN(r.URL.Path[1:], "/", 2)
 
 	// Save bucketName and objectName extracted from url Path.

pkg/fs/fs-bucket-listobjects.go

@@ -56,11 +56,11 @@ type listWorkerReq struct {
 	respCh chan ListObjectsResult
 }
 
-// listObjects - list objects lists objects upto maxKeys for a given prefix.
+// listObjects - list objects lists objects up to maxKeys for a given prefix.
 func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (chan<- listWorkerReq, *probe.Error) {
 	quitWalker := make(chan bool)
 	reqCh := make(chan listWorkerReq)
-	walkerCh := make(chan ObjectMetadata)
+	walkerCh := make(chan ObjectMetadata, 1000)
 	go func() {
 		defer close(walkerCh)
 		var walkPath string
@@ -144,8 +144,14 @@ func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKe
 				for object := range walkerCh {
 					// Verify if the object is lexically smaller than
 					// the marker, we will skip those objects.
-					if marker >= object.Object {
-						continue
+					if marker != "" {
+						if marker >= object.Object {
+							continue
+						} else {
+							// Reset marker so that we avoid comparing
+							// again and again in a loop unecessarily.
+							marker = ""
+						}
 					}
 					if delimiter != "" {
 						// Prefixes are only valid wth delimiters, and
@@ -275,7 +281,7 @@ func (fs *Filesystem) listObjectsService() *probe.Error {
 	return nil
 }
 
-// ListObjects - lists all objects for a given prefix, returns upto
+// ListObjects - lists all objects for a given prefix, returns up to
 // maxKeys number of objects per call.
 func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) {
 	// Input validation.

pkg/fs/signature.go

@@ -346,7 +346,7 @@ func (r *Signature) DoesPresignedSignatureMatch() (bool, *probe.Error) {
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
 // returns true if matches, false otherwise. if error is not nil then it is always false
 func (r *Signature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) {
-	// set new calulated payload
+	// set new calculated payload
 	r.Request.Header.Set("X-Amz-Content-Sha256", hashedPayload)
 
 	// Add date if not present throw error