From 6800902b43b208e1dfe619a4b53a000e58630938 Mon Sep 17 00:00:00 2001 From: Krishna Srinivas Date: Wed, 8 Feb 2017 23:39:08 -0800 Subject: [PATCH] web-handlers: Implement API to download files as a zip file. (#3715) --- cmd/web-handlers.go | 86 +++++++++++++++++++++++++++++++++++++++ cmd/web-handlers_test.go | 88 ++++++++++++++++++++++++++++++++++++++++ cmd/web-router.go | 1 + 3 files changed, 175 insertions(+) diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 53daca3cc..e11743e94 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -17,6 +17,7 @@ package cmd import ( + "archive/zip" "encoding/json" "errors" "fmt" @@ -530,6 +531,91 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) { } } +// DownloadZipArgs - Argument for downloading a bunch of files as a zip file. +// JSON will look like: +// '{"bucketname":"testbucket","prefix":"john/pics/","objects":["hawaii/","maldives/","sanjose.jpg"]}' +type DownloadZipArgs struct { + Objects []string `json:"objects"` // can be files or sub-directories + Prefix string `json:"prefix"` // current directory in the browser-ui + BucketName string `json:"bucketname"` // bucket name. +} + +// Takes a list of objects and creates a zip file that sent as the response body. +func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) { + objectAPI := web.ObjectAPI() + if objectAPI == nil { + writeWebErrorResponse(w, errServerNotInitialized) + return + } + + token := r.URL.Query().Get("token") + + if !isAuthTokenValid(token) { + writeWebErrorResponse(w, errAuthentication) + return + } + var args DownloadZipArgs + decodeErr := json.NewDecoder(r.Body).Decode(&args) + if decodeErr != nil { + writeWebErrorResponse(w, decodeErr) + return + } + + archive := zip.NewWriter(w) + defer archive.Close() + + for _, object := range args.Objects { + // Writes compressed object file to the response. + zipit := func(objectName string) error { + info, err := objectAPI.GetObjectInfo(args.BucketName, objectName) + if err != nil { + return err + } + header := &zip.FileHeader{ + Name: strings.TrimPrefix(objectName, args.Prefix), + Method: zip.Deflate, + UncompressedSize64: uint64(info.Size), + UncompressedSize: uint32(info.Size), + } + writer, err := archive.CreateHeader(header) + if err != nil { + writeWebErrorResponse(w, errUnexpected) + return err + } + return objectAPI.GetObject(args.BucketName, objectName, 0, info.Size, writer) + } + + if !strings.HasSuffix(object, "/") { + // If not a directory, compress the file and write it to response. + err := zipit(pathJoin(args.Prefix, object)) + if err != nil { + return + } + continue + } + + // For directories, list the contents recursively and write the objects as compressed + // date to the response writer. + marker := "" + for { + lo, err := objectAPI.ListObjects(args.BucketName, pathJoin(args.Prefix, object), marker, "", 1000) + if err != nil { + return + } + marker = lo.NextMarker + for _, obj := range lo.Objects { + err = zipit(obj.Name) + if err != nil { + return + } + } + if !lo.IsTruncated { + break + } + } + } +} + // GetBucketPolicyArgs - get bucket policy args. type GetBucketPolicyArgs struct { BucketName string `json:"bucketName"` diff --git a/cmd/web-handlers_test.go b/cmd/web-handlers_test.go index ae0c2576a..66f7cbd52 100644 --- a/cmd/web-handlers_test.go +++ b/cmd/web-handlers_test.go @@ -17,9 +17,14 @@ package cmd import ( + "archive/zip" "bytes" + "crypto/md5" + "encoding/hex" + "encoding/json" "errors" "fmt" + "io" "io/ioutil" "net/http" "net/http/httptest" @@ -796,6 +801,89 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl } } +// Test web.DownloadZip +func TestWebHandlerDownloadZip(t *testing.T) { + ExecObjectLayerTest(t, testWebHandlerDownloadZip) +} + +func testWebHandlerDownloadZip(obj ObjectLayer, instanceType string, t TestErrHandler) { + apiRouter := initTestWebRPCEndPoint(obj) + credentials := serverConfig.GetCredential() + + authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey) + if err != nil { + t.Fatal("Cannot authenticate") + } + + bucket := getRandomBucketName() + fileOne := "aaaaaaaaaaaaaa" + fileTwo := "bbbbbbbbbbbbbb" + fileThree := "cccccccccccccc" + + // Create bucket. + err = obj.MakeBucket(bucket) + if err != nil { + // failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err) + } + + obj.PutObject(bucket, "a/one", int64(len(fileOne)), strings.NewReader(fileOne), nil, "") + obj.PutObject(bucket, "a/b/two", int64(len(fileTwo)), strings.NewReader(fileTwo), nil, "") + obj.PutObject(bucket, "a/c/three", int64(len(fileThree)), strings.NewReader(fileThree), nil, "") + + test := func(token string) (int, []byte) { + rec := httptest.NewRecorder() + path := "/minio/zip" + "?token=" + if token != "" { + path = path + token + } + args := DownloadZipArgs{ + Objects: []string{"one", "b/", "c/"}, + Prefix: "a/", + BucketName: bucket, + } + + var argsData []byte + argsData, err = json.Marshal(args) + if err != nil { + return 0, nil + } + var req *http.Request + req, err = http.NewRequest("GET", path, bytes.NewBuffer(argsData)) + + if err != nil { + t.Fatalf("Cannot create upload request, %v", err) + } + + apiRouter.ServeHTTP(rec, req) + return rec.Code, rec.Body.Bytes() + } + code, data := test("") + if code != 403 { + t.Fatal("Expected to receive authentication error") + } + code, data = test(authorization) + if code != 200 { + t.Fatal("web.DownloadsZip() failed") + } + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatal(err) + } + h := md5.New() + for _, file := range reader.File { + fileReader, err := file.Open() + if err != nil { + t.Fatal(err) + } + io.Copy(h, fileReader) + } + // Verify the md5 of the response. + if hex.EncodeToString(h.Sum(nil)) != "ac7196449b14bea42775d29e8bb29f50" { + t.Fatal("Incorrect zip contents") + } +} + // Wrapper for calling PresignedGet handler func TestWebHandlerPresignedGetHandler(t *testing.T) { ExecObjectLayerTest(t, testWebPresignedGetHandler) diff --git a/cmd/web-router.go b/cmd/web-router.go index f478abfb8..c4fdfbb2c 100644 --- a/cmd/web-router.go +++ b/cmd/web-router.go @@ -84,6 +84,7 @@ func registerWebRouter(mux *router.Router) error { webBrowserRouter.Methods("POST").Path("/webrpc").Handler(webRPC) webBrowserRouter.Methods("PUT").Path("/upload/{bucket}/{object:.+}").HandlerFunc(web.Upload) webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(web.Download) + webBrowserRouter.Methods("GET").Path("/zip").Queries("token", "{token:.*}").HandlerFunc(web.DownloadZip) // Add compression for assets. compressedAssets := handlers.CompressHandler(http.StripPrefix(reservedBucket, http.FileServer(assetFS())))