From 5ff1203fc09317852864afcfa594f34fec02df2a Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Fri, 8 Jul 2016 22:28:06 +0100 Subject: [PATCH] Add PutObjectPart benchmark (#2145) --- benchmark-utils_test.go | 85 ++++++++++++++++++++++++++++++++ object-api-getobjectinfo_test.go | 55 +++++++++++++++++++++ test-utils_test.go | 6 +++ 3 files changed, 146 insertions(+) diff --git a/benchmark-utils_test.go b/benchmark-utils_test.go index 3faa60cb8..728e380c3 100644 --- a/benchmark-utils_test.go +++ b/benchmark-utils_test.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/md5" "encoding/hex" + "math" "math/rand" "strconv" "testing" @@ -67,6 +68,90 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { b.StopTimer() } +// Benchmark utility functions for ObjectLayer.PutObjectPart(). +// Creates Object layer setup ( MakeBucket ) and then runs the PutObjectPart benchmark. +func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) { + var err error + // obtains random bucket name. + bucket := getRandomBucketName() + object := getRandomObjectName() + + // create bucket. + err = obj.MakeBucket(bucket) + if err != nil { + b.Fatal(err) + } + + objSize := 128 * 1024 * 1024 + + // PutObjectPart returns md5Sum of the object inserted. + // md5Sum variable is assigned with that value. + var md5Sum, uploadID string + // get text data generated for number of bytes equal to object size. + textData := generateBytesData(objSize) + // generate md5sum for the generated data. + // md5sum of the data to written is required as input for NewMultipartUpload. + hasher := md5.New() + hasher.Write([]byte(textData)) + metadata := make(map[string]string) + metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + uploadID, err = obj.NewMultipartUpload(bucket, object, metadata) + if err != nil { + b.Fatal(err) + } + + var textPartData []byte + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for PutObjectPart starts here. Reset the benchmark timer. + b.ResetTimer() + for i := 0; i < b.N; i++ { + // insert the object. + totalPartsNR := int(math.Ceil(float64(objSize) / float64(partSize))) + for j := 0; j < totalPartsNR; j++ { + hasher.Reset() + if j < totalPartsNR-1 { + textPartData = textData[j*partSize : (j+1)*partSize-1] + } else { + textPartData = textData[j*partSize:] + } + hasher.Write([]byte(textPartData)) + metadata := make(map[string]string) + metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil)) + md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"]) + if err != nil { + b.Fatal(err) + } + if md5Sum != metadata["md5Sum"] { + b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"]) + } + } + } + // Benchmark ends here. Stop timer. + b.StopTimer() +} + +// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function. +func benchmarkPutObjectPart(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) { + // create a temp XL/FS backend. + objLayer, disks, err := makeTestBackend(instanceType) + if err != nil { + b.Fatalf("Failed obtaining Temp Backend: %s", err) + } + // cleaning up the backend by removing all the directories and files created on function return. + defer removeRoots(disks) + // calling runPutObjectBenchmark which uses *testing.B and the object Layer to run the benchmark. + runBenchMark(b, objLayer) +} + +// closure for returning the put object benchmark executor for given object size in bytes. +func returnPutObjectPartBenchmark(objSize int) func(*testing.B, ObjectLayer) { + // FIXME: Avoid closure. + return func(b *testing.B, obj ObjectLayer) { + runPutObjectPartBenchmark(b, obj, objSize) + } +} + // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function. func benchmarkPutObject(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) { // create a temp XL/FS backend. diff --git a/object-api-getobjectinfo_test.go b/object-api-getobjectinfo_test.go index 70453c8a5..175cc2d54 100644 --- a/object-api-getobjectinfo_test.go +++ b/object-api-getobjectinfo_test.go @@ -105,6 +105,61 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) { } } +// Benchmarks for ObjectLayer.PutObjectPart(). +// The intent is to benchamrk PutObjectPart for various sizes ranging from few bytes to 100MB. +// Also each of these Benchmarks are run both XL and FS backends. + +// BenchmarkPutObjectPart5MbFS - Benchmark FS.PutObjectPart() for object size of 5MB. +func BenchmarkPutObjectPart5MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(5*1024*1024)) +} + +// BenchmarkPutObjectPart5MbXL - Benchmark XL.PutObjectPart() for object size of 5MB. +func BenchmarkPutObjectPart5MbXL(b *testing.B) { + benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(5*1024*1024)) +} + +// BenchmarkPutObjectPart10MbFS - Benchmark FS.PutObjectPart() for object size of 10MB. +func BenchmarkPutObjectPart10MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(10*1024*1024)) +} + +// BenchmarkPutObjectPart10MbXL - Benchmark XL.PutObjectPart() for object size of 10MB. +func BenchmarkPutObjectPart10MbXL(b *testing.B) { + benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(10*1024*1024)) +} + +// BenchmarkPutObjectPart25MbFS - Benchmark FS.PutObjectPart() for object size of 25MB. +func BenchmarkPutObjectPart25MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(25*1024*1024)) + +} + +// BenchmarkPutObjectPart25MbXL - Benchmark XL.PutObjectPart() for object size of 25MB. +func BenchmarkPutObjectPart25MbXL(b *testing.B) { + benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(25*1024*1024)) +} + +// BenchmarkPutObjectPart50MbFS - Benchmark FS.PutObjectPart() for object size of 50MB. +func BenchmarkPutObjectPart50MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(50*1024*1024)) +} + +// BenchmarkPutObjectPart50MbXL - Benchmark XL.PutObjectPart() for object size of 50MB. +func BenchmarkPutObjectPart50MbXL(b *testing.B) { + benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(50*1024*1024)) +} + +// BenchmarkPutObjectPart100MbFS - Benchmark FS.PutObjectPart() for object size of 100MB. +func BenchmarkPutObjectPart100MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(100*1024*1024)) +} + +// BenchmarkPutObjectPart100MbXL - Benchmark XL.PutObjectPart() for object size of 100MB. +func BenchmarkPutObjectPart100MbXL(b *testing.B) { + benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(100*1024*1024)) +} + // Benchmarks for ObjectLayer.PutObject(). // The intent is to benchamrk PutObject for various sizes ranging from few bytes to 100MB. // Also each of these Benchmarks are run both XL and FS backends. diff --git a/test-utils_test.go b/test-utils_test.go index 2afb88e22..ce61cff25 100644 --- a/test-utils_test.go +++ b/test-utils_test.go @@ -341,6 +341,12 @@ func randString(n int) string { return string(b) } +// generate random object name. +func getRandomObjectName() string { + return randString(16) + +} + // generate random bucket name. func getRandomBucketName() string { return randString(60)