Benchmarks for ObjectLayer.PutObject() (#2029)

benchmark-utils_test.go

@@ -26,6 +26,69 @@ import (
 	"time"
 )
 
+// Benchmark utility functions for ObjectLayer.PutObject().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
+func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	hasher := md5.New()
+	hasher.Write([]byte(textData))
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// insert the object.
+		md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if md5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
+func benchmarkPutObject(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// calling runPutObjectBenchmark which uses *testing.B and the object Layer to run the benchmark.
+	runBenchMark(b, objLayer)
+}
+
+// closure for returning the put object benchmark executor for given object size in bytes.
+func returnPutObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) {
+	// FIXME: Avoid closure.
+	return func(b *testing.B, obj ObjectLayer) {
+		runPutObjectBenchmark(b, obj, objSize)
+	}
+}
+
+// Benchmark utility functions for ObjectLayer.GetObject().
 // Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
 func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	var err error
@@ -75,17 +138,20 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	b.StopTimer()
 
 }
+
+// randomly picks a character and returns its equivalent byte array.
+func getRandomByte() []byte {
+	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	// seeding the random number generator.
+	rand.Seed(time.Now().UnixNano())
+	var b byte
+	// pick a character randomly.
+	b = letterBytes[rand.Intn(len(letterBytes))]
+	return []byte{b}
+}
+
+// picks a random byte and repeats it to size bytes.
 func generateBytesData(size int) []byte {
-	// randomly picks a character and returns its equivalent byte array.
-	getRandomByte := func() []byte {
-		const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-		// seeding the random number generator.
-		rand.Seed(time.Now().UnixNano())
-		var b byte
-		// pick a character randomly.
-		b = letterBytes[rand.Intn(len(letterBytes))]
-		return []byte{b}
-	}
 	// repeat the random character chosen size
 	return bytes.Repeat(getRandomByte(), size)
 }
@@ -104,6 +170,7 @@ func benchmarkGetObject(b *testing.B, instanceType string, runBenchMark func(b *
 }
 
 // closure for returning the get object benchmark executor for given object size in bytes.
+// FIXME: Avoid closure.
 func returnGetObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) {
 	return func(b *testing.B, obj ObjectLayer) {
 		runGetObjectBenchmark(b, obj, objSize)

object-api-getobjectinfo_test.go

@@ -105,6 +105,131 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t *testing.T) {
 	}
 }
 
+// Benchmarks for ObjectLayer.PutObject().
+// The intent is to benchamrk PutObject for various sizes ranging from few bytes to 100MB.
+// Also each of these Benchmarks are run both XL and FS backends.
+
+// BenchmarkPutObjectVerySmallFS - Benchmark FS.PutObject() for object size of 10 bytes.
+func BenchmarkPutObjectVerySmallFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10))
+}
+
+// BenchmarkPutObjectVerySmallXL - Benchmark XL.PutObject() for object size of 10 bytes.
+func BenchmarkPutObjectVerySmallXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10))
+}
+
+// BenchmarkPutObject10KbFS - Benchmark FS.PutObject() for object size of 10KB.
+func BenchmarkPutObject10KbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10*1024))
+}
+
+// BenchmarkPutObject10KbXL - Benchmark XL.PutObject() for object size of 10KB.
+func BenchmarkPutObject10KbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10*1024))
+}
+
+// BenchmarkPutObject100KbFS - Benchmark FS.PutObject() for object size of 100KB.
+func BenchmarkPutObject100KbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(100*1024))
+}
+
+// BenchmarkPutObject100KbXL - Benchmark XL.PutObject() for object size of 100KB.
+func BenchmarkPutObject100KbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(100*1024))
+}
+
+// BenchmarkPutObject1MbFS - Benchmark FS.PutObject() for object size of 1MB.
+func BenchmarkPutObject1MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(1024*1024))
+}
+
+// BenchmarkPutObject1MbXL - Benchmark XL.PutObject() for object size of 1MB.
+func BenchmarkPutObject1MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024))
+}
+
+// BenchmarkPutObject5MbFS - Benchmark FS.PutObject() for object size of 5MB.
+func BenchmarkPutObject5MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(5*1024*1024))
+}
+
+// BenchmarkPutObject5MbXL - Benchmark XL.PutObject() for object size of 5MB.
+func BenchmarkPutObject5MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(5*1024*1024))
+}
+
+// BenchmarkPutObject10MbFS - Benchmark FS.PutObject() for object size of 10MB.
+func BenchmarkPutObject10MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10*1024*1024))
+}
+
+// BenchmarkPutObject10MbXL - Benchmark XL.PutObject() for object size of 10MB.
+func BenchmarkPutObject10MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10*1024*1024))
+}
+
+// BenchmarkPutObject25MbFS - Benchmark FS.PutObject() for object size of 25MB.
+func BenchmarkPutObject25MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(25*1024*1024))
+
+}
+
+// BenchmarkPutObject25MbXL - Benchmark XL.PutObject() for object size of 25MB.
+func BenchmarkPutObject25MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(25*1024*1024))
+}
+
+// BenchmarkPutObject50MbFS - Benchmark FS.PutObject() for object size of 50MB.
+func BenchmarkPutObject50MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(50*1024*1024))
+}
+
+// BenchmarkPutObject50MbXL - Benchmark XL.PutObject() for object size of 50MB.
+func BenchmarkPutObject50MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(50*1024*1024))
+}
+
+// BenchmarkPutObject100MbFS - Benchmark FS.PutObject() for object size of 100MB.
+func BenchmarkPutObject100MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(100*1024*1024))
+}
+
+// BenchmarkPutObject100MbXL - Benchmark XL.PutObject() for object size of 100MB.
+func BenchmarkPutObject100MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(100*1024*1024))
+}
+
+// BenchmarkPutObject200MbFS - Benchmark FS.PutObject() for object size of 200MB.
+func BenchmarkPutObject200MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(200*1024*1024))
+}
+
+// BenchmarkPutObject200MbXL - Benchmark XL.PutObject() for object size of 200MB.
+func BenchmarkPutObject200MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(200*1024*1024))
+}
+
+// BenchmarkPutObject500MbFS - Benchmark FS.PutObject() for object size of 500MB.
+func BenchmarkPutObject500MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(500*1024*1024))
+}
+
+// BenchmarkPutObject500MbXL - Benchmark XL.PutObject() for object size of 500MB.
+func BenchmarkPutObject500MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(500*1024*1024))
+}
+
+// BenchmarkPutObject1GbFS - Benchmark FS.PutObject() for object size of 1GB.
+func BenchmarkPutObject1GbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(1024*1024*1024))
+}
+
+// BenchmarkPutObjectGbXL - Benchmark XL.PutObject() for object size of 1GB.
+func BenchmarkPutObject1GbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024*1024))
+}
+
 // Benchmarks for ObjectLayer.GetObject().
 // The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB.
 // Also each of these Benchmarks are run both XL and FS backends.