From aa2e89bfe3043a7a39debf650583bfb9ac334041 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 8 Jan 2020 03:31:43 -0800 Subject: [PATCH] Use jsoniter whenever applicable instead of encoding/json (#8766) This PR adds jsoniter package to replace encoding/json in places where faster json unmarshal is necessary whenever input JSON is large enough. Some benchmarking comparison between jsoniter and enconding/json benchmark old MB/s new MB/s speedup BenchmarkParseUnmarshal/N10-4 110.02 331.17 3.01x BenchmarkParseUnmarshal/N100-4 125.74 524.09 4.17x BenchmarkParseUnmarshal/N500-4 131.68 542.60 4.12x BenchmarkParseUnmarshal/N1000-4 133.93 514.88 3.84x BenchmarkParseUnmarshal/N5000-4 122.10 415.36 3.40x BenchmarkParseUnmarshal/N10000-4 132.13 403.90 3.06x --- cmd/config.go | 2 ++ cmd/data-usage.go | 2 ++ cmd/format-disk-cache.go | 3 ++- cmd/fs-v1-metadata.go | 1 + cmd/policy.go | 3 +++ cmd/xl-v1-metadata.go | 2 ++ cmd/xl-v1-utils.go | 3 ++- 7 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 3d5c456e1..cd18ff3c3 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -26,6 +26,7 @@ import ( "strings" "time" + jsoniter "github.com/json-iterator/go" "github.com/minio/minio/cmd/config" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/madmin" @@ -170,6 +171,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e } var config = config.New() + var json = jsoniter.ConfigCompatibleWithStandardLibrary if err = json.Unmarshal(configData, &config); err != nil { return nil, err } diff --git a/cmd/data-usage.go b/cmd/data-usage.go index 1921afe48..a5611d270 100644 --- a/cmd/data-usage.go +++ b/cmd/data-usage.go @@ -22,6 +22,7 @@ import ( "encoding/json" "time" + jsoniter "github.com/json-iterator/go" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/hash" ) @@ -133,6 +134,7 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag } var dataUsageInfo DataUsageInfo + var json = jsoniter.ConfigCompatibleWithStandardLibrary err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo) if err != nil { return DataUsageInfo{}, err diff --git a/cmd/format-disk-cache.go b/cmd/format-disk-cache.go index 4beec1b1d..3dec6e56a 100644 --- a/cmd/format-disk-cache.go +++ b/cmd/format-disk-cache.go @@ -18,7 +18,6 @@ package cmd import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -29,6 +28,7 @@ import ( "reflect" "strings" + jsoniter "github.com/json-iterator/go" "github.com/minio/minio/cmd/logger" "github.com/minio/sio" ) @@ -444,6 +444,7 @@ func migrateOldCache(ctx context.Context, c *diskCache) error { } // marshal cache metadata after adding version and stat info meta := &cacheMeta{} + var json = jsoniter.ConfigCompatibleWithStandardLibrary if err = json.Unmarshal(metaBytes, &meta); err != nil { return err } diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go index d76abc899..83edd1849 100644 --- a/cmd/fs-v1-metadata.go +++ b/cmd/fs-v1-metadata.go @@ -90,6 +90,7 @@ func (c *FSChecksumInfoV1) UnmarshalJSON(data []byte) error { } var info checksuminfo + var json = jsoniter.ConfigCompatibleWithStandardLibrary err := json.Unmarshal(data, &info) if err != nil { return err diff --git a/cmd/policy.go b/cmd/policy.go index 7e1e5ba7c..cd32c875d 100644 --- a/cmd/policy.go +++ b/cmd/policy.go @@ -26,6 +26,7 @@ import ( "strings" "sync" + jsoniter "github.com/json-iterator/go" miniogopolicy "github.com/minio/minio-go/v6/pkg/policy" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/event" @@ -275,6 +276,7 @@ func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.Buc } var policyInfo miniogopolicy.BucketAccessPolicy + var json = jsoniter.ConfigCompatibleWithStandardLibrary if err = json.Unmarshal(data, &policyInfo); err != nil { // This should not happen because data is valid to JSON data. return nil, err @@ -292,6 +294,7 @@ func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (* } var bucketPolicy policy.Policy + var json = jsoniter.ConfigCompatibleWithStandardLibrary if err = json.Unmarshal(data, &bucketPolicy); err != nil { // This should not happen because data is valid to JSON data. return nil, err diff --git a/cmd/xl-v1-metadata.go b/cmd/xl-v1-metadata.go index e3e1df28d..5d4651092 100644 --- a/cmd/xl-v1-metadata.go +++ b/cmd/xl-v1-metadata.go @@ -26,6 +26,7 @@ import ( "sort" "time" + jsoniter "github.com/json-iterator/go" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/sync/errgroup" @@ -77,6 +78,7 @@ func (c ChecksumInfo) MarshalJSON() ([]byte, error) { // UnmarshalJSON - should never be called, instead xlMetaV1UnmarshalJSON() should be used. func (c *ChecksumInfo) UnmarshalJSON(data []byte) error { var info checksumInfoJSON + var json = jsoniter.ConfigCompatibleWithStandardLibrary if err := json.Unmarshal(data, &info); err != nil { return err } diff --git a/cmd/xl-v1-utils.go b/cmd/xl-v1-utils.go index 511f1455c..5e20a407d 100644 --- a/cmd/xl-v1-utils.go +++ b/cmd/xl-v1-utils.go @@ -18,11 +18,11 @@ package cmd import ( "context" - "encoding/json" "errors" "hash/crc32" "path" + jsoniter "github.com/json-iterator/go" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/sync/errgroup" ) @@ -117,6 +117,7 @@ func hashOrder(key string, cardinality int) []int { // Constructs xlMetaV1 using `jsoniter` lib. func xlMetaV1UnmarshalJSON(ctx context.Context, xlMetaBuf []byte) (xlMeta xlMetaV1, err error) { + var json = jsoniter.ConfigCompatibleWithStandardLibrary err = json.Unmarshal(xlMetaBuf, &xlMeta) return xlMeta, err }