From 63b3f1dcfd32ef787b979e645e47618e5d74a2f7 Mon Sep 17 00:00:00 2001 From: frankw Date: Fri, 8 Jul 2016 16:33:21 +0200 Subject: [PATCH] Use new algorithm to get fixed random order of disks (#2147) --- xl-v1-common.go | 7 +++++-- xl-v1-metadata.go | 2 +- xl-v1-utils.go | 26 +++++++++++++------------- xl-v1_test.go | 28 ++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/xl-v1-common.go b/xl-v1-common.go index cfb11a02d..c50062527 100644 --- a/xl-v1-common.go +++ b/xl-v1-common.go @@ -16,7 +16,10 @@ package main -import "path" +import ( + "path" + "time" +) // getLoadBalancedQuorumDisks - fetches load balanced sufficiently // randomized quorum disk slice. @@ -29,7 +32,7 @@ func (xl xlObjects) getLoadBalancedQuorumDisks() (disks []StorageAPI) { // randomized) disk slice. func (xl xlObjects) getLoadBalancedDisks() (disks []StorageAPI) { // Based on the random shuffling return back randomized disks. - for _, i := range randInts(len(xl.storageDisks)) { + for _, i := range hashOrder(time.Now().UTC().String(), len(xl.storageDisks)) { disks = append(disks, xl.storageDisks[i-1]) } return disks diff --git a/xl-v1-metadata.go b/xl-v1-metadata.go index d9604dd27..3233ff5db 100644 --- a/xl-v1-metadata.go +++ b/xl-v1-metadata.go @@ -119,7 +119,7 @@ func newXLMetaV1(dataBlocks, parityBlocks int) (xlMeta xlMetaV1) { DataBlocks: dataBlocks, ParityBlocks: parityBlocks, BlockSize: blockSizeV1, - Distribution: randInts(dataBlocks + parityBlocks), + Distribution: hashOrder(time.Now().UTC().String(), dataBlocks+parityBlocks), } return xlMeta } diff --git a/xl-v1-utils.go b/xl-v1-utils.go index deae95a42..3221668e2 100644 --- a/xl-v1-utils.go +++ b/xl-v1-utils.go @@ -18,9 +18,9 @@ package main import ( "encoding/json" - "math/rand" + "errors" + "hash/crc32" "path" - "time" ) // Validates if we have quorum based on the errors with errDiskNotFound. @@ -48,19 +48,19 @@ func diskCount(disks []StorageAPI) int { return diskCount } -// randInts - uses Knuth Fisher-Yates shuffle algorithm for generating uniform shuffling. -func randInts(count int) []int { - rand.Seed(time.Now().UTC().UnixNano()) // Seed with current time. - ints := make([]int, count) - for i := 0; i < count; i++ { - ints[i] = i + 1 +// hashOrder - returns consistent hashed integers of count slice, based on the input token. +func hashOrder(token string, count int) []int { + if count < 0 { + panic(errors.New("hashOrder count cannot be negative")) } - for i := 0; i < count; i++ { - // Choose index uniformly in [i, count-1] - r := i + rand.Intn(count-i) - ints[r], ints[i] = ints[i], ints[r] + nums := make([]int, count) + tokenCrc := crc32.Checksum([]byte(token), crc32.IEEETable) + + start := int(uint32(tokenCrc)%uint32(count)) | 1 + for i := 1; i <= count; i++ { + nums[i-1] = 1 + ((start + i) % count) } - return ints + return nums } // readXLMeta reads `xl.json` and returns back XL metadata structure. diff --git a/xl-v1_test.go b/xl-v1_test.go index a2cf6f1ba..df9f91b48 100644 --- a/xl-v1_test.go +++ b/xl-v1_test.go @@ -19,6 +19,7 @@ package main import ( "os" "path/filepath" + "reflect" "testing" ) @@ -136,3 +137,30 @@ func TestNewXL(t *testing.T) { t.Fatalf("Unable to initialize erasure, %s", err) } } + +// TestHashOrder - test order of ints in array +func TestHashOrder(t *testing.T) { + testCases := []struct { + objectName string + hashedOrder []int + }{ + // cases which should pass the test. + // passing in valid object name. + {"object", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, + {"The Shining Script .pdf", []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}}, + {"Cost Benefit Analysis (2009-2010).pptx", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, + {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}}, + {"SHØRT", []int{11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, + {"There are far too many object names, and far too few bucket names!", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, + {"a/b/c/", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}}, + {"/a/b/c", []int{7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6}}, + {string([]byte{0xff, 0xfe, 0xfd}), []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, + } + + for i, testCase := range testCases { + hashedOrder := hashOrder(testCase.objectName, 16) + if !reflect.DeepEqual(testCase.hashedOrder, hashedOrder) { + t.Errorf("Test case %d: Expected \"%#v\" but failed \"%#v\"", i+1, testCase.hashedOrder, hashedOrder) + } + } +}