diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 20179645b..79dd0fa58 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -18,6 +18,11 @@ "ImportPath": "github.com/gorilla/mux", "Rev": "e444e69cbd2e2e3e0749a2f3c717cec491552bbf" }, + { + "ImportPath": "github.com/minio-io/go-patricia/patricia", + "Comment": "v1.0.1-2-g9469ac2", + "Rev": "9469ac299b073cc3adb646559a176ba4041fefd5" + }, { "ImportPath": "github.com/spaolacci/murmur3", "Rev": "9d87265e0948d305ffe0383d54e59c3ac3482454" diff --git a/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/children.go b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/children.go new file mode 100644 index 000000000..67e2c7b7a --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/children.go @@ -0,0 +1,230 @@ +// Copyright (c) 2014 The go-patricia AUTHORS +// +// Use of this source code is governed by The MIT License +// that can be found in the LICENSE file. + +package patricia + +// Max prefix length that is kept in a single trie node. +var MaxPrefixPerNode = 10 + +// Max children to keep in a node in the sparse mode. +const MaxChildrenPerSparseNode = 8 + +type ChildList interface { + length() int + head() *Trie + add(child *Trie) ChildList + replace(b byte, child *Trie) + remove(child *Trie) + next(b byte) *Trie + walk(prefix *Prefix, visitor VisitorFunc) error +} + +type SparseChildList struct { + Children []*Trie +} + +func newSparseChildList() ChildList { + return &SparseChildList{ + Children: make([]*Trie, 0, MaxChildrenPerSparseNode), + } +} + +func (list *SparseChildList) length() int { + return len(list.Children) +} + +func (list *SparseChildList) head() *Trie { + return list.Children[0] +} + +func (list *SparseChildList) add(child *Trie) ChildList { + // Search for an empty spot and insert the child if possible. + if len(list.Children) != cap(list.Children) { + list.Children = append(list.Children, child) + return list + } + + // Otherwise we have to transform to the dense list type. + return newDenseChildList(list, child) +} + +func (list *SparseChildList) replace(b byte, child *Trie) { + // Seek the child and replace it. + for i, node := range list.Children { + if node.InternalPrefix[0] == b { + list.Children[i] = child + return + } + } +} + +func (list *SparseChildList) remove(child *Trie) { + for i, node := range list.Children { + if node.InternalPrefix[0] == child.InternalPrefix[0] { + list.Children = append(list.Children[:i], list.Children[i+1:]...) + return + } + } + + // This is not supposed to be reached. + panic("removing non-existent child") +} + +func (list *SparseChildList) next(b byte) *Trie { + for _, child := range list.Children { + if child.InternalPrefix[0] == b { + return child + } + } + return nil +} + +func (list *SparseChildList) walk(prefix *Prefix, visitor VisitorFunc) error { + for _, child := range list.Children { + *prefix = append(*prefix, child.InternalPrefix...) + if child.InternalItem != nil { + err := visitor(*prefix, child.InternalItem) + if err != nil { + if err == SkipSubtree { + *prefix = (*prefix)[:len(*prefix)-len(child.InternalPrefix)] + continue + } + *prefix = (*prefix)[:len(*prefix)-len(child.InternalPrefix)] + return err + } + } + + err := child.InternalChildren.walk(prefix, visitor) + *prefix = (*prefix)[:len(*prefix)-len(child.InternalPrefix)] + if err != nil { + return err + } + } + + return nil +} + +type DenseChildList struct { + Min int + Max int + Children []*Trie +} + +func newDenseChildList(list *SparseChildList, child *Trie) ChildList { + var ( + min int = 255 + max int = 0 + ) + for _, child := range list.Children { + b := int(child.InternalPrefix[0]) + if b < min { + min = b + } + if b > max { + max = b + } + } + + b := int(child.InternalPrefix[0]) + if b < min { + min = b + } + if b > max { + max = b + } + + children := make([]*Trie, max-min+1) + for _, child := range list.Children { + children[int(child.InternalPrefix[0])-min] = child + } + children[int(child.InternalPrefix[0])-min] = child + + return &DenseChildList{min, max, children} +} + +func (list *DenseChildList) length() int { + return list.Max - list.Min + 1 +} + +func (list *DenseChildList) head() *Trie { + return list.Children[0] +} + +func (list *DenseChildList) add(child *Trie) ChildList { + b := int(child.InternalPrefix[0]) + + switch { + case list.Min <= b && b <= list.Max: + if list.Children[b-list.Min] != nil { + panic("dense child list collision detected") + } + list.Children[b-list.Min] = child + + case b < list.Min: + children := make([]*Trie, list.Max-b+1) + children[0] = child + copy(children[list.Min-b:], list.Children) + list.Children = children + list.Min = b + + default: // b > list.max + children := make([]*Trie, b-list.Min+1) + children[b-list.Min] = child + copy(children, list.Children) + list.Children = children + list.Max = b + } + + return list +} + +func (list *DenseChildList) replace(b byte, child *Trie) { + list.Children[int(b)-list.Min] = nil + list.Children[int(child.InternalPrefix[0])-list.Min] = child +} + +func (list *DenseChildList) remove(child *Trie) { + i := int(child.InternalPrefix[0]) - list.Min + if list.Children[i] == nil { + // This is not supposed to be reached. + panic("removing non-existent child") + } + list.Children[i] = nil +} + +func (list *DenseChildList) next(b byte) *Trie { + i := int(b) + if i < list.Min || list.Max < i { + return nil + } + return list.Children[i-list.Min] +} + +func (list *DenseChildList) walk(prefix *Prefix, visitor VisitorFunc) error { + for _, child := range list.Children { + if child == nil { + continue + } + *prefix = append(*prefix, child.InternalPrefix...) + if child.InternalItem != nil { + if err := visitor(*prefix, child.InternalItem); err != nil { + if err == SkipSubtree { + *prefix = (*prefix)[:len(*prefix)-len(child.InternalPrefix)] + continue + } + *prefix = (*prefix)[:len(*prefix)-len(child.InternalPrefix)] + return err + } + } + + err := child.InternalChildren.walk(prefix, visitor) + *prefix = (*prefix)[:len(*prefix)-len(child.InternalPrefix)] + if err != nil { + return err + } + } + + return nil +} diff --git a/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia.go b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia.go new file mode 100644 index 000000000..355752dd3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia.go @@ -0,0 +1,432 @@ +// Copyright (c) 2014 The go-patricia AUTHORS +// +// Use of this source code is governed by The MIT License +// that can be found in the LICENSE file. + +package patricia + +import ( + "errors" +) + +//------------------------------------------------------------------------------ +// Trie +//------------------------------------------------------------------------------ + +type ( + Prefix []byte + Item interface{} + VisitorFunc func(prefix Prefix, item Item) error +) + +// Trie is a generic patricia trie that allows fast retrieval of items by prefix. +// and other funky stuff. +// +// Trie is not thread-safe. +type Trie struct { + InternalPrefix Prefix + InternalItem Item + + InternalChildren ChildList +} + +// Public API ------------------------------------------------------------------ + +// Trie constructor. +func NewTrie() *Trie { + return &Trie{ + InternalChildren: newSparseChildList(), + } +} + +// Item returns the item stored in the root of this trie. +func (trie *Trie) Item() Item { + return trie.InternalItem +} + +// Insert inserts a new item into the trie using the given prefix. Insert does +// not replace existing items. It returns false if an item was already in place. +func (trie *Trie) Insert(key Prefix, item Item) (inserted bool) { + return trie.put(key, item, false) +} + +// Set works much like Insert, but it always sets the item, possibly replacing +// the item previously inserted. +func (trie *Trie) Set(key Prefix, item Item) { + trie.put(key, item, true) +} + +// Get returns the item located at key. +// +// This method is a bit dangerous, because Get can as well end up in an internal +// node that is not really representing any user-defined value. So when nil is +// a valid value being used, it is not possible to tell if the value was inserted +// into the tree by the user or not. A possible workaround for this is not to use +// nil interface as a valid value, even using zero value of any type is enough +// to prevent this bad behaviour. +func (trie *Trie) Get(key Prefix) (item Item) { + _, node, found, leftover := trie.findSubtree(key) + if !found || len(leftover) != 0 { + return nil + } + return node.InternalItem +} + +// Match returns what Get(prefix) != nil would return. The same warning as for +// Get applies here as well. +func (trie *Trie) Match(prefix Prefix) (matchedExactly bool) { + return trie.Get(prefix) != nil +} + +// MatchSubtree returns true when there is a subtree representing extensions +// to key, that is if there are any keys in the tree which have key as prefix. +func (trie *Trie) MatchSubtree(key Prefix) (matched bool) { + _, _, matched, _ = trie.findSubtree(key) + return +} + +// Visit calls visitor on every node containing a non-nil item. +// +// If an error is returned from visitor, the function stops visiting the tree +// and returns that error, unless it is a special error - SkipSubtree. In that +// case Visit skips the subtree represented by the current node and continues +// elsewhere. +func (trie *Trie) Visit(visitor VisitorFunc) error { + return trie.walk(nil, visitor) +} + +// VisitSubtree works much like Visit, but it only visits nodes matching prefix. +func (trie *Trie) VisitSubtree(prefix Prefix, visitor VisitorFunc) error { + // Nil prefix not allowed. + if prefix == nil { + panic(ErrNilPrefix) + } + + // Empty trie must be handled explicitly. + if trie.InternalPrefix == nil { + return nil + } + + // Locate the relevant subtree. + _, root, found, leftover := trie.findSubtree(prefix) + if !found { + return nil + } + prefix = append(prefix, leftover...) + + // Visit it. + return root.walk(prefix, visitor) +} + +// VisitPrefixes visits only nodes that represent prefixes of key. +// To say the obvious, returning SkipSubtree from visitor makes no sense here. +func (trie *Trie) VisitPrefixes(key Prefix, visitor VisitorFunc) error { + // Nil key not allowed. + if key == nil { + panic(ErrNilPrefix) + } + + // Empty trie must be handled explicitly. + if trie.InternalPrefix == nil { + return nil + } + + // Walk the path matching key prefixes. + node := trie + prefix := key + offset := 0 + for { + // Compute what part of prefix matches. + common := node.longestCommonPrefixLength(key) + key = key[common:] + offset += common + + // Partial match means that there is no subtree matching prefix. + if common < len(node.InternalPrefix) { + return nil + } + + // Call the visitor. + if item := node.InternalItem; item != nil { + if err := visitor(prefix[:offset], item); err != nil { + return err + } + } + + if len(key) == 0 { + // This node represents key, we are finished. + return nil + } + + // There is some key suffix left, move to the children. + child := node.InternalChildren.next(key[0]) + if child == nil { + // There is nowhere to continue, return. + return nil + } + + node = child + } +} + +// Delete deletes the item represented by the given prefix. +// +// True is returned if the matching node was found and deleted. +func (trie *Trie) Delete(key Prefix) (deleted bool) { + // Nil prefix not allowed. + if key == nil { + panic(ErrNilPrefix) + } + + // Empty trie must be handled explicitly. + if trie.InternalPrefix == nil { + return false + } + + // Find the relevant node. + parent, node, _, leftover := trie.findSubtree(key) + if len(leftover) != 0 { + return false + } + + // If the item is already set to nil, there is nothing to do. + if node.InternalItem == nil { + return false + } + + // Delete the item. + node.InternalItem = nil + + // Compact since that might be possible now. + if compacted := node.compact(); compacted != node { + if parent == nil { + *node = *compacted + } else { + parent.InternalChildren.replace(node.InternalPrefix[0], compacted) + *parent = *parent.compact() + } + } + + return true +} + +// DeleteSubtree finds the subtree exactly matching prefix and deletes it. +// +// True is returned if the subtree was found and deleted. +func (trie *Trie) DeleteSubtree(prefix Prefix) (deleted bool) { + // Nil prefix not allowed. + if prefix == nil { + panic(ErrNilPrefix) + } + + // Empty trie must be handled explicitly. + if trie.InternalPrefix == nil { + return false + } + + // Locate the relevant subtree. + parent, root, found, _ := trie.findSubtree(prefix) + if !found { + return false + } + + // If we are in the root of the trie, reset the trie. + if parent == nil { + root.InternalPrefix = nil + root.InternalChildren = newSparseChildList() + return true + } + + // Otherwise remove the root node from its parent. + parent.InternalChildren.remove(root) + return true +} + +// Internal helper methods ----------------------------------------------------- + +func (trie *Trie) put(key Prefix, item Item, replace bool) (inserted bool) { + // Nil prefix not allowed. + if key == nil { + panic(ErrNilPrefix) + } + + var ( + common int + node *Trie = trie + child *Trie + ) + + if node.InternalPrefix == nil { + if len(key) <= MaxPrefixPerNode { + node.InternalPrefix = key + goto InsertItem + } + node.InternalPrefix = key[:MaxPrefixPerNode] + key = key[MaxPrefixPerNode:] + goto AppendChild + } + + for { + // Compute the longest common prefix length. + common = node.longestCommonPrefixLength(key) + key = key[common:] + + // Only a part matches, split. + if common < len(node.InternalPrefix) { + goto SplitPrefix + } + + // common == len(node.prefix) since never (common > len(node.prefix)) + // common == len(former key) <-> 0 == len(key) + // -> former key == node.prefix + if len(key) == 0 { + goto InsertItem + } + + // Check children for matching prefix. + child = node.InternalChildren.next(key[0]) + if child == nil { + goto AppendChild + } + node = child + } + +SplitPrefix: + // Split the prefix if necessary. + child = new(Trie) + *child = *node + *node = *NewTrie() + node.InternalPrefix = child.InternalPrefix[:common] + child.InternalPrefix = child.InternalPrefix[common:] + child = child.compact() + node.InternalChildren = node.InternalChildren.add(child) + +AppendChild: + // Keep appending children until whole prefix is inserted. + // This loop starts with empty node.prefix that needs to be filled. + for len(key) != 0 { + child := NewTrie() + if len(key) <= MaxPrefixPerNode { + child.InternalPrefix = key + node.InternalChildren = node.InternalChildren.add(child) + node = child + goto InsertItem + } else { + child.InternalPrefix = key[:MaxPrefixPerNode] + key = key[MaxPrefixPerNode:] + node.InternalChildren = node.InternalChildren.add(child) + node = child + } + } + +InsertItem: + // Try to insert the item if possible. + if replace || node.InternalItem == nil { + node.InternalItem = item + return true + } + return false +} + +func (trie *Trie) compact() *Trie { + // Only a node with a single child can be compacted. + if trie.InternalChildren.length() != 1 { + return trie + } + + child := trie.InternalChildren.head() + + // If any item is set, we cannot compact since we want to retain + // the ability to do searching by key. This makes compaction less usable, + // but that simply cannot be avoided. + if trie.InternalItem != nil || child.InternalItem != nil { + return trie + } + + // Make sure the combined prefixes fit into a single node. + if len(trie.InternalPrefix)+len(child.InternalPrefix) > MaxPrefixPerNode { + return trie + } + + // Concatenate the prefixes, move the items. + child.InternalPrefix = append(trie.InternalPrefix, child.InternalPrefix...) + if trie.InternalItem != nil { + child.InternalItem = trie.InternalItem + } + + return child +} + +func (trie *Trie) findSubtree(prefix Prefix) (parent *Trie, root *Trie, found bool, leftover Prefix) { + // Find the subtree matching prefix. + root = trie + for { + // Compute what part of prefix matches. + common := root.longestCommonPrefixLength(prefix) + prefix = prefix[common:] + + // We used up the whole prefix, subtree found. + if len(prefix) == 0 { + found = true + leftover = root.InternalPrefix[common:] + return + } + + // Partial match means that there is no subtree matching prefix. + if common < len(root.InternalPrefix) { + leftover = root.InternalPrefix[common:] + return + } + + // There is some prefix left, move to the children. + child := root.InternalChildren.next(prefix[0]) + if child == nil { + // There is nowhere to continue, there is no subtree matching prefix. + return + } + + parent = root + root = child + } +} + +func (trie *Trie) walk(actualRootPrefix Prefix, visitor VisitorFunc) error { + var prefix Prefix + // Allocate a bit more space for prefix at the beginning. + if actualRootPrefix == nil { + prefix = make(Prefix, 32+len(trie.InternalPrefix)) + copy(prefix, trie.InternalPrefix) + prefix = prefix[:len(trie.InternalPrefix)] + } else { + prefix = make(Prefix, 32+len(actualRootPrefix)) + copy(prefix, actualRootPrefix) + prefix = prefix[:len(actualRootPrefix)] + } + + // Visit the root first. Not that this works for empty trie as well since + // in that case item == nil && len(children) == 0. + if trie.InternalItem != nil { + if err := visitor(prefix, trie.InternalItem); err != nil { + if err == SkipSubtree { + return nil + } + return err + } + } + + // Then continue to the children. + return trie.InternalChildren.walk(&prefix, visitor) +} + +func (trie *Trie) longestCommonPrefixLength(prefix Prefix) (i int) { + for ; i < len(prefix) && i < len(trie.InternalPrefix) && prefix[i] == trie.InternalPrefix[i]; i++ { + } + return +} + +// Errors ---------------------------------------------------------------------- + +var ( + SkipSubtree = errors.New("Skip this subtree") + ErrNilPrefix = errors.New("Nil prefix passed into a method call") +) diff --git a/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_dense_test.go b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_dense_test.go new file mode 100644 index 000000000..346e9a66c --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_dense_test.go @@ -0,0 +1,161 @@ +// Copyright (c) 2014 The go-patricia AUTHORS +// +// Use of this source code is governed by The MIT License +// that can be found in the LICENSE file. + +package patricia + +import ( + "testing" +) + +// Tests ----------------------------------------------------------------------- + +func TestTrie_InsertDense(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"aba", 0, success}, + {"abb", 1, success}, + {"abc", 2, success}, + {"abd", 3, success}, + {"abe", 4, success}, + {"abf", 5, success}, + {"abg", 6, success}, + {"abh", 7, success}, + {"abi", 8, success}, + {"abj", 9, success}, + {"abk", 0, success}, + {"abl", 1, success}, + {"abm", 2, success}, + {"abn", 3, success}, + {"abo", 4, success}, + {"abp", 5, success}, + {"abq", 6, success}, + {"abr", 7, success}, + {"abs", 8, success}, + {"abt", 9, success}, + {"abu", 0, success}, + {"abv", 1, success}, + {"abw", 2, success}, + {"abx", 3, success}, + {"aby", 4, success}, + {"abz", 5, success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} + +func TestTrie_InsertDensePreceeding(t *testing.T) { + trie := NewTrie() + start := byte(70) + // create a dense node + for i := byte(0); i <= MaxChildrenPerSparseNode; i++ { + if !trie.Insert(Prefix([]byte{start + i}), true) { + t.Errorf("insert failed, prefix=%v", start+i) + } + } + // insert some preceeding keys + for i := byte(1); i < start; i *= i + 1 { + if !trie.Insert(Prefix([]byte{start - i}), true) { + t.Errorf("insert failed, prefix=%v", start-i) + } + } +} + +func TestTrie_InsertDenseDuplicatePrefixes(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"aba", 0, success}, + {"abb", 1, success}, + {"abc", 2, success}, + {"abd", 3, success}, + {"abe", 4, success}, + {"abf", 5, success}, + {"abg", 6, success}, + {"abh", 7, success}, + {"abi", 8, success}, + {"abj", 9, success}, + {"abk", 0, success}, + {"abl", 1, success}, + {"abm", 2, success}, + {"abn", 3, success}, + {"abo", 4, success}, + {"abp", 5, success}, + {"abq", 6, success}, + {"abr", 7, success}, + {"abs", 8, success}, + {"abt", 9, success}, + {"abu", 0, success}, + {"abv", 1, success}, + {"abw", 2, success}, + {"abx", 3, success}, + {"aby", 4, success}, + {"abz", 5, success}, + {"aba", 0, failure}, + {"abb", 1, failure}, + {"abc", 2, failure}, + {"abd", 3, failure}, + {"abe", 4, failure}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} + +func TestTrie_DeleteDense(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"aba", 0, success}, + {"abb", 1, success}, + {"abc", 2, success}, + {"abd", 3, success}, + {"abe", 4, success}, + {"abf", 5, success}, + {"abg", 6, success}, + {"abh", 7, success}, + {"abi", 8, success}, + {"abj", 9, success}, + {"abk", 0, success}, + {"abl", 1, success}, + {"abm", 2, success}, + {"abn", 3, success}, + {"abo", 4, success}, + {"abp", 5, success}, + {"abq", 6, success}, + {"abr", 7, success}, + {"abs", 8, success}, + {"abt", 9, success}, + {"abu", 0, success}, + {"abv", 1, success}, + {"abw", 2, success}, + {"abx", 3, success}, + {"aby", 4, success}, + {"abz", 5, success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + for _, v := range data { + t.Logf("DELETE word=%v, success=%v", v.key, v.retVal) + if ok := trie.Delete([]byte(v.key)); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} diff --git a/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_sparse_test.go b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_sparse_test.go new file mode 100644 index 000000000..b96325d19 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_sparse_test.go @@ -0,0 +1,659 @@ +// Copyright (c) 2014 The go-patricia AUTHORS +// +// Use of this source code is governed by The MIT License +// that can be found in the LICENSE file. + +package patricia + +import ( + "bytes" + "errors" + "fmt" + "strings" + "testing" +) + +const ( + success = true + failure = false +) + +type testData struct { + key string + value interface{} + retVal bool +} + +// Tests ----------------------------------------------------------------------- + +func TestTrie_InsertDifferentPrefixes(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepaneeeeeeeeeeeeee", "Pepan Zdepan", success}, + {"Honzooooooooooooooo", "Honza Novak", success}, + {"Jenikuuuuuuuuuuuuuu", "Jenik Poustevnicek", success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} + +func TestTrie_InsertDuplicatePrefixes(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepan", "Pepan Zdepan", success}, + {"Pepan", "Pepan Zdepan", failure}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} + +func TestTrie_InsertVariousPrefixes(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepan", "Pepan Zdepan", success}, + {"Pepin", "Pepin Omacka", success}, + {"Honza", "Honza Novak", success}, + {"Jenik", "Jenik Poustevnicek", success}, + {"Pepan", "Pepan Dupan", failure}, + {"Karel", "Karel Pekar", success}, + {"Jenik", "Jenik Poustevnicek", failure}, + {"Pepanek", "Pepanek Zemlicka", success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} + +func TestTrie_InsertAndMatchPrefix(t *testing.T) { + trie := NewTrie() + t.Log("INSERT prefix=by week") + trie.Insert(Prefix("by week"), 2) + t.Log("INSERT prefix=by") + trie.Insert(Prefix("by"), 1) + + if !trie.Match(Prefix("by")) { + t.Error("MATCH prefix=by, expected=true, got=false") + } +} + +func TestTrie_SetGet(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepan", "Pepan Zdepan", success}, + {"Pepin", "Pepin Omacka", success}, + {"Honza", "Honza Novak", success}, + {"Jenik", "Jenik Poustevnicek", success}, + {"Pepan", "Pepan Dupan", failure}, + {"Karel", "Karel Pekar", success}, + {"Jenik", "Jenik Poustevnicek", failure}, + {"Pepanek", "Pepanek Zemlicka", success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + for _, v := range data { + t.Logf("SET %q to 10", v.key) + trie.Set(Prefix(v.key), 10) + } + + for _, v := range data { + value := trie.Get(Prefix(v.key)) + t.Logf("GET %q => %v", v.key, value) + if value.(int) != 10 { + t.Errorf("Unexpected return value, != 10", value) + } + } + + if value := trie.Get(Prefix("random crap")); value != nil { + t.Errorf("Unexpected return value, %v != ", value) + } +} + +func TestTrie_Match(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepan", "Pepan Zdepan", success}, + {"Pepin", "Pepin Omacka", success}, + {"Honza", "Honza Novak", success}, + {"Jenik", "Jenik Poustevnicek", success}, + {"Pepan", "Pepan Dupan", failure}, + {"Karel", "Karel Pekar", success}, + {"Jenik", "Jenik Poustevnicek", failure}, + {"Pepanek", "Pepanek Zemlicka", success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + for _, v := range data { + matched := trie.Match(Prefix(v.key)) + t.Logf("MATCH %q => %v", v.key, matched) + if !matched { + t.Errorf("Inserted key %q was not matched", v.key) + } + } + + if trie.Match(Prefix("random crap")) { + t.Errorf("Key that was not inserted matched: %q", "random crap") + } +} + +func TestTrie_MatchFalsePositive(t *testing.T) { + trie := NewTrie() + + if ok := trie.Insert(Prefix("A"), 1); !ok { + t.Fatal("INSERT prefix=A, item=1 not ok") + } + + resultMatchSubtree := trie.MatchSubtree(Prefix("A extra")) + resultMatch := trie.Match(Prefix("A extra")) + + if resultMatchSubtree != false { + t.Error("MatchSubtree returned false positive") + } + + if resultMatch != false { + t.Error("Match returned false positive") + } +} + +func TestTrie_MatchSubtree(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepan", "Pepan Zdepan", success}, + {"Pepin", "Pepin Omacka", success}, + {"Honza", "Honza Novak", success}, + {"Jenik", "Jenik Poustevnicek", success}, + {"Pepan", "Pepan Dupan", failure}, + {"Karel", "Karel Pekar", success}, + {"Jenik", "Jenik Poustevnicek", failure}, + {"Pepanek", "Pepanek Zemlicka", success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + for _, v := range data { + key := Prefix(v.key[:3]) + matched := trie.MatchSubtree(key) + t.Logf("MATCH_SUBTREE %q => %v", key, matched) + if !matched { + t.Errorf("Subtree %q was not matched", v.key) + } + } +} + +func TestTrie_Visit(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepa", 0, success}, + {"Pepa Zdepa", 1, success}, + {"Pepa Kuchar", 2, success}, + {"Honza", 3, success}, + {"Jenik", 4, success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + if err := trie.Visit(func(prefix Prefix, item Item) error { + name := data[item.(int)].key + t.Logf("VISITING prefix=%q, item=%v", prefix, item) + if !strings.HasPrefix(string(prefix), name) { + t.Errorf("Unexpected prefix encountered, %q not a prefix of %q", prefix, name) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestTrie_VisitSkipSubtree(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepa", 0, success}, + {"Pepa Zdepa", 1, success}, + {"Pepa Kuchar", 2, success}, + {"Honza", 3, success}, + {"Jenik", 4, success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + if err := trie.Visit(func(prefix Prefix, item Item) error { + t.Logf("VISITING prefix=%q, item=%v", prefix, item) + if item.(int) == 0 { + t.Logf("SKIP %q", prefix) + return SkipSubtree + } + if strings.HasPrefix(string(prefix), "Pepa") { + t.Errorf("Unexpected prefix encountered, %q", prefix) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestTrie_VisitReturnError(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepa", 0, success}, + {"Pepa Zdepa", 1, success}, + {"Pepa Kuchar", 2, success}, + {"Honza", 3, success}, + {"Jenik", 4, success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + someErr := errors.New("Something exploded") + if err := trie.Visit(func(prefix Prefix, item Item) error { + t.Logf("VISITING prefix=%q, item=%v", prefix, item) + if item.(int) == 0 { + return someErr + } + if item.(int) != 0 { + t.Errorf("Unexpected prefix encountered, %q", prefix) + } + return nil + }); err != nil && err != someErr { + t.Fatal(err) + } +} + +func TestTrie_VisitSubtree(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepa", 0, success}, + {"Pepa Zdepa", 1, success}, + {"Pepa Kuchar", 2, success}, + {"Honza", 3, success}, + {"Jenik", 4, success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + var counter int + subtreePrefix := []byte("Pep") + t.Log("VISIT Pep") + if err := trie.VisitSubtree(subtreePrefix, func(prefix Prefix, item Item) error { + t.Logf("VISITING prefix=%q, item=%v", prefix, item) + if !bytes.HasPrefix(prefix, subtreePrefix) { + t.Errorf("Unexpected prefix encountered, %q does not extend %q", + prefix, subtreePrefix) + } + if len(prefix) > len(data[item.(int)].key) { + t.Fatalf("Something is rather fishy here, prefix=%q", prefix) + } + counter++ + return nil + }); err != nil { + t.Fatal(err) + } + + if counter != 3 { + t.Error("Unexpected number of nodes visited") + } +} + +func TestTrie_VisitPrefixes(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"P", 0, success}, + {"Pe", 1, success}, + {"Pep", 2, success}, + {"Pepa", 3, success}, + {"Pepa Zdepa", 4, success}, + {"Pepa Kuchar", 5, success}, + {"Honza", 6, success}, + {"Jenik", 7, success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + var counter int + word := []byte("Pepa") + if err := trie.VisitPrefixes(word, func(prefix Prefix, item Item) error { + t.Logf("VISITING prefix=%q, item=%v", prefix, item) + if !bytes.HasPrefix(word, prefix) { + t.Errorf("Unexpected prefix encountered, %q is not a prefix of %q", + prefix, word) + } + counter++ + return nil + }); err != nil { + t.Fatal(err) + } + + if counter != 4 { + t.Error("Unexpected number of nodes visited") + } +} + +func TestParticiaTrie_Delete(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Pepan", "Pepan Zdepan", success}, + {"Honza", "Honza Novak", success}, + {"Jenik", "Jenik Poustevnicek", success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + for _, v := range data { + t.Logf("DELETE word=%v, success=%v", v.key, v.retVal) + if ok := trie.Delete([]byte(v.key)); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} + +func TestParticiaTrie_DeleteNonExistent(t *testing.T) { + trie := NewTrie() + + insertData := []testData{ + {"Pepan", "Pepan Zdepan", success}, + {"Honza", "Honza Novak", success}, + {"Jenik", "Jenik Poustevnicek", success}, + } + deleteData := []testData{ + {"Pepan", "Pepan Zdepan", success}, + {"Honza", "Honza Novak", success}, + {"Pepan", "Pepan Zdepan", failure}, + {"Jenik", "Jenik Poustevnicek", success}, + {"Honza", "Honza Novak", failure}, + } + + for _, v := range insertData { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + for _, v := range deleteData { + t.Logf("DELETE word=%v, success=%v", v.key, v.retVal) + if ok := trie.Delete([]byte(v.key)); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} + +func TestParticiaTrie_DeleteSubtree(t *testing.T) { + trie := NewTrie() + + insertData := []testData{ + {"P", 0, success}, + {"Pe", 1, success}, + {"Pep", 2, success}, + {"Pepa", 3, success}, + {"Pepa Zdepa", 4, success}, + {"Pepa Kuchar", 5, success}, + {"Honza", 6, success}, + {"Jenik", 7, success}, + } + deleteData := []testData{ + {"Pe", -1, success}, + {"Pe", -1, failure}, + {"Honzik", -1, failure}, + {"Honza", -1, success}, + {"Honza", -1, failure}, + {"Pep", -1, failure}, + {"P", -1, success}, + {"Nobody", -1, failure}, + {"", -1, success}, + } + + for _, v := range insertData { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Fatalf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + for _, v := range deleteData { + t.Logf("DELETE_SUBTREE prefix=%v, success=%v", v.key, v.retVal) + if ok := trie.DeleteSubtree([]byte(v.key)); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } +} + +/* +func TestTrie_Dump(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"Honda", nil, success}, + {"Honza", nil, success}, + {"Jenik", nil, success}, + {"Pepan", nil, success}, + {"Pepin", nil, success}, + } + + for i, v := range data { + if _, ok := trie.Insert([]byte(v.key), v.value); ok != v.retVal { + t.Logf("INSERT %v %v", v.key, v.value) + t.Fatalf("Unexpected return value, expected=%v, got=%v", i, ok) + } + } + + dump := ` ++--+--+ Hon +--+--+ da + | | + | +--+ za + | + +--+ Jenik + | + +--+ Pep +--+--+ an + | + +--+ in +` + + var buf bytes.Buffer + trie.Dump(buf) + + if !bytes.Equal(buf.Bytes(), dump) { + t.Logf("DUMP") + t.Fatalf("Unexpected dump generated, expected\n\n%v\ngot\n\n%v", dump, buf.String()) + } +} +*/ + +func TestTrie_compact(t *testing.T) { + trie := NewTrie() + + trie.Insert(Prefix("a"), 0) + trie.Insert(Prefix("ab"), 0) + trie.Insert(Prefix("abc"), 0) + trie.Insert(Prefix("abcd"), 0) + trie.Insert(Prefix("abcde"), 0) + trie.Insert(Prefix("abcdef"), 0) + trie.Insert(Prefix("abcdefg"), 0) + trie.Insert(Prefix("abcdefgi"), 0) + trie.Insert(Prefix("abcdefgij"), 0) + trie.Insert(Prefix("abcdefgijk"), 0) + + trie.Delete(Prefix("abcdef")) + trie.Delete(Prefix("abcde")) + trie.Delete(Prefix("abcdefg")) + + trie.Delete(Prefix("a")) + trie.Delete(Prefix("abc")) + trie.Delete(Prefix("ab")) + + trie.Visit(func(prefix Prefix, item Item) error { + // 97 ~~ 'a', + for ch := byte(97); ch <= 107; ch++ { + if c := bytes.Count(prefix, []byte{ch}); c > 1 { + t.Errorf("%q appeared in %q %v times", ch, prefix, c) + } + } + return nil + }) +} + +func TestTrie_longestCommonPrefixLenght(t *testing.T) { + trie := NewTrie() + trie.InternalPrefix = []byte("1234567890") + + switch { + case trie.longestCommonPrefixLength([]byte("")) != 0: + t.Fail() + case trie.longestCommonPrefixLength([]byte("12345")) != 5: + t.Fail() + case trie.longestCommonPrefixLength([]byte("123789")) != 3: + t.Fail() + case trie.longestCommonPrefixLength([]byte("12345678901")) != 10: + t.Fail() + } +} + +// Examples -------------------------------------------------------------------- + +func ExampleTrie() { + // Create a new tree. + trie := NewTrie() + + // Insert some items. + trie.Insert(Prefix("Pepa Novak"), 1) + trie.Insert(Prefix("Pepa Sindelar"), 2) + trie.Insert(Prefix("Karel Macha"), 3) + trie.Insert(Prefix("Karel Hynek Macha"), 4) + + // Just check if some things are present in the tree. + key := Prefix("Pepa Novak") + fmt.Printf("%q present? %v\n", key, trie.Match(key)) + key = Prefix("Karel") + fmt.Printf("Anybody called %q here? %v\n", key, trie.MatchSubtree(key)) + + // Walk the tree. + trie.Visit(printItem) + // "Pepa Novak": 1 + // "Pepa Sindelar": 2 + // "Karel Macha": 3 + // "Karel Hynek Macha": 4 + + // Walk a subtree. + trie.VisitSubtree(Prefix("Pepa"), printItem) + // "Pepa Novak": 1 + // "Pepa Sindelar": 2 + + // Modify an item, then fetch it from the tree. + trie.Set(Prefix("Karel Hynek Macha"), 10) + key = Prefix("Karel Hynek Macha") + fmt.Printf("%q: %v\n", key, trie.Get(key)) + // "Karel Hynek Macha": 10 + + // Walk prefixes. + prefix := Prefix("Karel Hynek Macha je kouzelnik") + trie.VisitPrefixes(prefix, printItem) + // "Karel Hynek Macha": 10 + + // Delete some items. + trie.Delete(Prefix("Pepa Novak")) + trie.Delete(Prefix("Karel Macha")) + + // Walk again. + trie.Visit(printItem) + // "Pepa Sindelar": 2 + // "Karel Hynek Macha": 10 + + // Delete a subtree. + trie.DeleteSubtree(Prefix("Pepa")) + + // Print what is left. + trie.Visit(printItem) + // "Karel Hynek Macha": 10 + + // Output: + // "Pepa Novak" present? true + // Anybody called "Karel" here? true + // "Pepa Novak": 1 + // "Pepa Sindelar": 2 + // "Karel Macha": 3 + // "Karel Hynek Macha": 4 + // "Pepa Novak": 1 + // "Pepa Sindelar": 2 + // "Karel Hynek Macha": 10 + // "Karel Hynek Macha": 10 + // "Pepa Sindelar": 2 + // "Karel Hynek Macha": 10 + // "Karel Hynek Macha": 10 +} + +// Helpers --------------------------------------------------------------------- + +func printItem(prefix Prefix, item Item) error { + fmt.Printf("%q: %v\n", prefix, item) + return nil +} diff --git a/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_test.go b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_test.go new file mode 100644 index 000000000..0e16a4c46 --- /dev/null +++ b/Godeps/_workspace/src/github.com/minio-io/go-patricia/patricia/patricia_test.go @@ -0,0 +1,105 @@ +// Copyright (c) 2014 The go-patricia AUTHORS +// +// Use of this source code is governed by The MIT License +// that can be found in the LICENSE file. + +package patricia + +import ( + "bytes" + "crypto/rand" + "encoding/gob" + "log" + "reflect" + "testing" +) + +// Tests ----------------------------------------------------------------------- + +func TestTrie_GetNonexistentPrefix(t *testing.T) { + trie := NewTrie() + + data := []testData{ + {"aba", 0, success}, + } + + for _, v := range data { + t.Logf("INSERT prefix=%v, item=%v, success=%v", v.key, v.value, v.retVal) + if ok := trie.Insert(Prefix(v.key), v.value); ok != v.retVal { + t.Errorf("Unexpected return value, expected=%v, got=%v", v.retVal, ok) + } + } + + t.Logf("GET prefix=baa, expect item=nil") + if item := trie.Get(Prefix("baa")); item != nil { + t.Errorf("Unexpected return value, expected=, got=%v", item) + } +} + +func TestTrie_RandomKitchenSink(t *testing.T) { + if testing.Short() { + t.Skip() + } + const count, size = 750000, 16 + b := make([]byte, count+size+1) + if _, err := rand.Read(b); err != nil { + t.Fatal("error generating random bytes", err) + } + m := make(map[string]string) + for i := 0; i < count; i++ { + m[string(b[i:i+size])] = string(b[i+1 : i+size+1]) + } + trie := NewTrie() + getAndDelete := func(k, v string) { + i := trie.Get(Prefix(k)) + if i == nil { + t.Fatalf("item not found, prefix=%v", []byte(k)) + } else if s, ok := i.(string); !ok { + t.Fatalf("unexpected item type, expecting=%v, got=%v", reflect.TypeOf(k), reflect.TypeOf(i)) + } else if s != v { + t.Fatalf("unexpected item, expecting=%v, got=%v", []byte(k), []byte(s)) + } else if !trie.Delete(Prefix(k)) { + t.Fatalf("delete failed, prefix=%v", []byte(k)) + } else if i = trie.Get(Prefix(k)); i != nil { + t.Fatalf("unexpected item, expecting=, got=%v", i) + } else if trie.Delete(Prefix(k)) { + t.Fatalf("extra delete succeeded, prefix=%v", []byte(k)) + } + } + for k, v := range m { + if !trie.Insert(Prefix(k), v) { + t.Fatalf("insert failed, prefix=%v", []byte(k)) + } + if byte(k[size/2]) < 128 { + getAndDelete(k, v) + delete(m, k) + } + } + for k, v := range m { + getAndDelete(k, v) + } +} + +func TestTrieSerializes(t *testing.T) { + gob.Register(&SparseChildList{}) + gob.Register(&DenseChildList{}) + trie := NewTrie() + trie.Insert([]byte("hello1"), "world1") + trie.Insert([]byte("hello2"), "world2") + var bytesBuffer bytes.Buffer + encoder := gob.NewEncoder(&bytesBuffer) + if err := encoder.Encode(trie); err != nil { + t.Fatalf("Serialization failed:", err) + } + encodedReader := bytes.NewReader(bytesBuffer.Bytes()) + decoder := gob.NewDecoder(encodedReader) + trie2 := NewTrie() + decoder.Decode(&trie2) + log.Println(trie2) + if value := trie2.Get([]byte("hello1")); value == nil { + t.Fatalf("hello1 not serialized") + } + if value := trie2.Get([]byte("hello2")); value == nil { + t.Fatalf("hello1 not serialized") + } +} diff --git a/Makefile b/Makefile index dc58be645..7b2dfb045 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,7 @@ install: build-erasure @godep go install github.com/minio-io/minio/cmd/new-cmd && echo "Installed new-cmd into ${GOPATH}/bin" @godep go install github.com/minio-io/minio/cmd/crypto && echo "Installed crypto into ${GOPATH}/bin" @godep go install github.com/minio-io/minio/cmd/split && echo "Installed split into ${GOPATH}/bin" + @godep go install github.com/minio-io/minio/cmd/index && echo "Installed index into ${GOPATH}/bin" save: restore @godep save ./... diff --git a/cmd/index/index-options.go b/cmd/index/index-options.go new file mode 100644 index 000000000..451f5c713 --- /dev/null +++ b/cmd/index/index-options.go @@ -0,0 +1,188 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "encoding/gob" + "fmt" + "io/ioutil" + "log" + "os" + + "github.com/codegangsta/cli" + "github.com/minio-io/go-patricia/patricia" + "github.com/minio-io/minio/pkg/storage" +) + +var Options = []cli.Command{ + Add, + Get, + List, + Remove, +} + +var Add = cli.Command{ + Name: "add", + Usage: "", + Description: "", + Action: doAdd, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "path", + Value: "objects.trie", + }, + }, +} + +var Get = cli.Command{ + Name: "get", + Usage: "", + Description: "", + Action: doGet, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "path", + Value: "objects.trie", + }, + }, +} + +var List = cli.Command{ + Name: "list", + Usage: "", + Description: "", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "path", + Value: "objects.trie", + }, + }, + Action: doList, +} + +var Remove = cli.Command{ + Name: "remove", + Usage: "", + Description: "", + Action: doRemove, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "path", + Value: "objects.trie", + }, + }, +} + +func doAdd(c *cli.Context) { + // register patricia for gob + registerPatriciaWithGob() + + trie, err := readTrie(c.String("path")) + if err != nil { + log.Panic("err: ", trie, err) + } + log.Println("trie: ", trie, err) + + var object storage.ObjectDescription + object.Name = c.Args().Get(1) + object.Md5sum = "md5sum" + object.Murmur3 = "murmur3" + + key := []byte(c.Args().Get(0)) + + trie.Insert(key, object) + log.Println("newTrie:", trie) + saveTrie(c.String("path"), trie) +} + +func doGet(c *cli.Context) { + registerPatriciaWithGob() + + trie, err := readTrie(c.String("path")) + if err != nil { + log.Panic("err: ", trie, err) + } + description := trie.Get([]byte(c.Args().Get(0))) + log.Println(trie) + log.Println(description) + +} +func doList(c *cli.Context) { + registerPatriciaWithGob() + trie, err := readTrie(c.String("path")) + if err != nil { + log.Panic("err: ", trie, err) + } + prefix := patricia.Prefix(c.Args().Get(0)) + trie.VisitSubtree(prefix, func(prefix patricia.Prefix, item patricia.Item) error { + fmt.Println(string(prefix)) + return nil + }) +} + +func doRemove(c *cli.Context) { + registerPatriciaWithGob() + trie, err := readTrie(c.String("path")) + if err != nil { + log.Panic("err: ", trie, err) + } + prefix := patricia.Prefix(c.Args().Get(0)) + trie.Delete(prefix) + saveTrie(c.String("path"), trie) +} + +func readTrie(path string) (*patricia.Trie, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return patricia.NewTrie(), nil + } else { + return nil, err + } + } + file, err := os.Open(path) + if err != nil { + log.Fatal(err) + } + decoder := gob.NewDecoder(file) + trie := patricia.NewTrie() + if err = decoder.Decode(trie); err != nil { + return nil, err + } + return trie, nil +} + +func saveTrie(path string, trie *patricia.Trie) error { + // file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC, 0600) + // if err != nil { + // return err + // } + var buffer bytes.Buffer + encoder := gob.NewEncoder(&buffer) + if err := encoder.Encode(trie); err != nil { + log.Panic(err) + } + log.Println("marshalled trie:", buffer.Bytes()) + err := ioutil.WriteFile(path, buffer.Bytes(), 0600) + return err +} + +func registerPatriciaWithGob() { + gob.Register(storage.ObjectDescription{}) + gob.Register(&patricia.SparseChildList{}) + gob.Register(&patricia.DenseChildList{}) +} diff --git a/cmd/index/index.go b/cmd/index/index.go new file mode 100644 index 000000000..1dcb769c8 --- /dev/null +++ b/cmd/index/index.go @@ -0,0 +1,32 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "os" + + "github.com/codegangsta/cli" +) + +func main() { + app := cli.NewApp() + app.Name = "index" + app.Usage = "" + app.Commands = Options + app.Author = "Minio" + app.Run(os.Args) +} diff --git a/cmd/index/index.md b/cmd/index/index.md new file mode 100644 index 000000000..7f12428bc --- /dev/null +++ b/cmd/index/index.md @@ -0,0 +1,14 @@ + +% MINIO(1) Minio Manual +% Minio community +% January 2015 +# NAME +index - + +# SYNOPSIS + +# DESCRIPTION + +# EXAMPLES + +# AUTHORS