CEF/ergo
1
0
Fork 0
forked from External/ergo
Code Pull requests Activity

bump buntdb to v1.2.3

Browse source 
Potentially fixes the database corruption seen on #1603
This commit is contained in:
Shivaram Lingamneni 2021-04-01 20:45:15 -04:00
parent b022c34a23
commit fd3cbab6ee
36 changed files with 912 additions and 324 deletions
Download patch file Download diff file Expand all files Collapse all files

1
vendor/github.com/tidwall/btree/.travis.yml generated vendored
View file

@ -1 +0,0 @@
language: go

198
vendor/github.com/tidwall/btree/README.md generated vendored
View file

@ -1,5 +1,199 @@
# B-tree for Go
# btree
![Travis CI Build Status](https://api.travis-ci.org/tidwall/btree.svg?branch=master)
[![GoDoc](https://godoc.org/github.com/tidwall/btree?status.svg)](https://godoc.org/github.com/tidwall/btree)
An [efficient](#performance) [B-tree](https://en.wikipedia.org/wiki/B-tree) implementation in Go.
## Features
- `Copy()` method with copy-on-write support.
- Fast bulk loading for pre-ordered data using the `Load()` method.
- All operations are thread-safe.
- Path hinting optimization for operations with nearby keys.
## Installing
To start using btree, install Go and run `go get`:
```sh
$ go get -u github.com/tidwall/btree
```
## Usage
```go
package main
import (
"fmt"
"github.com/tidwall/btree"
)
type Item struct {
Key, Val string
}
// byKeys is a comparison function that compares item keys and returns true
// when a is less than b.
func byKeys(a, b interface{}) bool {
i1, i2 := a.(*Item), b.(*Item)
return i1.Key < i2.Key
}
// byVals is a comparison function that compares item values and returns true
// when a is less than b.
func byVals(a, b interface{}) bool {
i1, i2 := a.(*Item), b.(*Item)
if i1.Val < i2.Val {
return true
}
if i1.Val > i2.Val {
return false
}
// Both vals are equal so we should fall though
// and let the key comparison take over.
return byKeys(a, b)
}
func main() {
// Create a tree for keys and a tree for values.
// The "keys" tree will be sorted on the Keys field.
// The "values" tree will be sorted on the Values field.
keys := btree.New(byKeys)
vals := btree.New(byVals)
// Create some items.
users := []*Item{
&Item{Key: "user:1", Val: "Jane"},
&Item{Key: "user:2", Val: "Andy"},
&Item{Key: "user:3", Val: "Steve"},
&Item{Key: "user:4", Val: "Andrea"},
&Item{Key: "user:5", Val: "Janet"},
&Item{Key: "user:6", Val: "Andy"},
}
// Insert each user into both trees
for _, user := range users {
keys.Set(user)
vals.Set(user)
}
// Iterate over each user in the key tree
keys.Ascend(nil, func(item interface{}) bool {
kvi := item.(*Item)
fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
return true
})
fmt.Printf("\n")
// Iterate over each user in the val tree
vals.Ascend(nil, func(item interface{}) bool {
kvi := item.(*Item)
fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
return true
})
// Output:
// user:1 Jane
// user:2 Andy
// user:3 Steve
// user:4 Andrea
// user:5 Janet
// user:6 Andy
//
// user:4 Andrea
// user:2 Andy
// user:6 Andy
// user:1 Jane
// user:5 Janet
// user:3 Steve
}
```
## Operations
### Basic
```
Len() # return the number of items in the btree
Set(item) # insert or replace an existing item
Get(item) # get an existing item
Delete(item) # delete an item
```
### Iteration
```
Ascend(pivot, iter) # scan items in ascending order starting at pivot.
Descend(pivot, iter) # scan items in descending order starting at pivot.
```
### Queues
```
Min() # return the first item in the btree
Max() # return the last item in the btree
PopMin() # remove and return the first item in the btree
PopMax() # remove and return the last item in the btree
```
### Bulk loading
```
Load(item) # load presorted items into tree
```
### Path hints
```
SetHint(item, *hint) # insert or replace an existing item
GetHint(item, *hint) # get an existing item
DeleteHint(item, *hint) # delete an item
```
## Performance
This implementation was designed with performance in mind.
The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel Core i9) using Go 1.15.3. The items are simple 8-byte ints.
- `google`: The [google/btree](https://github.com/google/btree) package
- `tidwall`: The [tidwall/btree](https://github.com/tidwall/btree) package
- `go-arr`: Just a simple Go array
```
** sequential set **
google: set-seq 1,000,000 ops in 160ms, 6,262,097/sec, 159 ns/op, 31.0 MB, 32 bytes/op
tidwall: set-seq 1,000,000 ops in 142ms, 7,020,721/sec, 142 ns/op, 36.6 MB, 38 bytes/op
tidwall: set-seq-hint 1,000,000 ops in 87ms, 11,503,315/sec, 86 ns/op, 36.6 MB, 38 bytes/op
tidwall: load-seq 1,000,000 ops in 37ms, 27,177,242/sec, 36 ns/op, 36.6 MB, 38 bytes/op
go-arr: append 1,000,000 ops in 49ms, 20,574,760/sec, 48 ns/op
** random set **
google: set-rand 1,000,000 ops in 606ms, 1,649,921/sec, 606 ns/op, 21.5 MB, 22 bytes/op
tidwall: set-rand 1,000,000 ops in 543ms, 1,841,590/sec, 543 ns/op, 26.7 MB, 27 bytes/op
tidwall: set-rand-hint 1,000,000 ops in 573ms, 1,745,624/sec, 572 ns/op, 26.4 MB, 27 bytes/op
tidwall: set-again 1,000,000 ops in 452ms, 2,212,581/sec, 451 ns/op, 27.1 MB, 28 bytes/op
tidwall: set-after-copy 1,000,000 ops in 472ms, 2,117,457/sec, 472 ns/op, 27.9 MB, 29 bytes/op
tidwall: load-rand 1,000,000 ops in 551ms, 1,816,498/sec, 550 ns/op, 26.1 MB, 27 bytes/op
** sequential get **
google: get-seq 1,000,000 ops in 133ms, 7,497,604/sec, 133 ns/op
tidwall: get-seq 1,000,000 ops in 110ms, 9,082,972/sec, 110 ns/op
tidwall: get-seq-hint 1,000,000 ops in 55ms, 18,289,945/sec, 54 ns/op
** random get **
google: get-rand 1,000,000 ops in 149ms, 6,704,337/sec, 149 ns/op
tidwall: get-rand 1,000,000 ops in 131ms, 7,616,296/sec, 131 ns/op
tidwall: get-rand-hint 1,000,000 ops in 216ms, 4,632,532/sec, 215 ns/op
```
*You can find the benchmark utility at [tidwall/btree-benchmark](https://github.com/tidwall/btree-benchmark)*
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
## License
Source code is available under the MIT [License](/LICENSE).

260
vendor/github.com/tidwall/btree/btree.go generated vendored
View file

@ -4,35 +4,39 @@
package btree
import "sync"
const maxItems = 255
const minItems = maxItems * 40 / 100
type cow struct {
_ int // it cannot be an empty struct
}
type node struct {
cow *cow
leaf bool
numItems int16
items [maxItems]interface{}
children *[maxItems + 1]*node
}
type justaLeaf struct {
leaf bool
numItems int16
items [maxItems]interface{}
}
// BTree is an ordered set items
type BTree struct {
mu *sync.RWMutex
cow *cow
root *node
length int
less func(a, b interface{}) bool
lnode *node
}
func newNode(leaf bool) *node {
func (tr *BTree) newNode(leaf bool) *node {
n := &node{leaf: leaf}
if !leaf {
n.children = new([maxItems + 1]*node)
}
n.cow = tr.cow
return n
}
@ -48,6 +52,7 @@ func New(less func(a, b interface{}) bool) *BTree {
panic("nil less")
}
tr := new(BTree)
tr.mu = new(sync.RWMutex)
tr.less = less
return tr
}
@ -85,8 +90,7 @@ func (n *node) find(key interface{}, less func(a, b interface{}) bool,
high = mid - 1
}
}
if low > 0 && !less(n.items[low-1], key) &&
!less(key, n.items[low-1]) {
if low > 0 && !less(n.items[low-1], key) {
index = low - 1
found = true
} else {
@ -109,22 +113,29 @@ func (tr *BTree) SetHint(item interface{}, hint *PathHint) (prev interface{}) {
if item == nil {
panic("nil item")
}
tr.mu.Lock()
prev = tr.setHint(item, hint)
tr.mu.Unlock()
return prev
}
func (tr *BTree) setHint(item interface{}, hint *PathHint) (prev interface{}) {
if tr.root == nil {
tr.root = newNode(true)
tr.root = tr.newNode(true)
tr.root.items[0] = item
tr.root.numItems = 1
tr.length = 1
return
}
prev = tr.root.set(item, tr.less, hint, 0)
prev = tr.nodeSet(&tr.root, item, tr.less, hint, 0)
if prev != nil {
return prev
}
tr.lnode = nil
if tr.root.numItems == maxItems {
n := tr.root
right, median := n.split()
tr.root = newNode(false)
n := tr.cowLoad(&tr.root)
right, median := tr.nodeSplit(n)
tr.root = tr.newNode(false)
tr.root.children[0] = n
tr.root.items[0] = median
tr.root.children[1] = right
@ -139,8 +150,8 @@ func (tr *BTree) Set(item interface{}) (prev interface{}) {
return tr.SetHint(item, nil)
}
func (n *node) split() (right *node, median interface{}) {
right = newNode(n.leaf)
func (tr *BTree) nodeSplit(n *node) (right *node, median interface{}) {
right = tr.newNode(n.leaf)
median = n.items[maxItems/2]
copy(right.items[:maxItems/2], n.items[maxItems/2+1:])
if !n.leaf {
@ -159,9 +170,30 @@ func (n *node) split() (right *node, median interface{}) {
return right, median
}
func (n *node) set(item interface{}, less func(a, b interface{}) bool,
hint *PathHint, depth int,
//go:noinline
func (tr *BTree) copy(n *node) *node {
n2 := *n
n2.cow = tr.cow
copy(n2.items[:], n.items[:])
if n.children != nil {
n2.children = new([maxItems + 1]*node)
copy(n2.children[:], n.children[:])
}
return &n2
}
// cowLoad loaded the provide node and, if needed, performs a copy-on-write.
func (tr *BTree) cowLoad(cn **node) *node {
if (*cn).cow != tr.cow {
*cn = tr.copy(*cn)
}
return *cn
}
func (tr *BTree) nodeSet(cn **node, item interface{},
less func(a, b interface{}) bool, hint *PathHint, depth int,
) (prev interface{}) {
n := tr.cowLoad(cn)
i, found := n.find(item, less, hint, depth)
if found {
prev = n.items[i]
@ -174,12 +206,12 @@ func (n *node) set(item interface{}, less func(a, b interface{}) bool,
n.numItems++
return nil
}
prev = n.children[i].set(item, less, hint, depth+1)
prev = tr.nodeSet(&n.children[i], item, less, hint, depth+1)
if prev != nil {
return prev
}
if n.children[i].numItems == maxItems {
right, median := n.children[i].split()
right, median := tr.nodeSplit(n.children[i])
copy(n.children[i+1:], n.children[i:])
copy(n.items[i+1:], n.items[i:])
n.items[i] = median
@ -216,6 +248,8 @@ func (tr *BTree) Get(key interface{}) interface{} {
// GetHint gets a value for key using a path hint
func (tr *BTree) GetHint(key interface{}, hint *PathHint) interface{} {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil || key == nil {
return nil
}
@ -246,10 +280,17 @@ func (tr *BTree) Delete(key interface{}) interface{} {
// DeleteHint deletes a value for a key using a path hint
func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} {
tr.mu.Lock()
prev := tr.deleteHint(key, hint)
tr.mu.Unlock()
return prev
}
func (tr *BTree) deleteHint(key interface{}, hint *PathHint) interface{} {
if tr.root == nil || key == nil {
return nil
}
prev := tr.root.delete(false, key, tr.less, hint, 0)
prev := tr.delete(&tr.root, false, key, tr.less, hint, 0)
if prev == nil {
return nil
}
@ -264,9 +305,10 @@ func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} {
return prev
}
func (n *node) delete(max bool, key interface{},
func (tr *BTree) delete(cn **node, max bool, key interface{},
less func(a, b interface{}) bool, hint *PathHint, depth int,
) interface{} {
n := tr.cowLoad(cn)
var i int16
var found bool
if max {
@ -290,74 +332,79 @@ func (n *node) delete(max bool, key interface{},
if found {
if max {
i++
prev = n.children[i].delete(true, "", less, nil, 0)
prev = tr.delete(&n.children[i], true, "", less, nil, 0)
} else {
prev = n.items[i]
maxItem := n.children[i].delete(true, "", less, nil, 0)
maxItem := tr.delete(&n.children[i], true, "", less, nil, 0)
n.items[i] = maxItem
}
} else {
prev = n.children[i].delete(max, key, less, hint, depth+1)
prev = tr.delete(&n.children[i], max, key, less, hint, depth+1)
}
if prev == nil {
return nil
}
if n.children[i].numItems < minItems {
if i == n.numItems {
i--
if n.children[i].numItems >= minItems {
return prev
}
// merge / rebalance nodes
if i == n.numItems {
i--
}
n.children[i] = tr.cowLoad(&n.children[i])
n.children[i+1] = tr.cowLoad(&n.children[i+1])
if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems {
// merge left + item + right
n.children[i].items[n.children[i].numItems] = n.items[i]
copy(n.children[i].items[n.children[i].numItems+1:],
n.children[i+1].items[:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i].children[n.children[i].numItems+1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems {
// merge left + item + right
n.children[i].items[n.children[i].numItems] = n.items[i]
copy(n.children[i].items[n.children[i].numItems+1:],
n.children[i+1].items[:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i].children[n.children[i].numItems+1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
n.children[i].numItems += n.children[i+1].numItems + 1
copy(n.items[i:], n.items[i+1:n.numItems])
copy(n.children[i+1:], n.children[i+2:n.numItems+1])
n.items[n.numItems] = nil
n.children[n.numItems+1] = nil
n.numItems--
} else if n.children[i].numItems > n.children[i+1].numItems {
// move left -> right
copy(n.children[i+1].items[1:],
n.children[i+1].items[:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i+1].children[1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
n.children[i+1].items[0] = n.items[i]
if !n.children[0].leaf {
n.children[i+1].children[0] =
n.children[i].children[n.children[i].numItems]
}
n.children[i+1].numItems++
n.items[i] = n.children[i].items[n.children[i].numItems-1]
n.children[i].items[n.children[i].numItems-1] = nil
if !n.children[0].leaf {
n.children[i].children[n.children[i].numItems] = nil
}
n.children[i].numItems--
} else {
// move right -> left
n.children[i].items[n.children[i].numItems] = n.items[i]
if !n.children[0].leaf {
n.children[i].children[n.children[i].numItems+1] =
n.children[i+1].children[0]
}
n.children[i].numItems++
n.items[i] = n.children[i+1].items[0]
copy(n.children[i+1].items[:],
n.children[i+1].items[1:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i+1].children[:],
n.children[i+1].children[1:n.children[i+1].numItems+1])
}
n.children[i+1].numItems--
n.children[i].numItems += n.children[i+1].numItems + 1
copy(n.items[i:], n.items[i+1:n.numItems])
copy(n.children[i+1:], n.children[i+2:n.numItems+1])
n.items[n.numItems] = nil
n.children[n.numItems+1] = nil
n.numItems--
} else if n.children[i].numItems > n.children[i+1].numItems {
// move left -> right
copy(n.children[i+1].items[1:],
n.children[i+1].items[:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i+1].children[1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
n.children[i+1].items[0] = n.items[i]
if !n.children[0].leaf {
n.children[i+1].children[0] =
n.children[i].children[n.children[i].numItems]
}
n.children[i+1].numItems++
n.items[i] = n.children[i].items[n.children[i].numItems-1]
n.children[i].items[n.children[i].numItems-1] = nil
if !n.children[0].leaf {
n.children[i].children[n.children[i].numItems] = nil
}
n.children[i].numItems--
} else {
// move right -> left
n.children[i].items[n.children[i].numItems] = n.items[i]
if !n.children[0].leaf {
n.children[i].children[n.children[i].numItems+1] =
n.children[i+1].children[0]
}
n.children[i].numItems++
n.items[i] = n.children[i+1].items[0]
copy(n.children[i+1].items[:],
n.children[i+1].items[1:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i+1].children[:],
n.children[i+1].children[1:n.children[i+1].numItems+1])
}
n.children[i+1].numItems--
}
return prev
}
@ -366,6 +413,8 @@ func (n *node) delete(max bool, key interface{},
// Pass nil for pivot to scan all item in ascending order
// Return false to stop iterating
func (tr *BTree) Ascend(pivot interface{}, iter func(item interface{}) bool) {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil {
return
}
@ -427,6 +476,8 @@ func (n *node) reverse(iter func(item interface{}) bool) bool {
// Pass nil for pivot to scan all item in descending order
// Return false to stop iterating
func (tr *BTree) Descend(pivot interface{}, iter func(item interface{}) bool) {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil {
return
}
@ -467,6 +518,12 @@ func (tr *BTree) Load(item interface{}) interface{} {
if item == nil {
panic("nil item")
}
tr.mu.Lock()
defer tr.mu.Unlock()
// Load does not need a cowGrid because the Copy operation sets the
// lnode to nil.
if tr.lnode != nil && tr.lnode.numItems < maxItems-2 {
if tr.less(tr.lnode.items[tr.lnode.numItems-1], item) {
tr.lnode.items[tr.lnode.numItems] = item
@ -475,7 +532,7 @@ func (tr *BTree) Load(item interface{}) interface{} {
return nil
}
}
prev := tr.Set(item)
prev := tr.setHint(item, nil)
if prev != nil {
return prev
}
@ -493,6 +550,8 @@ func (tr *BTree) Load(item interface{}) interface{} {
// Min returns the minimum item in tree.
// Returns nil if the tree has no items.
func (tr *BTree) Min() interface{} {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil {
return nil
}
@ -508,6 +567,8 @@ func (tr *BTree) Min() interface{} {
// Max returns the maximum item in tree.
// Returns nil if the tree has no items.
func (tr *BTree) Max() interface{} {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil {
return nil
}
@ -523,53 +584,65 @@ func (tr *BTree) Max() interface{} {
// PopMin removes the minimum item in tree and returns it.
// Returns nil if the tree has no items.
func (tr *BTree) PopMin() interface{} {
tr.mu.Lock()
defer tr.mu.Unlock()
if tr.root == nil {
return nil
}
tr.lnode = nil
n := tr.root
n := tr.cowLoad(&tr.root)
for {
if n.leaf {
item := n.items[0]
if n.numItems == minItems {
return tr.Delete(item)
return tr.deleteHint(item, nil)
}
copy(n.items[:], n.items[1:])
n.items[n.numItems-1] = nil
n.numItems--
tr.length--
if tr.length == 0 {
tr.root = nil
}
return item
}
n = n.children[0]
n = tr.cowLoad(&n.children[0])
}
}
// PopMax removes the minimum item in tree and returns it.
// Returns nil if the tree has no items.
func (tr *BTree) PopMax() interface{} {
tr.mu.Lock()
defer tr.mu.Unlock()
if tr.root == nil {
return nil
}
tr.lnode = nil
n := tr.root
n := tr.cowLoad(&tr.root)
for {
if n.leaf {
item := n.items[n.numItems-1]
if n.numItems == minItems {
return tr.Delete(item)
return tr.deleteHint(item, nil)
}
n.items[n.numItems-1] = nil
n.numItems--
tr.length--
if tr.length == 0 {
tr.root = nil
}
return item
}
n = n.children[n.numItems]
n = tr.cowLoad(&n.children[n.numItems])
}
}
// Height returns the height of the tree.
// Returns zero if tree has no items.
func (tr *BTree) Height() int {
tr.mu.RLock()
defer tr.mu.RUnlock()
var height int
if tr.root != nil {
n := tr.root
@ -587,6 +660,8 @@ func (tr *BTree) Height() int {
// Walk iterates over all items in tree, in order.
// The items param will contain one or more items.
func (tr *BTree) Walk(iter func(item []interface{})) {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root != nil {
tr.root.walk(iter)
}
@ -603,3 +678,16 @@ func (n *node) walk(iter func(item []interface{})) {
n.children[n.numItems].walk(iter)
}
}
// Copy the tree. This operation is very fast because it only performs a
// shadowed copy.
func (tr *BTree) Copy() *BTree {
tr.mu.Lock()
tr.lnode = nil
tr.cow = new(cow)
tr2 := *tr
tr2.mu = new(sync.RWMutex)
tr2.cow = new(cow)
tr.mu.Unlock()
return &tr2
}