From 86f124e938fcde2155b2fb1a4b4d286c644781a8 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Thu, 16 Jun 2022 13:36:02 -0400 Subject: [PATCH] upgrade buntdb --- go.mod | 8 +- go.sum | 8 + vendor/github.com/tidwall/btree/PATH_HINT.md | 4 +- vendor/github.com/tidwall/btree/README.md | 50 +- vendor/github.com/tidwall/btree/btree.go | 899 ++---------- .../tidwall/btree/internal/btree.go | 1275 +++++++++++++++++ vendor/github.com/tidwall/buntdb/README.md | 1 + vendor/github.com/tidwall/gjson/README.md | 4 +- vendor/github.com/tidwall/gjson/SYNTAX.md | 26 +- vendor/github.com/tidwall/gjson/gjson.go | 341 ++++- vendor/modules.txt | 9 +- 11 files changed, 1760 insertions(+), 865 deletions(-) create mode 100644 vendor/github.com/tidwall/btree/internal/btree.go diff --git a/go.mod b/go.mod index a91e2a4b..d7b258d5 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/onsi/ginkgo v1.12.0 // indirect github.com/onsi/gomega v1.9.0 // indirect github.com/stretchr/testify v1.4.0 // indirect - github.com/tidwall/buntdb v1.2.7 + github.com/tidwall/buntdb v1.2.9 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 github.com/xdg-go/scram v1.0.2 golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8 @@ -28,9 +28,9 @@ require ( require github.com/gofrs/flock v0.8.1 require ( - github.com/tidwall/btree v0.6.1 // indirect - github.com/tidwall/gjson v1.10.2 // indirect - github.com/tidwall/grect v0.1.3 // indirect + github.com/tidwall/btree v1.1.0 // indirect + github.com/tidwall/gjson v1.12.1 // indirect + github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/rtred v0.1.2 // indirect diff --git a/go.sum b/go.sum index 594f4d14..76727599 100644 --- a/go.sum +++ b/go.sum @@ -47,12 +47,20 @@ github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v0.6.1 h1:75VVgBeviiDO+3g4U+7+BaNBNhNINxB0ULPT3fs9pMY= github.com/tidwall/btree v0.6.1/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= +github.com/tidwall/btree v1.1.0 h1:5P+9WU8ui5uhmcg3SoPyTwoI0mVyZ1nps7YQzTZFkYM= +github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= github.com/tidwall/buntdb v1.2.7 h1:SIyObKAymzLyGhDeIhVk2Yc1/EwfCC75Uyu77CHlVoA= github.com/tidwall/buntdb v1.2.7/go.mod h1:b6KvZM27x/8JLI5hgRhRu60pa3q0Tz9c50TyD46OHUM= +github.com/tidwall/buntdb v1.2.9 h1:XVz684P7X6HCTrdr385yDZWB1zt/n20ZNG3M1iGyFm4= +github.com/tidwall/buntdb v1.2.9/go.mod h1:IwyGSvvDg6hnKSIhtdZ0AqhCZGH8ukdtCAzaP8fI1X4= github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo= github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo= +github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.3 h1:z9YwQAMUxVSBde3b7Sl8Da37rffgNfZ6Fq6h9t6KdXE= github.com/tidwall/grect v0.1.3/go.mod h1:8GMjwh3gPZVpLBI/jDz9uslCe0dpxRpWDdtN0lWAS/E= +github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= +github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= diff --git a/vendor/github.com/tidwall/btree/PATH_HINT.md b/vendor/github.com/tidwall/btree/PATH_HINT.md index 0b2c4055..4f243c47 100644 --- a/vendor/github.com/tidwall/btree/PATH_HINT.md +++ b/vendor/github.com/tidwall/btree/PATH_HINT.md @@ -26,7 +26,7 @@ Take the first example image. The item 9 is at path “1/0”. The item 16 is at A Path Hint is a predefined path that is provided to B-tree operations. It’s just a hint that says, “Hey B-tree, instead of starting your binary search with the middle index, start with what I provide you. My path may be wrong, and if so please provide me with the correct path so I get it right the next time.” -I’ve found using path hints can lead to a little performance increase of 150% - 300%. This is because in real-world cases the items that I’m working with are usually nearby each other in the tree. +I’ve found using path hints can lead to a little performance increase of 150% - 300%. This is because in real-world cases the items that I’m working with are usually nearby each other in the tree. Take for example inserting a group of timeseries points. They may often be received as chucks of near-contiguous items. Or, I'm sequentially inserting an ordered group of rows somewhere in the middle of a table. @@ -37,6 +37,8 @@ While I may see a 3x boost in when the path hint is right on, I'll only see arou ## Using a Path Hint +All of the functions that take in a path hint argument mutate the path hint argument. + For single-threaded programs, it’s possible to use one shared path hint per B-tree for the life of the program. For multi-threaded programs, I find it best to use one path hint per B-tree , per thread. For server-client programs, one path hint per B-tree, per client should suffice. diff --git a/vendor/github.com/tidwall/btree/README.md b/vendor/github.com/tidwall/btree/README.md index 3612d113..7d00f5a3 100644 --- a/vendor/github.com/tidwall/btree/README.md +++ b/vendor/github.com/tidwall/btree/README.md @@ -4,6 +4,8 @@ An [efficient](#performance) [B-tree](https://en.wikipedia.org/wiki/B-tree) implementation in Go. +*Check out the [generics branch](https://github.com/tidwall/btree/tree/generics) if you want to try out btree with generic support for Go 1.18+* + ## Features - `Copy()` method with copy-on-write support. @@ -116,10 +118,10 @@ func main() { ### Basic ``` -Len() # return the number of items in the btree -Set(item) # insert or replace an existing item Get(item) # get an existing item +Set(item) # insert or replace an existing item Delete(item) # delete an item +Len() # return the number of items in the btree ``` ### Iteration @@ -127,6 +129,7 @@ Delete(item) # delete an item ``` Ascend(pivot, iter) # scan items in ascending order starting at pivot. Descend(pivot, iter) # scan items in descending order starting at pivot. +Iter() # returns a read-only iterator for for-loops. ``` ### Queues @@ -151,11 +154,18 @@ GetHint(item, *hint) # get an existing item DeleteHint(item, *hint) # delete an item ``` +### Array-like operations + +``` +GetAt(index) # returns the value at index +DeleteAt(index) # deletes the item at index +``` + ## Performance This implementation was designed with performance in mind. -The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel Core i9) using Go 1.16.5. The items are simple 8-byte ints. +The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel Core i9) using Go 1.17.3. The items are simple 8-byte ints. - `google`: The [google/btree](https://github.com/google/btree) package - `tidwall`: The [tidwall/btree](https://github.com/tidwall/btree) package @@ -163,29 +173,29 @@ The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel C ``` ** sequential set ** -google: set-seq 1,000,000 ops in 163ms, 6,140,597/sec, 162 ns/op, 30.9 MB, 32 bytes/op -tidwall: set-seq 1,000,000 ops in 141ms, 7,075,240/sec, 141 ns/op, 36.6 MB, 38 bytes/op -tidwall: set-seq-hint 1,000,000 ops in 79ms, 12,673,902/sec, 78 ns/op, 36.6 MB, 38 bytes/op -tidwall: load-seq 1,000,000 ops in 40ms, 24,887,293/sec, 40 ns/op, 36.6 MB, 38 bytes/op -go-arr: append 1,000,000 ops in 51ms, 19,617,269/sec, 50 ns/op +google: set-seq 1,000,000 ops in 178ms, 5,618,049/sec, 177 ns/op, 39.0 MB, 40 bytes/op +tidwall: set-seq 1,000,000 ops in 156ms, 6,389,837/sec, 156 ns/op, 23.5 MB, 24 bytes/op +tidwall: set-seq-hint 1,000,000 ops in 78ms, 12,895,355/sec, 77 ns/op, 23.5 MB, 24 bytes/op +tidwall: load-seq 1,000,000 ops in 53ms, 18,937,400/sec, 52 ns/op, 23.5 MB, 24 bytes/op +go-arr: append 1,000,000 ops in 78ms, 12,843,432/sec, 77 ns/op ** random set ** -google: set-rand 1,000,000 ops in 666ms, 1,501,583/sec, 665 ns/op, 21.5 MB, 22 bytes/op -tidwall: set-rand 1,000,000 ops in 569ms, 1,756,845/sec, 569 ns/op, 26.7 MB, 27 bytes/op -tidwall: set-rand-hint 1,000,000 ops in 670ms, 1,491,637/sec, 670 ns/op, 26.4 MB, 27 bytes/op -tidwall: set-again 1,000,000 ops in 488ms, 2,050,667/sec, 487 ns/op, 27.1 MB, 28 bytes/op -tidwall: set-after-copy 1,000,000 ops in 494ms, 2,022,980/sec, 494 ns/op, 27.9 MB, 29 bytes/op -tidwall: load-rand 1,000,000 ops in 594ms, 1,682,937/sec, 594 ns/op, 26.1 MB, 27 bytes/op +google: set-rand 1,000,000 ops in 555ms, 1,803,133/sec, 554 ns/op, 29.7 MB, 31 bytes/op +tidwall: set-rand 1,000,000 ops in 545ms, 1,835,818/sec, 544 ns/op, 29.6 MB, 31 bytes/op +tidwall: set-rand-hint 1,000,000 ops in 670ms, 1,493,473/sec, 669 ns/op, 29.6 MB, 31 bytes/op +tidwall: set-again 1,000,000 ops in 681ms, 1,469,038/sec, 680 ns/op +tidwall: set-after-copy 1,000,000 ops in 670ms, 1,493,230/sec, 669 ns/op +tidwall: load-rand 1,000,000 ops in 569ms, 1,756,187/sec, 569 ns/op, 29.6 MB, 31 bytes/op ** sequential get ** -google: get-seq 1,000,000 ops in 141ms, 7,078,690/sec, 141 ns/op -tidwall: get-seq 1,000,000 ops in 124ms, 8,075,925/sec, 123 ns/op -tidwall: get-seq-hint 1,000,000 ops in 40ms, 25,142,979/sec, 39 ns/op +google: get-seq 1,000,000 ops in 165ms, 6,048,307/sec, 165 ns/op +tidwall: get-seq 1,000,000 ops in 144ms, 6,940,120/sec, 144 ns/op +tidwall: get-seq-hint 1,000,000 ops in 78ms, 12,815,243/sec, 78 ns/op ** random get ** -google: get-rand 1,000,000 ops in 152ms, 6,593,518/sec, 151 ns/op -tidwall: get-rand 1,000,000 ops in 128ms, 7,783,293/sec, 128 ns/op -tidwall: get-rand-hint 1,000,000 ops in 135ms, 7,403,823/sec, 135 ns/op +google: get-rand 1,000,000 ops in 701ms, 1,427,507/sec, 700 ns/op +tidwall: get-rand 1,000,000 ops in 679ms, 1,473,531/sec, 678 ns/op +tidwall: get-rand-hint 1,000,000 ops in 824ms, 1,213,805/sec, 823 ns/op ``` *You can find the benchmark utility at [tidwall/btree-benchmark](https://github.com/tidwall/btree-benchmark)* diff --git a/vendor/github.com/tidwall/btree/btree.go b/vendor/github.com/tidwall/btree/btree.go index 21afc5b3..c8cdac29 100644 --- a/vendor/github.com/tidwall/btree/btree.go +++ b/vendor/github.com/tidwall/btree/btree.go @@ -1,58 +1,28 @@ // Copyright 2020 Joshua J Baker. All rights reserved. // Use of this source code is governed by an MIT-style // license that can be found in the LICENSE file. - package btree -import ( - "sync" -) +import btree "github.com/tidwall/btree/internal" -const maxItems = 255 // max items per node. max children is +1 -const minItems = maxItems * 40 / 100 - -type cow struct { - _ int // it cannot be an empty struct -} - -type node struct { - cow *cow - leaf bool - numItems int16 - count int - items [maxItems]interface{} - children *[maxItems + 1]*node -} - -// BTree is an ordered set items type BTree struct { - mu *sync.RWMutex - cow *cow - root *node - count int - less func(a, b interface{}) bool - locks bool -} - -func (tr *BTree) newNode(leaf bool) *node { - n := &node{leaf: leaf} - if !leaf { - n.children = new([maxItems + 1]*node) - } - n.cow = tr.cow - return n + base *btree.BTree } // PathHint is a utility type used with the *Hint() functions. Hints provide // faster operations for clustered keys. -type PathHint struct { - used [8]bool - path [8]uint8 -} +type PathHint = btree.PathHint // New returns a new BTree func New(less func(a, b interface{}) bool) *BTree { - return newBTree(less, true) + if less == nil { + panic("nil less") + } + return &BTree{ + base: btree.NewOptions(btree.Options{ + Context: less, + }), + } } // NewNonConcurrent returns a new BTree which is not safe for concurrent @@ -61,86 +31,26 @@ func New(less func(a, b interface{}) bool) *BTree { // This is useful for when you do not need the BTree to manage the locking, // but would rather do it yourself. func NewNonConcurrent(less func(a, b interface{}) bool) *BTree { - return newBTree(less, false) -} - -func newBTree(less func(a, b interface{}) bool, locks bool) *BTree { if less == nil { panic("nil less") } - tr := new(BTree) - tr.mu = new(sync.RWMutex) - tr.less = less - tr.locks = locks - return tr + return &BTree{ + base: btree.NewOptions(btree.Options{ + Context: less, + NoLocks: true, + }), + } } // Less is a convenience function that performs a comparison of two items // using the same "less" function provided to New. func (tr *BTree) Less(a, b interface{}) bool { - return tr.less(a, b) + return tr.base.Less(a, b) } -func (n *node) find(key interface{}, less func(a, b interface{}) bool, - hint *PathHint, depth int, -) (index int16, found bool) { - low := int16(0) - high := n.numItems - 1 - if hint != nil && depth < 8 && hint.used[depth] { - index = int16(hint.path[depth]) - if index >= n.numItems { - // tail item - if less(n.items[n.numItems-1], key) { - if less(key, n.items[n.numItems-1]) { - index = n.numItems - 1 - found = true - goto path_match - } else { - index = n.numItems - goto path_match - } - } - index = n.numItems - 1 - } - if less(key, n.items[index]) { - if index == 0 || less(n.items[index-1], key) { - goto path_match - } - high = index - 1 - } else if less(n.items[index], key) { - low = index + 1 - } else { - found = true - goto path_match - } - } - for low <= high { - mid := low + ((high+1)-low)/2 - if !less(key, n.items[mid]) { - low = mid + 1 - } else { - high = mid - 1 - } - } - if low > 0 && !less(n.items[low-1], key) { - index = low - 1 - found = true - } else { - index = low - found = false - } - if hint == nil || depth >= 8 { - return index, found - } - -path_match: - hint.used[depth] = true - if n.leaf && found { - hint.path[depth] = byte(index + 1) - } else { - hint.path[depth] = byte(index) - } - return index, found +// Set or replace a value for a key +func (tr *BTree) Set(item interface{}) interface{} { + return tr.SetHint(item, nil) } // SetHint sets or replace a value for a key using a path hint @@ -148,214 +58,33 @@ func (tr *BTree) SetHint(item interface{}, hint *PathHint) (prev interface{}) { if item == nil { panic("nil item") } - if tr.lock() { - defer tr.unlock() - } - return tr.setHint(item, hint) -} - -func (tr *BTree) setHint(item interface{}, hint *PathHint) (prev interface{}) { - if tr.root == nil { - tr.root = tr.newNode(true) - tr.root.items[0] = item - tr.root.numItems = 1 - tr.root.count = 1 - tr.count = 1 - return - } - prev = tr.nodeSet(&tr.root, item, tr.less, hint, 0) - if prev != nil { - return prev - } - if tr.root.numItems == maxItems { - n := tr.cowLoad(&tr.root) - right, median := tr.nodeSplit(n) - tr.root = tr.newNode(false) - tr.root.children[0] = n - tr.root.items[0] = median - tr.root.children[1] = right - tr.root.numItems = 1 - tr.root.count = n.count + 1 + right.count - } - tr.count++ - return prev -} - -// Set or replace a value for a key -func (tr *BTree) Set(item interface{}) (prev interface{}) { - return tr.SetHint(item, nil) -} - -func (tr *BTree) nodeSplit(n *node) (right *node, median interface{}) { - right = tr.newNode(n.leaf) - median = n.items[maxItems/2] - copy(right.items[:maxItems/2], n.items[maxItems/2+1:]) - if !n.leaf { - copy(right.children[:maxItems/2+1], n.children[maxItems/2+1:]) - } - right.numItems = maxItems / 2 - if !n.leaf { - for i := maxItems/2 + 1; i < maxItems+1; i++ { - n.children[i] = nil - } - } - for i := maxItems / 2; i < maxItems; i++ { - n.items[i] = nil - } - n.numItems = maxItems / 2 - // update counts - n.updateCount() - right.updateCount() - return right, median -} - -func (n *node) updateCount() { - n.count = int(n.numItems) - if !n.leaf { - for i := 0; i <= int(n.numItems); i++ { - n.count += n.children[i].count - } - } -} - -// This operation should not be inlined because it's expensive and rarely -// called outside of heavy copy-on-write situations. Marking it "noinline" -// allows for the parent cowLoad to be inlined. -// go:noinline -func (tr *BTree) copy(n *node) *node { - n2 := *n - n2.cow = tr.cow - copy(n2.items[:], n.items[:]) - if n.children != nil { - n2.children = new([maxItems + 1]*node) - copy(n2.children[:], n.children[:]) - } - return &n2 -} - -// cowLoad loads the provide node and, if needed, performs a copy-on-write. -func (tr *BTree) cowLoad(cn **node) *node { - if (*cn).cow != tr.cow { - *cn = tr.copy(*cn) - } - return *cn -} - -func (tr *BTree) nodeSet(cn **node, item interface{}, - less func(a, b interface{}) bool, hint *PathHint, depth int, -) (prev interface{}) { - n := tr.cowLoad(cn) - i, found := n.find(item, less, hint, depth) - if found { - prev = n.items[i] - n.items[i] = item - return prev - } - if n.leaf { - copy(n.items[i+1:n.numItems+1], n.items[i:n.numItems]) - n.items[i] = item - n.numItems++ - n.count++ + v, ok := tr.base.SetHint(item, hint) + if !ok { return nil } - prev = tr.nodeSet(&n.children[i], item, less, hint, depth+1) - if prev != nil { - return prev - } - if n.children[i].numItems == maxItems { - right, median := tr.nodeSplit(n.children[i]) - copy(n.children[i+1:], n.children[i:]) - copy(n.items[i+1:], n.items[i:]) - n.items[i] = median - n.children[i+1] = right - n.numItems++ - } - n.count++ - return nil -} - -func (n *node) scan(iter func(item interface{}) bool) bool { - if n.leaf { - for i := int16(0); i < n.numItems; i++ { - if !iter(n.items[i]) { - return false - } - } - return true - } - for i := int16(0); i < n.numItems; i++ { - if !n.children[i].scan(iter) { - return false - } - if !iter(n.items[i]) { - return false - } - } - return n.children[n.numItems].scan(iter) + return v } // Get a value for key func (tr *BTree) Get(key interface{}) interface{} { - // This operation is basically the same as calling: - // return tr.GetHint(key, nil) - // But here we inline the bsearch to avoid the hint logic and extra - // function call. - if tr.rlock() { - defer tr.runlock() - } - if tr.root == nil || key == nil { - return nil - } - depth := 0 - n := tr.root - for { - low := int16(0) - high := n.numItems - 1 - for low <= high { - mid := low + ((high+1)-low)/2 - if !tr.less(key, n.items[mid]) { - low = mid + 1 - } else { - high = mid - 1 - } - } - if low > 0 && !tr.less(n.items[low-1], key) { - return n.items[low-1] - } - if n.leaf { - return nil - } - n = n.children[low] - depth++ - } + return tr.GetHint(key, nil) } // GetHint gets a value for key using a path hint func (tr *BTree) GetHint(key interface{}, hint *PathHint) interface{} { - if tr.rlock() { - defer tr.runlock() - } - if tr.root == nil || key == nil { + if key == nil { return nil } - depth := 0 - n := tr.root - for { - index, found := n.find(key, tr.less, hint, depth) - if found { - return n.items[index] - } - if n.leaf { - return nil - } - n = n.children[index] - depth++ + v, ok := tr.base.GetHint(key, hint) + if !ok { + return nil } + return v } // Len returns the number of items in the tree func (tr *BTree) Len() int { - return tr.count + return tr.base.Len() } // Delete a value for a key @@ -365,574 +94,180 @@ func (tr *BTree) Delete(key interface{}) interface{} { // DeleteHint deletes a value for a key using a path hint func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} { - if tr.lock() { - defer tr.unlock() - } - return tr.deleteHint(key, hint) -} - -func (tr *BTree) deleteHint(key interface{}, hint *PathHint) interface{} { - if tr.root == nil || key == nil { + if key == nil { return nil } - prev := tr.delete(&tr.root, false, key, tr.less, hint, 0) - if prev == nil { + v, ok := tr.base.DeleteHint(key, nil) + if !ok { return nil } - if tr.root.numItems == 0 && !tr.root.leaf { - tr.root = tr.root.children[0] - } - tr.count-- - if tr.count == 0 { - tr.root = nil - } - return prev -} - -func (tr *BTree) delete(cn **node, max bool, key interface{}, - less func(a, b interface{}) bool, hint *PathHint, depth int, -) interface{} { - n := tr.cowLoad(cn) - var i int16 - var found bool - if max { - i, found = n.numItems-1, true - } else { - i, found = n.find(key, less, hint, depth) - } - if n.leaf { - if found { - prev := n.items[i] - // found the items at the leaf, remove it and return. - copy(n.items[i:], n.items[i+1:n.numItems]) - n.items[n.numItems-1] = nil - n.numItems-- - n.count-- - return prev - } - return nil - } - - var prev interface{} - if found { - if max { - i++ - prev = tr.delete(&n.children[i], true, "", less, nil, 0) - } else { - prev = n.items[i] - maxItem := tr.delete(&n.children[i], true, "", less, nil, 0) - n.items[i] = maxItem - } - } else { - prev = tr.delete(&n.children[i], max, key, less, hint, depth+1) - } - if prev == nil { - return nil - } - n.count-- - if n.children[i].numItems >= minItems { - return prev - } - - // merge / rebalance nodes - if i == n.numItems { - i-- - } - n.children[i] = tr.cowLoad(&n.children[i]) - n.children[i+1] = tr.cowLoad(&n.children[i+1]) - if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems { - // merge left + item + right - n.children[i].items[n.children[i].numItems] = n.items[i] - copy(n.children[i].items[n.children[i].numItems+1:], - n.children[i+1].items[:n.children[i+1].numItems]) - if !n.children[0].leaf { - copy(n.children[i].children[n.children[i].numItems+1:], - n.children[i+1].children[:n.children[i+1].numItems+1]) - } - n.children[i].numItems += n.children[i+1].numItems + 1 - n.children[i].count += n.children[i+1].count + 1 - copy(n.items[i:], n.items[i+1:n.numItems]) - copy(n.children[i+1:], n.children[i+2:n.numItems+1]) - n.items[n.numItems-1] = nil - n.children[n.numItems] = nil - n.numItems-- - } else if n.children[i].numItems > n.children[i+1].numItems { - // move left -> right - copy(n.children[i+1].items[1:], - n.children[i+1].items[:n.children[i+1].numItems]) - if !n.children[0].leaf { - copy(n.children[i+1].children[1:], - n.children[i+1].children[:n.children[i+1].numItems+1]) - } - n.children[i+1].items[0] = n.items[i] - if !n.children[0].leaf { - n.children[i+1].children[0] = - n.children[i].children[n.children[i].numItems] - n.children[i+1].count += n.children[i+1].children[0].count - } - n.children[i+1].numItems++ - n.children[i+1].count++ - n.items[i] = n.children[i].items[n.children[i].numItems-1] - n.children[i].items[n.children[i].numItems-1] = nil - if !n.children[0].leaf { - n.children[i].children[n.children[i].numItems] = nil - n.children[i].count -= n.children[i+1].children[0].count - } - n.children[i].numItems-- - n.children[i].count-- - } else { - // move left <- right - n.children[i].items[n.children[i].numItems] = n.items[i] - if !n.children[0].leaf { - n.children[i].children[n.children[i].numItems+1] = - n.children[i+1].children[0] - n.children[i].count += - n.children[i].children[n.children[i].numItems+1].count - } - n.children[i].numItems++ - n.children[i].count++ - n.items[i] = n.children[i+1].items[0] - copy(n.children[i+1].items[:], - n.children[i+1].items[1:n.children[i+1].numItems]) - n.children[i+1].items[n.children[i+1].numItems-1] = nil - if !n.children[0].leaf { - copy(n.children[i+1].children[:], - n.children[i+1].children[1:n.children[i+1].numItems+1]) - n.children[i+1].children[n.children[i+1].numItems] = nil - n.children[i+1].count -= - n.children[i].children[n.children[i].numItems].count - } - n.children[i+1].numItems-- - n.children[i+1].count-- - } - return prev + return v } // Ascend the tree within the range [pivot, last] // Pass nil for pivot to scan all item in ascending order // Return false to stop iterating func (tr *BTree) Ascend(pivot interface{}, iter func(item interface{}) bool) { - if tr.rlock() { - defer tr.runlock() - } - if tr.root == nil { - return - } if pivot == nil { - tr.root.scan(iter) - } else if tr.root != nil { - tr.root.ascend(pivot, tr.less, nil, 0, iter) + tr.base.Scan(iter) + } else { + tr.base.Ascend(pivot, iter) } } -func (n *node) ascend(pivot interface{}, less func(a, b interface{}) bool, - hint *PathHint, depth int, iter func(item interface{}) bool, -) bool { - i, found := n.find(pivot, less, hint, depth) - if !found { - if !n.leaf { - if !n.children[i].ascend(pivot, less, hint, depth+1, iter) { - return false - } - } - } - for ; i < n.numItems; i++ { - if !iter(n.items[i]) { - return false - } - if !n.leaf { - if !n.children[i+1].scan(iter) { - return false - } - } - } - return true -} - -func (n *node) reverse(iter func(item interface{}) bool) bool { - if n.leaf { - for i := n.numItems - 1; i >= 0; i-- { - if !iter(n.items[i]) { - return false - } - } - return true - } - if !n.children[n.numItems].reverse(iter) { - return false - } - for i := n.numItems - 1; i >= 0; i-- { - if !iter(n.items[i]) { - return false - } - if !n.children[i].reverse(iter) { - return false - } - } - return true -} - // Descend the tree within the range [pivot, first] // Pass nil for pivot to scan all item in descending order // Return false to stop iterating func (tr *BTree) Descend(pivot interface{}, iter func(item interface{}) bool) { - if tr.rlock() { - defer tr.runlock() - } - if tr.root == nil { - return - } if pivot == nil { - tr.root.reverse(iter) - } else if tr.root != nil { - tr.root.descend(pivot, tr.less, nil, 0, iter) + tr.base.Reverse(iter) + } else { + tr.base.Descend(pivot, iter) } } -func (n *node) descend(pivot interface{}, less func(a, b interface{}) bool, - hint *PathHint, depth int, iter func(item interface{}) bool, -) bool { - i, found := n.find(pivot, less, hint, depth) - if !found { - if !n.leaf { - if !n.children[i].descend(pivot, less, hint, depth+1, iter) { - return false - } - } - i-- - } - for ; i >= 0; i-- { - if !iter(n.items[i]) { - return false - } - if !n.leaf { - if !n.children[i].reverse(iter) { - return false - } - } - } - return true -} - // Load is for bulk loading pre-sorted items func (tr *BTree) Load(item interface{}) interface{} { if item == nil { panic("nil item") } - if tr.lock() { - defer tr.unlock() + v, ok := tr.base.Load(item) + if !ok { + return nil } - if tr.root == nil { - return tr.setHint(item, nil) - } - n := tr.cowLoad(&tr.root) - for { - n.count++ // optimistically update counts - if n.leaf { - if n.numItems < maxItems-2 { - if tr.less(n.items[n.numItems-1], item) { - n.items[n.numItems] = item - n.numItems++ - tr.count++ - return nil - } - } - break - } - n = tr.cowLoad(&n.children[n.numItems]) - } - // revert the counts - n = tr.root - for { - n.count-- - if n.leaf { - break - } - n = n.children[n.numItems] - } - return tr.setHint(item, nil) + return v } // Min returns the minimum item in tree. // Returns nil if the tree has no items. func (tr *BTree) Min() interface{} { - if tr.rlock() { - defer tr.runlock() - } - if tr.root == nil { + v, ok := tr.base.Min() + if !ok { return nil } - n := tr.root - for { - if n.leaf { - return n.items[0] - } - n = n.children[0] - } + return v } // Max returns the maximum item in tree. // Returns nil if the tree has no items. func (tr *BTree) Max() interface{} { - if tr.rlock() { - defer tr.runlock() - } - if tr.root == nil { + v, ok := tr.base.Max() + if !ok { return nil } - n := tr.root - for { - if n.leaf { - return n.items[n.numItems-1] - } - n = n.children[n.numItems] - } + return v } // PopMin removes the minimum item in tree and returns it. // Returns nil if the tree has no items. func (tr *BTree) PopMin() interface{} { - if tr.lock() { - defer tr.unlock() - } - if tr.root == nil { + v, ok := tr.base.PopMin() + if !ok { return nil } - n := tr.cowLoad(&tr.root) - var item interface{} - for { - n.count-- // optimistically update counts - if n.leaf { - item = n.items[0] - if n.numItems == minItems { - break - } - copy(n.items[:], n.items[1:]) - n.items[n.numItems-1] = nil - n.numItems-- - tr.count-- - if tr.count == 0 { - tr.root = nil - } - return item - } - n = tr.cowLoad(&n.children[0]) - } - // revert the counts - n = tr.root - for { - n.count++ - if n.leaf { - break - } - n = n.children[0] - } - return tr.deleteHint(item, nil) + return v } // PopMax removes the minimum item in tree and returns it. // Returns nil if the tree has no items. func (tr *BTree) PopMax() interface{} { - if tr.lock() { - defer tr.unlock() - } - if tr.root == nil { + v, ok := tr.base.PopMax() + if !ok { return nil } - n := tr.cowLoad(&tr.root) - var item interface{} - for { - n.count-- // optimistically update counts - if n.leaf { - item = n.items[n.numItems-1] - if n.numItems == minItems { - break - } - n.items[n.numItems-1] = nil - n.numItems-- - tr.count-- - if tr.count == 0 { - tr.root = nil - } - return item - } - n = tr.cowLoad(&n.children[n.numItems]) - } - // revert the counts - n = tr.root - for { - n.count++ - if n.leaf { - break - } - n = n.children[n.numItems] - } - return tr.deleteHint(item, nil) + return v } // GetAt returns the value at index. // Return nil if the tree is empty or the index is out of bounds. func (tr *BTree) GetAt(index int) interface{} { - if tr.rlock() { - defer tr.runlock() - } - if tr.root == nil || index < 0 || index >= tr.count { + v, ok := tr.base.GetAt(index) + if !ok { return nil } - n := tr.root - for { - if n.leaf { - return n.items[index] - } - i := 0 - for ; i < int(n.numItems); i++ { - if index < n.children[i].count { - break - } else if index == n.children[i].count { - return n.items[i] - } - index -= n.children[i].count + 1 - } - n = n.children[i] - } + return v } // DeleteAt deletes the item at index. // Return nil if the tree is empty or the index is out of bounds. func (tr *BTree) DeleteAt(index int) interface{} { - if tr.lock() { - defer tr.unlock() - } - if tr.root == nil || index < 0 || index >= tr.count { + v, ok := tr.base.DeleteAt(index) + if !ok { return nil } - var pathbuf [8]uint8 // track the path - path := pathbuf[:0] - var item interface{} - n := tr.cowLoad(&tr.root) -outer: - for { - n.count-- // optimistically update counts - if n.leaf { - // the index is the item position - item = n.items[index] - if n.numItems == minItems { - path = append(path, uint8(index)) - break outer - } - copy(n.items[index:], n.items[index+1:n.numItems]) - n.items[n.numItems-1] = nil - n.numItems-- - tr.count-- - if tr.count == 0 { - tr.root = nil - } - return item - } - i := 0 - for ; i < int(n.numItems); i++ { - if index < n.children[i].count { - break - } else if index == n.children[i].count { - item = n.items[i] - path = append(path, uint8(i)) - break outer - } - index -= n.children[i].count + 1 - } - path = append(path, uint8(i)) - n = tr.cowLoad(&n.children[i]) - } - // revert the counts - var hint PathHint - n = tr.root - for i := 0; i < len(path); i++ { - if i < len(hint.path) { - hint.path[i] = path[i] - hint.used[i] = true - } - n.count++ - if !n.leaf { - n = n.children[uint8(path[i])] - } - } - return tr.deleteHint(item, &hint) + return v } // Height returns the height of the tree. // Returns zero if tree has no items. func (tr *BTree) Height() int { - if tr.rlock() { - defer tr.runlock() - } - var height int - if tr.root != nil { - n := tr.root - for { - height++ - if n.leaf { - break - } - n = n.children[n.numItems] - } - } - return height + return tr.base.Height() } // Walk iterates over all items in tree, in order. // The items param will contain one or more items. -func (tr *BTree) Walk(iter func(item []interface{})) { - if tr.rlock() { - defer tr.runlock() - } - if tr.root != nil { - tr.root.walk(iter) - } +func (tr *BTree) Walk(iter func(items []interface{})) { + tr.base.Walk(func(items []interface{}) bool { + iter(items) + return true + }) } -func (n *node) walk(iter func(item []interface{})) { - if n.leaf { - iter(n.items[:n.numItems]) - } else { - for i := int16(0); i < n.numItems; i++ { - n.children[i].walk(iter) - iter(n.items[i : i+1]) - } - n.children[n.numItems].walk(iter) - } -} - -// Copy the tree. This operation is very fast because it only performs a -// shadowed copy. +// Copy the tree. This is a copy-on-write operation and is very fast because +// it only performs a shadowed copy. func (tr *BTree) Copy() *BTree { - if tr.lock() { - defer tr.unlock() - } - tr.cow = new(cow) - tr2 := *tr - tr2.mu = new(sync.RWMutex) - tr2.cow = new(cow) - return &tr2 + return &BTree{base: tr.base.Copy()} } -func (tr *BTree) lock() bool { - if tr.locks { - tr.mu.Lock() - } - return tr.locks +type Iter struct { + base btree.Iter } -func (tr *BTree) unlock() { - tr.mu.Unlock() +// Iter returns a read-only iterator. +// The Release method must be called finished with iterator. +func (tr *BTree) Iter() Iter { + return Iter{tr.base.Iter()} } -func (tr *BTree) rlock() bool { - if tr.locks { - tr.mu.RLock() - } - return tr.locks +// Seek to item greater-or-equal-to key. +// Returns false if there was no item found. +func (iter *Iter) Seek(key interface{}) bool { + return iter.base.Seek(key) } -func (tr *BTree) runlock() { - tr.mu.RUnlock() +// First moves iterator to first item in tree. +// Returns false if the tree is empty. +func (iter *Iter) First() bool { + return iter.base.First() +} + +// Last moves iterator to last item in tree. +// Returns false if the tree is empty. +func (iter *Iter) Last() bool { + return iter.base.Last() +} + +// First moves iterator to first item in tree. +// Returns false if the tree is empty. +func (iter *Iter) Release() { + iter.base.Release() +} + +// Next moves iterator to the next item in iterator. +// Returns false if the tree is empty or the iterator is at the end of +// the tree. +func (iter *Iter) Next() bool { + return iter.base.Next() +} + +// Prev moves iterator to the previous item in iterator. +// Returns false if the tree is empty or the iterator is at the beginning of +// the tree. +func (iter *Iter) Prev() bool { + return iter.base.Prev() +} + +// Item returns the current iterator item. +func (iter *Iter) Item() interface{} { + return iter.base.Item() } diff --git a/vendor/github.com/tidwall/btree/internal/btree.go b/vendor/github.com/tidwall/btree/internal/btree.go new file mode 100644 index 00000000..bde59ec5 --- /dev/null +++ b/vendor/github.com/tidwall/btree/internal/btree.go @@ -0,0 +1,1275 @@ +// Copyright 2020 Joshua J Baker. All rights reserved. +// Use of this source code is governed by an MIT-style license that can be +// found in the LICENSE file at https://github.com/tidwall/btree/LICENSE + +/////////////////////////////////////////////////////////////////////////////// +// BEGIN PARAMS +/////////////////////////////////////////////////////////////////////////////// + +package btree + +import "sync" + +// degree is the B-Tree degree, which is equal to maximum number of children +// pre node times two. +// The default is 128, which means each node can have 255 items and 256 child +// nodes. +const degree = 128 + +// kind is the item type. +// It's important to use the equal symbol, which tells Go to create an alias of +// the type, rather than creating an entirely new type. +type kind = interface{} + +// contextKind is the kind of context that can be passed to NewOptions and the +// less function +type contextKind = interface{} + +// less returns true if A is less than B. +// The value of context will be whatever was passed to NewOptions through the +// Options.Context field, otherwise nil if the field was not set. +func less(a, b kind, context contextKind) bool { + return context.(func(a, b contextKind) bool)(a, b) +} + +// BTree aliases +// These are aliases to the local bTree types and functions, which are exported +// to allow for public use at a package level. +// Rename them if desired, or comment them out to make the library private. +type BTree = bTree +type Options = bOptions +type PathHint = bPathHint +type Iter = bIter + +func New(less func(a, b kind) bool) *bTree { return bNew() } +func NewOptions(opts bOptions) *bTree { return bNewOptions(opts) } + +// The functions below, which begin with "test*", are required by the +// btree_test.go file. If you choose not use include the btree_test.go file in +// your project then these functions may be omitted. + +// testCustomSeed can be used to generate a custom random seed for testing. +// Returning false will use time.Now().UnixNano() +func testCustomSeed() (seed int64, ok bool) { + return 0, false +} + +// testMakeItem must return a valid item for testing. +// It's required that the returned item maintains equal order as the +// provided int, such that: +// testMakeItem(0) < testMakeItem(1) < testMakeItem(2) < testMakeItem(10) +func testMakeItem(x int) (item kind) { + return x +} + +// testNewBTree must return an operational btree for testing. +func testNewBTree() *bTree { + return bNewOptions(bOptions{ + Context: func(a, b contextKind) bool { + if a == nil { + return b != nil + } else if b == nil { + return false + } + return a.(int) < b.(int) + }, + }) +} + +/////////////////////////////////////////////////////////////////////////////// +// END PARAMS +/////////////////////////////////////////////////////////////////////////////// + +// Do not edit code below this line. + +const maxItems = degree*2 - 1 // max items per node. max children is +1 +const minItems = maxItems / 2 + +type bTree struct { + mu *sync.RWMutex + cow *cow + root *node + count int + ctx contextKind + locks bool + empty kind +} + +type node struct { + cow *cow + count int + items []kind + children *[]*node +} + +type cow struct { + _ int // cannot be an empty struct +} + +func (tr *bTree) newNode(leaf bool) *node { + n := &node{cow: tr.cow} + if !leaf { + n.children = new([]*node) + } + return n +} + +// leaf returns true if the node is a leaf. +func (n *node) leaf() bool { + return n.children == nil +} + +// PathHint is a utility type used with the *Hint() functions. Hints provide +// faster operations for clustered keys. +type bPathHint struct { + used [8]bool + path [8]uint8 +} + +type bOptions struct { + NoLocks bool + Context contextKind +} + +// New returns a new BTree +func bNew() *bTree { + return bNewOptions(bOptions{}) +} + +func bNewOptions(opts bOptions) *bTree { + tr := new(bTree) + tr.cow = new(cow) + tr.mu = new(sync.RWMutex) + tr.ctx = opts.Context + tr.locks = !opts.NoLocks + return tr +} + +// Less is a convenience function that performs a comparison of two items +// using the same "less" function provided to New. +func (tr *bTree) Less(a, b kind) bool { + return less(a, b, tr.ctx) +} + +func (tr *bTree) find(n *node, key kind, + hint *bPathHint, depth int, +) (index int, found bool) { + if hint == nil { + // fast path for no hinting + low := 0 + high := len(n.items) + for low < high { + mid := (low + high) / 2 + if !tr.Less(key, n.items[mid]) { + low = mid + 1 + } else { + high = mid + } + } + if low > 0 && !tr.Less(n.items[low-1], key) { + return low - 1, true + } + return low, false + } + + // Try using hint. + // Best case finds the exact match, updates the hint and returns. + // Worst case, updates the low and high bounds to binary search between. + low := 0 + high := len(n.items) - 1 + if depth < 8 && hint.used[depth] { + index = int(hint.path[depth]) + if index >= len(n.items) { + // tail item + if tr.Less(n.items[len(n.items)-1], key) { + index = len(n.items) + goto path_match + } + index = len(n.items) - 1 + } + if tr.Less(key, n.items[index]) { + if index == 0 || tr.Less(n.items[index-1], key) { + goto path_match + } + high = index - 1 + } else if tr.Less(n.items[index], key) { + low = index + 1 + } else { + found = true + goto path_match + } + } + + // Do a binary search between low and high + // keep on going until low > high, where the guarantee on low is that + // key >= items[low - 1] + for low <= high { + mid := low + ((high+1)-low)/2 + // if key >= n.items[mid], low = mid + 1 + // which implies that key >= everything below low + if !tr.Less(key, n.items[mid]) { + low = mid + 1 + } else { + high = mid - 1 + } + } + + // if low > 0, n.items[low - 1] >= key, + // we have from before that key >= n.items[low - 1] + // therefore key = n.items[low - 1], + // and we have found the entry for key. + // Otherwise we must keep searching for the key in index `low`. + if low > 0 && !tr.Less(n.items[low-1], key) { + index = low - 1 + found = true + } else { + index = low + found = false + } + +path_match: + if depth < 8 { + hint.used[depth] = true + var pathIndex uint8 + if n.leaf() && found { + pathIndex = uint8(index + 1) + } else { + pathIndex = uint8(index) + } + if pathIndex != hint.path[depth] { + hint.path[depth] = pathIndex + for i := depth + 1; i < 8; i++ { + hint.used[i] = false + } + } + } + return index, found +} + +// SetHint sets or replace a value for a key using a path hint +func (tr *bTree) SetHint(item kind, hint *bPathHint) (prev kind, replaced bool) { + if tr.lock() { + defer tr.unlock() + } + return tr.setHint(item, hint) +} + +func (tr *bTree) setHint(item kind, hint *bPathHint) (prev kind, replaced bool) { + if tr.root == nil { + tr.root = tr.newNode(true) + tr.root.items = append([]kind{}, item) + tr.root.count = 1 + tr.count = 1 + return tr.empty, false + } + prev, replaced, split := tr.nodeSet(&tr.root, item, hint, 0) + if split { + left := tr.cowLoad(&tr.root) + right, median := tr.nodeSplit(left) + tr.root = tr.newNode(false) + *tr.root.children = make([]*node, 0, maxItems+1) + *tr.root.children = append([]*node{}, left, right) + tr.root.items = append([]kind{}, median) + tr.root.updateCount() + return tr.setHint(item, hint) + } + if replaced { + return prev, true + } + tr.count++ + return tr.empty, false +} + +// Set or replace a value for a key +func (tr *bTree) Set(item kind) (kind, bool) { + return tr.SetHint(item, nil) +} + +func (tr *bTree) nodeSplit(n *node) (right *node, median kind) { + i := maxItems / 2 + median = n.items[i] + + // left node + left := tr.newNode(n.leaf()) + left.items = make([]kind, len(n.items[:i]), maxItems/2) + copy(left.items, n.items[:i]) + if !n.leaf() { + *left.children = make([]*node, len((*n.children)[:i+1]), maxItems+1) + copy(*left.children, (*n.children)[:i+1]) + } + left.updateCount() + + // right node + right = tr.newNode(n.leaf()) + right.items = make([]kind, len(n.items[i+1:]), maxItems/2) + copy(right.items, n.items[i+1:]) + if !n.leaf() { + *right.children = make([]*node, len((*n.children)[i+1:]), maxItems+1) + copy(*right.children, (*n.children)[i+1:]) + } + right.updateCount() + + *n = *left + return right, median +} + +func (n *node) updateCount() { + n.count = len(n.items) + if !n.leaf() { + for i := 0; i < len(*n.children); i++ { + n.count += (*n.children)[i].count + } + } +} + +// This operation should not be inlined because it's expensive and rarely +// called outside of heavy copy-on-write situations. Marking it "noinline" +// allows for the parent cowLoad to be inlined. +// go:noinline +func (tr *bTree) copy(n *node) *node { + n2 := new(node) + n2.cow = tr.cow + n2.count = n.count + n2.items = make([]kind, len(n.items), cap(n.items)) + copy(n2.items, n.items) + if !n.leaf() { + n2.children = new([]*node) + *n2.children = make([]*node, len(*n.children), maxItems+1) + copy(*n2.children, *n.children) + } + return n2 +} + +// cowLoad loads the provided node and, if needed, performs a copy-on-write. +func (tr *bTree) cowLoad(cn **node) *node { + if (*cn).cow != tr.cow { + *cn = tr.copy(*cn) + } + return *cn +} + +func (tr *bTree) nodeSet(cn **node, item kind, + hint *bPathHint, depth int, +) (prev kind, replaced bool, split bool) { + n := tr.cowLoad(cn) + i, found := tr.find(n, item, hint, depth) + if found { + prev = n.items[i] + n.items[i] = item + return prev, true, false + } + if n.leaf() { + if len(n.items) == maxItems { + return tr.empty, false, true + } + n.items = append(n.items, tr.empty) + copy(n.items[i+1:], n.items[i:]) + n.items[i] = item + n.count++ + return tr.empty, false, false + } + prev, replaced, split = tr.nodeSet(&(*n.children)[i], item, hint, depth+1) + if split { + if len(n.items) == maxItems { + return tr.empty, false, true + } + right, median := tr.nodeSplit((*n.children)[i]) + *n.children = append(*n.children, nil) + copy((*n.children)[i+1:], (*n.children)[i:]) + (*n.children)[i+1] = right + n.items = append(n.items, tr.empty) + copy(n.items[i+1:], n.items[i:]) + n.items[i] = median + return tr.nodeSet(&n, item, hint, depth) + } + if !replaced { + n.count++ + } + return prev, replaced, false +} + +func (tr *bTree) Scan(iter func(item kind) bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil { + return + } + tr.root.scan(iter) +} + +func (n *node) scan(iter func(item kind) bool) bool { + if n.leaf() { + for i := 0; i < len(n.items); i++ { + if !iter(n.items[i]) { + return false + } + } + return true + } + for i := 0; i < len(n.items); i++ { + if !(*n.children)[i].scan(iter) { + return false + } + if !iter(n.items[i]) { + return false + } + } + return (*n.children)[len(*n.children)-1].scan(iter) +} + +// Get a value for key +func (tr *bTree) Get(key kind) (kind, bool) { + return tr.GetHint(key, nil) +} + +// GetHint gets a value for key using a path hint +func (tr *bTree) GetHint(key kind, hint *bPathHint) (kind, bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil { + return tr.empty, false + } + n := tr.root + depth := 0 + for { + i, found := tr.find(n, key, hint, depth) + if found { + return n.items[i], true + } + if n.children == nil { + return tr.empty, false + } + n = (*n.children)[i] + depth++ + } +} + +// Len returns the number of items in the tree +func (tr *bTree) Len() int { + return tr.count +} + +// Delete a value for a key +func (tr *bTree) Delete(key kind) (kind, bool) { + return tr.DeleteHint(key, nil) +} + +// DeleteHint deletes a value for a key using a path hint +func (tr *bTree) DeleteHint(key kind, hint *bPathHint) (kind, bool) { + if tr.lock() { + defer tr.unlock() + } + return tr.deleteHint(key, hint) +} + +func (tr *bTree) deleteHint(key kind, hint *bPathHint) (kind, bool) { + if tr.root == nil { + return tr.empty, false + } + prev, deleted := tr.delete(&tr.root, false, key, hint, 0) + if !deleted { + return tr.empty, false + } + if len(tr.root.items) == 0 && !tr.root.leaf() { + tr.root = (*tr.root.children)[0] + } + tr.count-- + if tr.count == 0 { + tr.root = nil + } + return prev, true +} + +func (tr *bTree) delete(cn **node, max bool, key kind, + hint *bPathHint, depth int, +) (kind, bool) { + n := tr.cowLoad(cn) + var i int + var found bool + if max { + i, found = len(n.items)-1, true + } else { + i, found = tr.find(n, key, hint, depth) + } + if n.leaf() { + if found { + // found the items at the leaf, remove it and return. + prev := n.items[i] + copy(n.items[i:], n.items[i+1:]) + n.items[len(n.items)-1] = tr.empty + n.items = n.items[:len(n.items)-1] + n.count-- + return prev, true + } + return tr.empty, false + } + + var prev kind + var deleted bool + if found { + if max { + i++ + prev, deleted = tr.delete(&(*n.children)[i], true, tr.empty, nil, 0) + } else { + prev = n.items[i] + maxItem, _ := tr.delete(&(*n.children)[i], true, tr.empty, nil, 0) + deleted = true + n.items[i] = maxItem + } + } else { + prev, deleted = tr.delete(&(*n.children)[i], max, key, hint, depth+1) + } + if !deleted { + return tr.empty, false + } + n.count-- + if len((*n.children)[i].items) < minItems { + tr.nodeRebalance(n, i) + } + return prev, true + +} + +// nodeRebalance rebalances the child nodes following a delete operation. +// Provide the index of the child node with the number of items that fell +// below minItems. +func (tr *bTree) nodeRebalance(n *node, i int) { + if i == len(n.items) { + i-- + } + + // ensure copy-on-write + left := tr.cowLoad(&(*n.children)[i]) + right := tr.cowLoad(&(*n.children)[i+1]) + + if len(left.items)+len(right.items) < maxItems { + // Merges the left and right children nodes together as a single node + // that includes (left,item,right), and places the contents into the + // existing left node. Delete the right node altogether and move the + // following items and child nodes to the left by one slot. + + // merge (left,item,right) + left.items = append(left.items, n.items[i]) + left.items = append(left.items, right.items...) + if !left.leaf() { + *left.children = append(*left.children, *right.children...) + } + left.count += right.count + 1 + + // move the items over one slot + copy(n.items[i:], n.items[i+1:]) + n.items[len(n.items)-1] = tr.empty + n.items = n.items[:len(n.items)-1] + + // move the children over one slot + copy((*n.children)[i+1:], (*n.children)[i+2:]) + (*n.children)[len(*n.children)-1] = nil + (*n.children) = (*n.children)[:len(*n.children)-1] + } else if len(left.items) > len(right.items) { + // move left -> right over one slot + + // Move the item of the parent node at index into the right-node first + // slot, and move the left-node last item into the previously moved + // parent item slot. + right.items = append(right.items, tr.empty) + copy(right.items[1:], right.items) + right.items[0] = n.items[i] + right.count++ + n.items[i] = left.items[len(left.items)-1] + left.items[len(left.items)-1] = tr.empty + left.items = left.items[:len(left.items)-1] + left.count-- + + if !left.leaf() { + // move the left-node last child into the right-node first slot + *right.children = append(*right.children, nil) + copy((*right.children)[1:], *right.children) + (*right.children)[0] = (*left.children)[len(*left.children)-1] + (*left.children)[len(*left.children)-1] = nil + (*left.children) = (*left.children)[:len(*left.children)-1] + left.count -= (*right.children)[0].count + right.count += (*right.children)[0].count + } + } else { + // move left <- right over one slot + + // Same as above but the other direction + left.items = append(left.items, n.items[i]) + left.count++ + n.items[i] = right.items[0] + copy(right.items, right.items[1:]) + right.items[len(right.items)-1] = tr.empty + right.items = right.items[:len(right.items)-1] + right.count-- + + if !left.leaf() { + *left.children = append(*left.children, (*right.children)[0]) + copy(*right.children, (*right.children)[1:]) + (*right.children)[len(*right.children)-1] = nil + *right.children = (*right.children)[:len(*right.children)-1] + left.count += (*left.children)[len(*left.children)-1].count + right.count -= (*left.children)[len(*left.children)-1].count + } + } +} + +// Ascend the tree within the range [pivot, last] +// Pass nil for pivot to scan all item in ascending order +// Return false to stop iterating +func (tr *bTree) Ascend(pivot kind, iter func(item kind) bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil { + return + } + tr.ascend(tr.root, pivot, nil, 0, iter) +} + +// The return value of this function determines whether we should keep iterating +// upon this functions return. +func (tr *bTree) ascend(n *node, pivot kind, + hint *bPathHint, depth int, iter func(item kind) bool, +) bool { + i, found := tr.find(n, pivot, hint, depth) + if !found { + if !n.leaf() { + if !tr.ascend((*n.children)[i], pivot, hint, depth+1, iter) { + return false + } + } + } + // We are either in the case that + // - node is found, we should iterate through it starting at `i`, + // the index it was located at. + // - node is not found, and TODO: fill in. + for ; i < len(n.items); i++ { + if !iter(n.items[i]) { + return false + } + if !n.leaf() { + if !(*n.children)[i+1].scan(iter) { + return false + } + } + } + return true +} + +func (tr *bTree) Reverse(iter func(item kind) bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil { + return + } + tr.root.reverse(iter) +} + +func (n *node) reverse(iter func(item kind) bool) bool { + if n.leaf() { + for i := len(n.items) - 1; i >= 0; i-- { + if !iter(n.items[i]) { + return false + } + } + return true + } + if !(*n.children)[len(*n.children)-1].reverse(iter) { + return false + } + for i := len(n.items) - 1; i >= 0; i-- { + if !iter(n.items[i]) { + return false + } + if !(*n.children)[i].reverse(iter) { + return false + } + } + return true +} + +// Descend the tree within the range [pivot, first] +// Pass nil for pivot to scan all item in descending order +// Return false to stop iterating +func (tr *bTree) Descend(pivot kind, iter func(item kind) bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil { + return + } + tr.descend(tr.root, pivot, nil, 0, iter) +} + +func (tr *bTree) descend(n *node, pivot kind, + hint *bPathHint, depth int, iter func(item kind) bool, +) bool { + i, found := tr.find(n, pivot, hint, depth) + if !found { + if !n.leaf() { + if !tr.descend((*n.children)[i], pivot, hint, depth+1, iter) { + return false + } + } + i-- + } + for ; i >= 0; i-- { + if !iter(n.items[i]) { + return false + } + if !n.leaf() { + if !(*n.children)[i].reverse(iter) { + return false + } + } + } + return true +} + +// Load is for bulk loading pre-sorted items +func (tr *bTree) Load(item kind) (kind, bool) { + if tr.lock() { + defer tr.unlock() + } + if tr.root == nil { + return tr.setHint(item, nil) + } + n := tr.cowLoad(&tr.root) + for { + n.count++ // optimistically update counts + if n.leaf() { + if len(n.items) < maxItems { + if tr.Less(n.items[len(n.items)-1], item) { + n.items = append(n.items, item) + tr.count++ + return tr.empty, false + } + } + break + } + n = tr.cowLoad(&(*n.children)[len(*n.children)-1]) + } + // revert the counts + n = tr.root + for { + n.count-- + if n.leaf() { + break + } + n = (*n.children)[len(*n.children)-1] + } + return tr.setHint(item, nil) +} + +// Min returns the minimum item in tree. +// Returns nil if the tree has no items. +func (tr *bTree) Min() (kind, bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil { + return tr.empty, false + } + n := tr.root + for { + if n.leaf() { + return n.items[0], true + } + n = (*n.children)[0] + } +} + +// Max returns the maximum item in tree. +// Returns nil if the tree has no items. +func (tr *bTree) Max() (kind, bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil { + return tr.empty, false + } + n := tr.root + for { + if n.leaf() { + return n.items[len(n.items)-1], true + } + n = (*n.children)[len(*n.children)-1] + } +} + +// PopMin removes the minimum item in tree and returns it. +// Returns nil if the tree has no items. +func (tr *bTree) PopMin() (kind, bool) { + if tr.lock() { + defer tr.unlock() + } + if tr.root == nil { + return tr.empty, false + } + n := tr.cowLoad(&tr.root) + var item kind + for { + n.count-- // optimistically update counts + if n.leaf() { + item = n.items[0] + if len(n.items) == minItems { + break + } + copy(n.items[:], n.items[1:]) + n.items[len(n.items)-1] = tr.empty + n.items = n.items[:len(n.items)-1] + tr.count-- + if tr.count == 0 { + tr.root = nil + } + return item, true + } + n = tr.cowLoad(&(*n.children)[0]) + } + // revert the counts + n = tr.root + for { + n.count++ + if n.leaf() { + break + } + n = (*n.children)[0] + } + return tr.deleteHint(item, nil) +} + +// PopMax removes the minimum item in tree and returns it. +// Returns nil if the tree has no items. +func (tr *bTree) PopMax() (kind, bool) { + if tr.lock() { + defer tr.unlock() + } + if tr.root == nil { + return tr.empty, false + } + n := tr.cowLoad(&tr.root) + var item kind + for { + n.count-- // optimistically update counts + if n.leaf() { + item = n.items[len(n.items)-1] + if len(n.items) == minItems { + break + } + n.items[len(n.items)-1] = tr.empty + n.items = n.items[:len(n.items)-1] + tr.count-- + if tr.count == 0 { + tr.root = nil + } + return item, true + } + n = tr.cowLoad(&(*n.children)[len(*n.children)-1]) + } + // revert the counts + n = tr.root + for { + n.count++ + if n.leaf() { + break + } + n = (*n.children)[len(*n.children)-1] + } + return tr.deleteHint(item, nil) +} + +// GetAt returns the value at index. +// Return nil if the tree is empty or the index is out of bounds. +func (tr *bTree) GetAt(index int) (kind, bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil || index < 0 || index >= tr.count { + return tr.empty, false + } + n := tr.root + for { + if n.leaf() { + return n.items[index], true + } + i := 0 + for ; i < len(n.items); i++ { + if index < (*n.children)[i].count { + break + } else if index == (*n.children)[i].count { + return n.items[i], true + } + index -= (*n.children)[i].count + 1 + } + n = (*n.children)[i] + } +} + +// DeleteAt deletes the item at index. +// Return nil if the tree is empty or the index is out of bounds. +func (tr *bTree) DeleteAt(index int) (kind, bool) { + if tr.lock() { + defer tr.unlock() + } + if tr.root == nil || index < 0 || index >= tr.count { + return tr.empty, false + } + var pathbuf [8]uint8 // track the path + path := pathbuf[:0] + var item kind + n := tr.cowLoad(&tr.root) +outer: + for { + n.count-- // optimistically update counts + if n.leaf() { + // the index is the item position + item = n.items[index] + if len(n.items) == minItems { + path = append(path, uint8(index)) + break outer + } + copy(n.items[index:], n.items[index+1:]) + n.items[len(n.items)-1] = tr.empty + n.items = n.items[:len(n.items)-1] + tr.count-- + if tr.count == 0 { + tr.root = nil + } + return item, true + } + i := 0 + for ; i < len(n.items); i++ { + if index < (*n.children)[i].count { + break + } else if index == (*n.children)[i].count { + item = n.items[i] + path = append(path, uint8(i)) + break outer + } + index -= (*n.children)[i].count + 1 + } + path = append(path, uint8(i)) + n = tr.cowLoad(&(*n.children)[i]) + } + // revert the counts + var hint bPathHint + n = tr.root + for i := 0; i < len(path); i++ { + if i < len(hint.path) { + hint.path[i] = uint8(path[i]) + hint.used[i] = true + } + n.count++ + if !n.leaf() { + n = (*n.children)[uint8(path[i])] + } + } + return tr.deleteHint(item, &hint) +} + +// Height returns the height of the tree. +// Returns zero if tree has no items. +func (tr *bTree) Height() int { + if tr.rlock() { + defer tr.runlock() + } + var height int + if tr.root != nil { + n := tr.root + for { + height++ + if n.leaf() { + break + } + n = (*n.children)[0] + } + } + return height +} + +// Walk iterates over all items in tree, in order. +// The items param will contain one or more items. +func (tr *bTree) Walk(iter func(item []kind) bool) { + if tr.rlock() { + defer tr.runlock() + } + if tr.root != nil { + tr.root.walk(iter) + } +} + +func (n *node) walk(iter func(item []kind) bool) bool { + if n.leaf() { + if !iter(n.items) { + return false + } + } else { + for i := 0; i < len(n.items); i++ { + (*n.children)[i].walk(iter) + if !iter(n.items[i : i+1]) { + return false + } + } + (*n.children)[len(n.items)].walk(iter) + } + return true +} + +// Copy the tree. This is a copy-on-write operation and is very fast because +// it only performs a shadowed copy. +func (tr *bTree) Copy() *bTree { + if tr.lock() { + defer tr.unlock() + } + tr.cow = new(cow) + tr2 := new(bTree) + *tr2 = *tr + tr2.mu = new(sync.RWMutex) + tr2.cow = new(cow) + return tr2 +} + +func (tr *bTree) lock() bool { + if tr.locks { + tr.mu.Lock() + } + return tr.locks +} + +func (tr *bTree) unlock() { + tr.mu.Unlock() +} + +func (tr *bTree) rlock() bool { + if tr.locks { + tr.mu.RLock() + } + return tr.locks +} + +func (tr *bTree) runlock() { + tr.mu.RUnlock() +} + +// Iter represents an iterator +type bIter struct { + tr *bTree + locked bool + seeked bool + atstart bool + atend bool + stack []iterStackItem + item kind +} + +type iterStackItem struct { + n *node + i int +} + +// Iter returns a read-only iterator. +// The Release method must be called finished with iterator. +func (tr *bTree) Iter() bIter { + var iter bIter + iter.tr = tr + iter.locked = tr.rlock() + return iter +} + +// Seek to item greater-or-equal-to key. +// Returns false if there was no item found. +func (iter *bIter) Seek(key kind) bool { + if iter.tr == nil { + return false + } + iter.seeked = true + iter.stack = iter.stack[:0] + if iter.tr.root == nil { + return false + } + n := iter.tr.root + for { + i, found := iter.tr.find(n, key, nil, 0) + iter.stack = append(iter.stack, iterStackItem{n, i}) + if found { + return true + } + if n.leaf() { + if i == len(n.items) { + iter.stack = iter.stack[:0] + return false + } + return true + } + n = (*n.children)[i] + } +} + +// First moves iterator to first item in tree. +// Returns false if the tree is empty. +func (iter *bIter) First() bool { + if iter.tr == nil { + return false + } + iter.atend = false + iter.atstart = false + iter.seeked = true + iter.stack = iter.stack[:0] + if iter.tr.root == nil { + return false + } + n := iter.tr.root + for { + iter.stack = append(iter.stack, iterStackItem{n, 0}) + if n.leaf() { + break + } + n = (*n.children)[0] + } + s := &iter.stack[len(iter.stack)-1] + iter.item = s.n.items[s.i] + return true +} + +// Last moves iterator to last item in tree. +// Returns false if the tree is empty. +func (iter *bIter) Last() bool { + if iter.tr == nil { + return false + } + iter.seeked = true + iter.stack = iter.stack[:0] + if iter.tr.root == nil { + return false + } + n := iter.tr.root + for { + iter.stack = append(iter.stack, iterStackItem{n, len(n.items)}) + if n.leaf() { + iter.stack[len(iter.stack)-1].i-- + break + } + n = (*n.children)[len(n.items)] + } + s := &iter.stack[len(iter.stack)-1] + iter.item = s.n.items[s.i] + return true +} + +// First moves iterator to first item in tree. +// Returns false if the tree is empty. +func (iter *bIter) Release() { + if iter.tr == nil { + return + } + if iter.locked { + iter.tr.runlock() + iter.locked = false + } + iter.stack = nil + iter.tr = nil +} + +// Next moves iterator to the next item in iterator. +// Returns false if the tree is empty or the iterator is at the end of +// the tree. +func (iter *bIter) Next() bool { + if iter.tr == nil { + return false + } + if !iter.seeked { + return iter.First() + } + if len(iter.stack) == 0 { + if iter.atstart { + return iter.First() && iter.Next() + } + return false + } + s := &iter.stack[len(iter.stack)-1] + s.i++ + if s.n.leaf() { + if s.i == len(s.n.items) { + for { + iter.stack = iter.stack[:len(iter.stack)-1] + if len(iter.stack) == 0 { + iter.atend = true + return false + } + s = &iter.stack[len(iter.stack)-1] + if s.i < len(s.n.items) { + break + } + } + } + } else { + n := (*s.n.children)[s.i] + for { + iter.stack = append(iter.stack, iterStackItem{n, 0}) + if n.leaf() { + break + } + n = (*n.children)[0] + } + } + s = &iter.stack[len(iter.stack)-1] + iter.item = s.n.items[s.i] + return true +} + +// Prev moves iterator to the previous item in iterator. +// Returns false if the tree is empty or the iterator is at the beginning of +// the tree. +func (iter *bIter) Prev() bool { + if iter.tr == nil { + return false + } + if !iter.seeked { + return false + } + if len(iter.stack) == 0 { + if iter.atend { + return iter.Last() && iter.Prev() + } + return false + } + s := &iter.stack[len(iter.stack)-1] + if s.n.leaf() { + s.i-- + if s.i == -1 { + for { + iter.stack = iter.stack[:len(iter.stack)-1] + if len(iter.stack) == 0 { + iter.atstart = true + return false + } + s = &iter.stack[len(iter.stack)-1] + s.i-- + if s.i > -1 { + break + } + } + } + } else { + n := (*s.n.children)[s.i] + for { + iter.stack = append(iter.stack, iterStackItem{n, len(n.items)}) + if n.leaf() { + iter.stack[len(iter.stack)-1].i-- + break + } + n = (*n.children)[len(n.items)] + } + } + s = &iter.stack[len(iter.stack)-1] + iter.item = s.n.items[s.i] + return true +} + +// Item returns the current iterator item. +func (iter *bIter) Item() kind { + return iter.item +} diff --git a/vendor/github.com/tidwall/buntdb/README.md b/vendor/github.com/tidwall/buntdb/README.md index 34a35531..32c11fe4 100644 --- a/vendor/github.com/tidwall/buntdb/README.md +++ b/vendor/github.com/tidwall/buntdb/README.md @@ -137,6 +137,7 @@ All keys/value pairs are ordered in the database by the key. To iterate over the err := db.View(func(tx *buntdb.Tx) error { err := tx.Ascend("", func(key, value string) bool { fmt.Printf("key: %s, value: %s\n", key, value) + return true // continue iteration }) return err }) diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index f4898478..bb56b3d9 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -4,7 +4,9 @@ width="240" height="78" border="0" alt="GJSON">
GoDoc -GJSON Playground +GJSON Playground +GJSON Syntax +

get json values quickly

diff --git a/vendor/github.com/tidwall/gjson/SYNTAX.md b/vendor/github.com/tidwall/gjson/SYNTAX.md index 9bc18c88..1b866346 100644 --- a/vendor/github.com/tidwall/gjson/SYNTAX.md +++ b/vendor/github.com/tidwall/gjson/SYNTAX.md @@ -13,11 +13,11 @@ This document is designed to explain the structure of a GJSON Path through examp - [Dot vs Pipe](#dot-vs-pipe) - [Modifiers](#modifiers) - [Multipaths](#multipaths) +- [Literals](#literals) The definitive implemenation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson). Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online. - ## Path structure A GJSON Path is intended to be easily expressed as a series of components seperated by a `.` character. @@ -296,7 +296,7 @@ Starting with v1.3.0, GJSON added the ability to join multiple paths together to form new documents. Wrapping comma-separated paths between `[...]` or `{...}` will result in a new array or object, respectively. -For example, using the given multipath +For example, using the given multipath: ``` {name.first,age,"the_murphys":friends.#(last="Murphy")#.first} @@ -312,8 +312,28 @@ determined, then "_" is used. This results in -``` +```json {"first":"Tom","age":37,"the_murphys":["Dale","Jane"]} ``` +### Literals +Starting with v1.12.0, GJSON added support of json literals, which provides a way for constructing static blocks of json. This is can be particularly useful when constructing a new json document using [multipaths](#multipaths). + +A json literal begins with the '!' declaration character. + +For example, using the given multipath: + +``` +{name.first,age,"company":!"Happysoft","employed":!true} +``` + +Here we selected the first name and age. Then add two new fields, "company" and "employed". + +This results in + +```json +{"first":"Tom","age":37,"company":"Happysoft","employed":true} +``` + +*See issue [#249](https://github.com/tidwall/gjson/issues/249) for additional context on JSON Literals.* diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index 279649ee..9920c4d2 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -229,17 +229,19 @@ func (t Result) ForEach(iterator func(key, value Result) bool) { return } json := t.Raw - var keys bool + var obj bool var i int var key, value Result for ; i < len(json); i++ { if json[i] == '{' { i++ key.Type = String - keys = true + obj = true break } else if json[i] == '[' { i++ + key.Type = Number + key.Num = -1 break } if json[i] > ' ' { @@ -249,8 +251,9 @@ func (t Result) ForEach(iterator func(key, value Result) bool) { var str string var vesc bool var ok bool + var idx int for ; i < len(json); i++ { - if keys { + if obj { if json[i] != '"' { continue } @@ -265,7 +268,9 @@ func (t Result) ForEach(iterator func(key, value Result) bool) { key.Str = str[1 : len(str)-1] } key.Raw = str - key.Index = s + key.Index = s + t.Index + } else { + key.Num += 1 } for ; i < len(json); i++ { if json[i] <= ' ' || json[i] == ',' || json[i] == ':' { @@ -278,10 +283,17 @@ func (t Result) ForEach(iterator func(key, value Result) bool) { if !ok { return } - value.Index = s + if t.Indexes != nil { + if idx < len(t.Indexes) { + value.Index = t.Indexes[idx] + } + } else { + value.Index = s + t.Index + } if !iterator(key, value) { return } + idx++ } } @@ -298,7 +310,15 @@ func (t Result) Map() map[string]Result { // Get searches result for the specified path. // The result should be a JSON array or object. func (t Result) Get(path string) Result { - return Get(t.Raw, path) + r := Get(t.Raw, path) + if r.Indexes != nil { + for i := 0; i < len(r.Indexes); i++ { + r.Indexes[i] += t.Index + } + } else { + r.Index += t.Index + } + return r } type arrayOrMapResult struct { @@ -389,6 +409,8 @@ func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) { value.Raw, value.Str = tostr(json[i:]) value.Num = 0 } + value.Index = i + t.Index + i += len(value.Raw) - 1 if r.vc == '{' { @@ -415,6 +437,17 @@ func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) { } } end: + if t.Indexes != nil { + if len(t.Indexes) != len(r.a) { + for i := 0; i < len(r.a); i++ { + r.a[i].Index = 0 + } + } else { + for i := 0; i < len(r.a); i++ { + r.a[i].Index = t.Indexes[i] + } + } + } return } @@ -426,7 +459,8 @@ end: // use the Valid function first. func Parse(json string) Result { var value Result - for i := 0; i < len(json); i++ { + i := 0 + for ; i < len(json); i++ { if json[i] == '{' || json[i] == '[' { value.Type = JSON value.Raw = json[i:] // just take the entire raw @@ -436,16 +470,20 @@ func Parse(json string) Result { continue } switch json[i] { - default: - if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + case 'n': + if i+1 < len(json) && json[i+1] != 'u' { + // nan value.Type = Number value.Raw, value.Num = tonum(json[i:]) } else { - return Result{} + // null + value.Type = Null + value.Raw = tolit(json[i:]) } - case 'n': - value.Type = Null - value.Raw = tolit(json[i:]) case 't': value.Type = True value.Raw = tolit(json[i:]) @@ -455,9 +493,14 @@ func Parse(json string) Result { case '"': value.Type = String value.Raw, value.Str = tostr(json[i:]) + default: + return Result{} } break } + if value.Exists() { + value.Index = i + } return value } @@ -531,20 +574,12 @@ func tonum(json string) (raw string, num float64) { return } // could be a '+' or '-'. let's assume so. - continue + } else if json[i] == ']' || json[i] == '}' { + // break on ']' or '}' + raw = json[:i] + num, _ = strconv.ParseFloat(raw, 64) + return } - if json[i] < ']' { - // probably a valid number - continue - } - if json[i] == 'e' || json[i] == 'E' { - // allow for exponential numbers - continue - } - // likely a ']' or '}' - raw = json[:i] - num, _ = strconv.ParseFloat(raw, 64) - return } raw = json num, _ = strconv.ParseFloat(raw, 64) @@ -1513,7 +1548,6 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } if idx < len(c.json) && c.json[idx] != ']' { _, res, ok := parseAny(c.json, idx, true) - parentIndex := res.Index if ok { res := res.Get(rp.alogkey) if res.Exists() { @@ -1525,8 +1559,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { raw = res.String() } jsons = append(jsons, []byte(raw)...) - indexes = append(indexes, - res.Index+parentIndex) + indexes = append(indexes, res.Index) k++ } } @@ -1699,7 +1732,7 @@ type subSelector struct { // first character in path is either '[' or '{', and has already been checked // prior to calling this function. func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) { - modifer := 0 + modifier := 0 depth := 1 colon := 0 start := 1 @@ -1714,6 +1747,7 @@ func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) { } sels = append(sels, sel) colon = 0 + modifier = 0 start = i + 1 } for ; i < len(path); i++ { @@ -1721,11 +1755,11 @@ func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) { case '\\': i++ case '@': - if modifer == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') { - modifer = i + if modifier == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') { + modifier = i } case ':': - if modifer == 0 && colon == 0 && depth == 1 { + if modifier == 0 && colon == 0 && depth == 1 { colon = i } case ',': @@ -1778,7 +1812,7 @@ func isSimpleName(component string) bool { return false } switch component[i] { - case '[', ']', '{', '}', '(', ')', '#', '|': + case '[', ']', '{', '}', '(', ')', '#', '|', '!': return false } } @@ -1842,23 +1876,25 @@ type parseContext struct { // use the Valid function first. func Get(json, path string) Result { if len(path) > 1 { - if !DisableModifiers { - if path[0] == '@' { - // possible modifier - var ok bool - var npath string - var rjson string + if (path[0] == '@' && !DisableModifiers) || path[0] == '!' { + // possible modifier + var ok bool + var npath string + var rjson string + if path[0] == '@' && !DisableModifiers { npath, rjson, ok = execModifier(json, path) - if ok { - path = npath - if len(path) > 0 && (path[0] == '|' || path[0] == '.') { - res := Get(rjson, path[1:]) - res.Index = 0 - res.Indexes = nil - return res - } - return Parse(rjson) + } else if path[0] == '!' { + npath, rjson, ok = execStatic(json, path) + } + if ok { + path = npath + if len(path) > 0 && (path[0] == '|' || path[0] == '.') { + res := Get(rjson, path[1:]) + res.Index = 0 + res.Indexes = nil + return res } + return Parse(rjson) } } if path[0] == '[' || path[0] == '{' { @@ -2527,8 +2563,40 @@ func safeInt(f float64) (n int64, ok bool) { return int64(f), true } +// execStatic parses the path to find a static value. +// The input expects that the path already starts with a '!' +func execStatic(json, path string) (pathOut, res string, ok bool) { + name := path[1:] + if len(name) > 0 { + switch name[0] { + case '{', '[', '"', '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9': + _, res = parseSquash(name, 0) + pathOut = name[len(res):] + return pathOut, res, true + } + } + for i := 1; i < len(path); i++ { + if path[i] == '|' { + pathOut = path[i:] + name = path[1:i] + break + } + if path[i] == '.' { + pathOut = path[i:] + name = path[1:i] + break + } + } + switch strings.ToLower(name) { + case "true", "false", "null", "nan", "inf": + return pathOut, name, true + } + return pathOut, res, false +} + // execModifier parses the path to find a matching modifier function. -// then input expects that the path already starts with a '@' +// The input expects that the path already starts with a '@' func execModifier(json, path string) (pathOut, res string, ok bool) { name := path[1:] var hasArgs bool @@ -2971,3 +3039,176 @@ func stringBytes(s string) []byte { func bytesString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } + +func revSquash(json string) string { + // reverse squash + // expects that the tail character is a ']' or '}' or ')' or '"' + // squash the value, ignoring all nested arrays and objects. + i := len(json) - 1 + var depth int + if json[i] != '"' { + depth++ + } + if json[i] == '}' || json[i] == ']' || json[i] == ')' { + i-- + } + for ; i >= 0; i-- { + switch json[i] { + case '"': + i-- + for ; i >= 0; i-- { + if json[i] == '"' { + esc := 0 + for i > 0 && json[i-1] == '\\' { + i-- + esc++ + } + if esc%2 == 1 { + continue + } + i += esc + break + } + } + if depth == 0 { + if i < 0 { + i = 0 + } + return json[i:] + } + case '}', ']', ')': + depth++ + case '{', '[', '(': + depth-- + if depth == 0 { + return json[i:] + } + } + } + return json +} + +func (t Result) Paths(json string) []string { + if t.Indexes == nil { + return nil + } + paths := make([]string, 0, len(t.Indexes)) + t.ForEach(func(_, value Result) bool { + paths = append(paths, value.Path(json)) + return true + }) + if len(paths) != len(t.Indexes) { + return nil + } + return paths +} + +// Path returns the original GJSON path for Result. +// The json param must be the original JSON used when calling Get. +func (t Result) Path(json string) string { + var path []byte + var comps []string // raw components + i := t.Index - 1 + if t.Index+len(t.Raw) > len(json) { + // JSON cannot safely contain Result. + goto fail + } + if !strings.HasPrefix(json[t.Index:], t.Raw) { + // Result is not at the JSON index as exepcted. + goto fail + } + for ; i >= 0; i-- { + if json[i] <= ' ' { + continue + } + if json[i] == ':' { + // inside of object, get the key + for ; i >= 0; i-- { + if json[i] != '"' { + continue + } + break + } + raw := revSquash(json[:i+1]) + i = i - len(raw) + comps = append(comps, raw) + // key gotten, now squash the rest + raw = revSquash(json[:i+1]) + i = i - len(raw) + i++ // increment the index for next loop step + } else if json[i] == '{' { + // Encountered an open object. The original result was probably an + // object key. + goto fail + } else if json[i] == ',' || json[i] == '[' { + // inside of an array, count the position + var arrIdx int + if json[i] == ',' { + arrIdx++ + i-- + } + for ; i >= 0; i-- { + if json[i] == ':' { + // Encountered an unexpected colon. The original result was + // probably an object key. + goto fail + } else if json[i] == ',' { + arrIdx++ + } else if json[i] == '[' { + comps = append(comps, strconv.Itoa(arrIdx)) + break + } else if json[i] == ']' || json[i] == '}' || json[i] == '"' { + raw := revSquash(json[:i+1]) + i = i - len(raw) + 1 + } + } + } + } + if len(comps) == 0 { + if DisableModifiers { + goto fail + } + return "@this" + } + for i := len(comps) - 1; i >= 0; i-- { + rcomp := Parse(comps[i]) + if !rcomp.Exists() { + goto fail + } + comp := escapeComp(rcomp.String()) + path = append(path, '.') + path = append(path, comp...) + } + if len(path) > 0 { + path = path[1:] + } + return string(path) +fail: + return "" +} + +// isSafePathKeyChar returns true if the input character is safe for not +// needing escaping. +func isSafePathKeyChar(c byte) bool { + return c <= ' ' || c > '~' || c == '_' || c == '-' || c == ':' || + (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') +} + +// escapeComp escaped a path compontent, making it safe for generating a +// path for later use. +func escapeComp(comp string) string { + for i := 0; i < len(comp); i++ { + if !isSafePathKeyChar(comp[i]) { + ncomp := []byte(comp[:i]) + for ; i < len(comp); i++ { + if !isSafePathKeyChar(comp[i]) { + ncomp = append(ncomp, '\\') + } + ncomp = append(ncomp, comp[i]) + } + return string(ncomp) + } + } + return comp +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6c23d3fc..de148610 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -45,16 +45,17 @@ github.com/okzk/sdnotify ## explicit # github.com/stretchr/testify v1.4.0 ## explicit -# github.com/tidwall/btree v0.6.1 +# github.com/tidwall/btree v1.1.0 ## explicit; go 1.16 github.com/tidwall/btree -# github.com/tidwall/buntdb v1.2.7 +github.com/tidwall/btree/internal +# github.com/tidwall/buntdb v1.2.9 ## explicit; go 1.16 github.com/tidwall/buntdb -# github.com/tidwall/gjson v1.10.2 +# github.com/tidwall/gjson v1.12.1 ## explicit; go 1.12 github.com/tidwall/gjson -# github.com/tidwall/grect v0.1.3 +# github.com/tidwall/grect v0.1.4 ## explicit; go 1.15 github.com/tidwall/grect # github.com/tidwall/match v1.1.1