forked from External/ergo
upgrade buntdb
This commit is contained in:
Shivaram Lingamneni
parent
1389d89a9b
commit
c5a9916302
15 changed files with 519 additions and 288 deletions
7
vendor/github.com/tidwall/buntdb/README.md
generated
vendored
7
vendor/github.com/tidwall/buntdb/README.md
generated
vendored
|
|
@ -3,7 +3,6 @@
|
|||
src="logo.png"
|
||||
width="307" height="150" border="0" alt="BuntDB">
|
||||
<br>
|
||||
<a href="https://goreportcard.com/report/github.com/tidwall/buntdb"><img src="https://goreportcard.com/badge/github.com/tidwall/buntdb?style=flat-square" alt="Go Report Card"></a>
|
||||
<a href="https://godoc.org/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square" alt="Godoc"></a>
|
||||
<a href="https://github.com/tidwall/buntdb/blob/master/LICENSE"><img src="https://img.shields.io/github/license/tidwall/buntdb.svg?style=flat-square" alt="LICENSE"></a>
|
||||
</p>
|
||||
|
|
@ -28,7 +27,6 @@ Features
|
|||
- Flexible [iteration](#iterating) of data; ascending, descending, and ranges
|
||||
- [Durable append-only file](#append-only-file) format for persistence
|
||||
- Option to evict old items with an [expiration](#data-expiration) TTL
|
||||
- Tight codebase, under 2K loc using the `cloc` command
|
||||
- ACID semantics with locking [transactions](#transactions) that support rollbacks
|
||||
|
||||
|
||||
|
|
@ -457,8 +455,9 @@ Any index can be put in descending order by wrapping it's less function with `bu
|
|||
|
||||
```go
|
||||
db.CreateIndex("last_name_age", "*",
|
||||
buntdb.IndexJSON("name.last"),
|
||||
buntdb.Desc(buntdb.IndexJSON("age")))
|
||||
buntdb.IndexJSON("name.last"),
|
||||
buntdb.Desc(buntdb.IndexJSON("age")),
|
||||
)
|
||||
```
|
||||
|
||||
This will create a multi value index where the last name is ascending and the age is descending.
|
||||
|
|
|
|||
110
vendor/github.com/tidwall/buntdb/buntdb.go
generated
vendored
110
vendor/github.com/tidwall/buntdb/buntdb.go
generated
vendored
|
|
@ -69,6 +69,7 @@ type DB struct {
|
|||
keys *btree.BTree // a tree of all item ordered by key
|
||||
exps *btree.BTree // a tree of items ordered by expiration
|
||||
idxs map[string]*index // the index trees.
|
||||
insIdxs []*index // a reuse buffer for gathering indexes
|
||||
flushes int // a count of the number of disk flushes
|
||||
closed bool // set when the database has been closed
|
||||
config Config // the database configuration
|
||||
|
|
@ -139,8 +140,8 @@ type exctx struct {
|
|||
func Open(path string) (*DB, error) {
|
||||
db := &DB{}
|
||||
// initialize trees and indexes
|
||||
db.keys = btree.New(lessCtx(nil))
|
||||
db.exps = btree.New(lessCtx(&exctx{db}))
|
||||
db.keys = btreeNew(lessCtx(nil))
|
||||
db.exps = btreeNew(lessCtx(&exctx{db}))
|
||||
db.idxs = make(map[string]*index)
|
||||
// initialize default configuration
|
||||
db.config = Config{
|
||||
|
|
@ -200,10 +201,11 @@ func (db *DB) Save(wr io.Writer) error {
|
|||
defer db.mu.RUnlock()
|
||||
// use a buffered writer and flush every 4MB
|
||||
var buf []byte
|
||||
now := time.Now()
|
||||
// iterated through every item in the database and write to the buffer
|
||||
btreeAscend(db.keys, func(item interface{}) bool {
|
||||
dbi := item.(*dbItem)
|
||||
buf = dbi.writeSetTo(buf)
|
||||
buf = dbi.writeSetTo(buf, now)
|
||||
if len(buf) > 1024*1024*4 {
|
||||
// flush when buffer is over 4MB
|
||||
_, err = wr.Write(buf)
|
||||
|
|
@ -283,7 +285,7 @@ func (idx *index) clearCopy() *index {
|
|||
}
|
||||
// initialize with empty trees
|
||||
if nidx.less != nil {
|
||||
nidx.btr = btree.New(lessCtx(nidx))
|
||||
nidx.btr = btreeNew(lessCtx(nidx))
|
||||
}
|
||||
if nidx.rect != nil {
|
||||
nidx.rtr = rtred.New(nidx)
|
||||
|
|
@ -295,7 +297,7 @@ func (idx *index) clearCopy() *index {
|
|||
func (idx *index) rebuild() {
|
||||
// initialize trees
|
||||
if idx.less != nil {
|
||||
idx.btr = btree.New(lessCtx(idx))
|
||||
idx.btr = btreeNew(lessCtx(idx))
|
||||
}
|
||||
if idx.rect != nil {
|
||||
idx.rtr = rtred.New(idx)
|
||||
|
|
@ -454,16 +456,23 @@ func (db *DB) SetConfig(config Config) error {
|
|||
// will be replaced with the new one, and return the previous item.
|
||||
func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
|
||||
var pdbi *dbItem
|
||||
// Generate a list of indexes that this item will be inserted in to.
|
||||
idxs := db.insIdxs
|
||||
for _, idx := range db.idxs {
|
||||
if idx.match(item.key) {
|
||||
idxs = append(idxs, idx)
|
||||
}
|
||||
}
|
||||
prev := db.keys.Set(item)
|
||||
if prev != nil {
|
||||
// A previous item was removed from the keys tree. Let's
|
||||
// fully delete this item from all indexes.
|
||||
pdbi = prev.(*dbItem)
|
||||
if pdbi.opts != nil && pdbi.opts.ex {
|
||||
// Remove it from the exipres tree.
|
||||
// Remove it from the expires tree.
|
||||
db.exps.Delete(pdbi)
|
||||
}
|
||||
for _, idx := range db.idxs {
|
||||
for _, idx := range idxs {
|
||||
if idx.btr != nil {
|
||||
// Remove it from the btree index.
|
||||
idx.btr.Delete(pdbi)
|
||||
|
|
@ -479,10 +488,7 @@ func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
|
|||
// expires tree
|
||||
db.exps.Set(item)
|
||||
}
|
||||
for _, idx := range db.idxs {
|
||||
if !idx.match(item.key) {
|
||||
continue
|
||||
}
|
||||
for i, idx := range idxs {
|
||||
if idx.btr != nil {
|
||||
// Add new item to btree index.
|
||||
idx.btr.Set(item)
|
||||
|
|
@ -491,7 +497,11 @@ func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
|
|||
// Add new item to rtree index.
|
||||
idx.rtr.Insert(item)
|
||||
}
|
||||
// clear the index
|
||||
idxs[i] = nil
|
||||
}
|
||||
// reuse the index list slice
|
||||
db.insIdxs = idxs[:0]
|
||||
// we must return the previous item to the caller.
|
||||
return pdbi
|
||||
}
|
||||
|
|
@ -512,6 +522,9 @@ func (db *DB) deleteFromDatabase(item *dbItem) *dbItem {
|
|||
db.exps.Delete(pdbi)
|
||||
}
|
||||
for _, idx := range db.idxs {
|
||||
if !idx.match(pdbi.key) {
|
||||
continue
|
||||
}
|
||||
if idx.btr != nil {
|
||||
// Remove it from the btree index.
|
||||
idx.btr.Delete(pdbi)
|
||||
|
|
@ -672,6 +685,7 @@ func (db *DB) Shrink() error {
|
|||
}
|
||||
done = true
|
||||
var n int
|
||||
now := time.Now()
|
||||
btreeAscendGreaterOrEqual(db.keys, &dbItem{key: pivot},
|
||||
func(item interface{}) bool {
|
||||
dbi := item.(*dbItem)
|
||||
|
|
@ -681,7 +695,7 @@ func (db *DB) Shrink() error {
|
|||
done = false
|
||||
return false
|
||||
}
|
||||
buf = dbi.writeSetTo(buf)
|
||||
buf = dbi.writeSetTo(buf, now)
|
||||
n++
|
||||
return true
|
||||
},
|
||||
|
|
@ -908,8 +922,8 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) (n int64, err error) {
|
|||
db.deleteFromDatabase(&dbItem{key: parts[1]})
|
||||
} else if (parts[0][0] == 'f' || parts[0][0] == 'F') &&
|
||||
strings.ToLower(parts[0]) == "flushdb" {
|
||||
db.keys = btree.New(lessCtx(nil))
|
||||
db.exps = btree.New(lessCtx(&exctx{db}))
|
||||
db.keys = btreeNew(lessCtx(nil))
|
||||
db.exps = btreeNew(lessCtx(&exctx{db}))
|
||||
db.idxs = make(map[string]*index)
|
||||
} else {
|
||||
return totalSize, ErrInvalid
|
||||
|
|
@ -941,11 +955,16 @@ func (db *DB) load() error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
pos, err := db.file.Seek(n, 0)
|
||||
if err != nil {
|
||||
if _, err := db.file.Seek(n, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
db.lastaofsz = int(pos)
|
||||
var estaofsz int
|
||||
db.keys.Walk(func(items []interface{}) {
|
||||
for _, v := range items {
|
||||
estaofsz += v.(*dbItem).estAOFSetSize()
|
||||
}
|
||||
})
|
||||
db.lastaofsz += estaofsz
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -1054,8 +1073,8 @@ func (tx *Tx) DeleteAll() error {
|
|||
}
|
||||
|
||||
// now reset the live database trees
|
||||
tx.db.keys = btree.New(lessCtx(nil))
|
||||
tx.db.exps = btree.New(lessCtx(&exctx{tx.db}))
|
||||
tx.db.keys = btreeNew(lessCtx(nil))
|
||||
tx.db.exps = btreeNew(lessCtx(&exctx{tx.db}))
|
||||
tx.db.idxs = make(map[string]*index)
|
||||
|
||||
// finally re-create the indexes
|
||||
|
|
@ -1165,12 +1184,13 @@ func (tx *Tx) Commit() error {
|
|||
if tx.wc.rbkeys != nil {
|
||||
tx.db.buf = append(tx.db.buf, "*1\r\n$7\r\nflushdb\r\n"...)
|
||||
}
|
||||
now := time.Now()
|
||||
// Each committed record is written to disk
|
||||
for key, item := range tx.wc.commitItems {
|
||||
if item == nil {
|
||||
tx.db.buf = (&dbItem{key: key}).writeDeleteTo(tx.db.buf)
|
||||
} else {
|
||||
tx.db.buf = item.writeSetTo(tx.db.buf)
|
||||
tx.db.buf = item.writeSetTo(tx.db.buf, now)
|
||||
}
|
||||
}
|
||||
// Flushing the buffer only once per transaction.
|
||||
|
|
@ -1243,16 +1263,53 @@ type dbItem struct {
|
|||
keyless bool // keyless item for scanning
|
||||
}
|
||||
|
||||
func estIntSize(x int) int {
|
||||
if x == 0 {
|
||||
return 1
|
||||
}
|
||||
var n int
|
||||
for x > 0 {
|
||||
n++
|
||||
x /= 10
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func estArraySize(count int) int {
|
||||
return 1 + estIntSize(count) + 2
|
||||
}
|
||||
|
||||
func estBulkStringSize(s string) int {
|
||||
return 1 + estIntSize(len(s)) + 2 + len(s) + 2
|
||||
}
|
||||
|
||||
func (dbi *dbItem) estAOFSetSize() (n int) {
|
||||
if dbi.opts != nil && dbi.opts.ex {
|
||||
n += estArraySize(5)
|
||||
n += estBulkStringSize("set")
|
||||
n += estBulkStringSize(dbi.key)
|
||||
n += estBulkStringSize(dbi.val)
|
||||
n += estBulkStringSize("ex")
|
||||
n += estBulkStringSize("99") // estimate two byte bulk string
|
||||
} else {
|
||||
n += estArraySize(3)
|
||||
n += estBulkStringSize("set")
|
||||
n += estBulkStringSize(dbi.key)
|
||||
n += estBulkStringSize(dbi.val)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func appendArray(buf []byte, count int) []byte {
|
||||
buf = append(buf, '*')
|
||||
buf = append(buf, strconv.FormatInt(int64(count), 10)...)
|
||||
buf = strconv.AppendInt(buf, int64(count), 10)
|
||||
buf = append(buf, '\r', '\n')
|
||||
return buf
|
||||
}
|
||||
|
||||
func appendBulkString(buf []byte, s string) []byte {
|
||||
buf = append(buf, '$')
|
||||
buf = append(buf, strconv.FormatInt(int64(len(s)), 10)...)
|
||||
buf = strconv.AppendInt(buf, int64(len(s)), 10)
|
||||
buf = append(buf, '\r', '\n')
|
||||
buf = append(buf, s...)
|
||||
buf = append(buf, '\r', '\n')
|
||||
|
|
@ -1260,9 +1317,9 @@ func appendBulkString(buf []byte, s string) []byte {
|
|||
}
|
||||
|
||||
// writeSetTo writes an item as a single SET record to the a bufio Writer.
|
||||
func (dbi *dbItem) writeSetTo(buf []byte) []byte {
|
||||
func (dbi *dbItem) writeSetTo(buf []byte, now time.Time) []byte {
|
||||
if dbi.opts != nil && dbi.opts.ex {
|
||||
ex := time.Until(dbi.opts.exat) / time.Second
|
||||
ex := dbi.opts.exat.Sub(now) / time.Second
|
||||
buf = appendArray(buf, 5)
|
||||
buf = appendBulkString(buf, "set")
|
||||
buf = appendBulkString(buf, dbi.key)
|
||||
|
|
@ -2300,3 +2357,8 @@ func btreeDescendLessOrEqual(tr *btree.BTree, pivot interface{},
|
|||
) {
|
||||
tr.Descend(pivot, iter)
|
||||
}
|
||||
|
||||
func btreeNew(less func(a, b interface{}) bool) *btree.BTree {
|
||||
// Using NewNonConcurrent because we're managing our own locks.
|
||||
return btree.NewNonConcurrent(less)
|
||||
}
|
||||
|
|
|
|||
6
vendor/github.com/tidwall/buntdb/go.mod
generated
vendored
6
vendor/github.com/tidwall/buntdb/go.mod
generated
vendored
|
|
@ -3,9 +3,9 @@ module github.com/tidwall/buntdb
|
|||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/tidwall/btree v0.4.2
|
||||
github.com/tidwall/gjson v1.7.4
|
||||
github.com/tidwall/grect v0.1.1
|
||||
github.com/tidwall/btree v0.6.0
|
||||
github.com/tidwall/gjson v1.8.0
|
||||
github.com/tidwall/grect v0.1.2
|
||||
github.com/tidwall/lotsa v1.0.2
|
||||
github.com/tidwall/match v1.0.3
|
||||
github.com/tidwall/rtred v0.1.2
|
||||
|
|
|
|||
12
vendor/github.com/tidwall/buntdb/go.sum
generated
vendored
12
vendor/github.com/tidwall/buntdb/go.sum
generated
vendored
|
|
@ -1,9 +1,9 @@
|
|||
github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs=
|
||||
github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
|
||||
github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8=
|
||||
github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
|
||||
github.com/tidwall/grect v0.1.1 h1:+kMEkxhoqB7rniVXzMEIA66XwU07STgINqxh+qVIndY=
|
||||
github.com/tidwall/grect v0.1.1/go.mod h1:CzvbGiFbWUwiJ1JohXLb28McpyBsI00TK9Y6pDWLGRQ=
|
||||
github.com/tidwall/btree v0.6.0 h1:JLYAFGV+1gjyFi3iQbO/fupBin+Ooh7dxqVV0twJ1Bo=
|
||||
github.com/tidwall/btree v0.6.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4=
|
||||
github.com/tidwall/gjson v1.8.0 h1:Qt+orfosKn0rbNTZqHYDqBrmm3UDA4KRkv70fDzG+PQ=
|
||||
github.com/tidwall/gjson v1.8.0/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
|
||||
github.com/tidwall/grect v0.1.2 h1:wKVeQVZhjaFCKTTlpkDe3Ex4ko3cMGW3MRKawRe8uQ4=
|
||||
github.com/tidwall/grect v0.1.2/go.mod h1:v+n4ewstPGduVJebcp5Eh2WXBJBumNzyhK8GZt4gHNw=
|
||||
github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
|
||||
github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
|
||||
github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue