bump buntdb to v1.2.3

Potentially fixes the database corruption seen on #1603
2024-11-25 05:19:25 +01:00 · 2021-04-01 20:45:15 -04:00 · 2021-04-01 20:45:15 -04:00 · fd3cbab6ee
commit fd3cbab6ee
parent b022c34a23
36 changed files with 912 additions and 324 deletions
--- a/go.mod
+++ b/go.mod
@ -16,7 +16,8 @@ require (
 	github.com/oragono/confusables v0.0.0-20201108231250-4ab98ab61fb1
 	github.com/oragono/go-ident v0.0.0-20200511222032-830550b1d775
 	github.com/stretchr/testify v1.4.0 // indirect
-	github.com/tidwall/buntdb v1.1.4
+	github.com/tidwall/buntdb v1.2.3
+	github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 // indirect
 	github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208
 	golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
 	golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf // indirect
--- a/go.sum
+++ b/go.sum
@ -67,20 +67,37 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY=
 github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs=
+github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
 github.com/tidwall/buntdb v1.1.4 h1:W7y9+2dM3GOswU0t3pz6+BcwZXjj/tVOhPcO6EHufME=
 github.com/tidwall/buntdb v1.1.4/go.mod h1:06+/n7EFf6uUaIG5r9xZcExYN3H0Lnc+g/Kqx0fZFkI=
+github.com/tidwall/buntdb v1.2.3 h1:AoGVe4yrhKmnEPHrPrW5EUOATHOCIk4VtFvd8xn/ZtU=
+github.com/tidwall/buntdb v1.2.3/go.mod h1:+i/gBwYOHWG19wLgwMXFLkl00twh9+VWkkaOhuNQ4PA=
 github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
 github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
+github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8=
+github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
 github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
 github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
+github.com/tidwall/grect v0.1.1 h1:+kMEkxhoqB7rniVXzMEIA66XwU07STgINqxh+qVIndY=
+github.com/tidwall/grect v0.1.1/go.mod h1:CzvbGiFbWUwiJ1JohXLb28McpyBsI00TK9Y6pDWLGRQ=
+github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
 github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
+github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
 github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
+github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
+github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
 github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8SutN9KQHc6vqHEXJTcAFwyGNius=
 github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
 github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
 github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
+github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM=
 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
--- a/vendor/github.com/tidwall/btree/.travis.yml
+++ b/vendor/github.com/tidwall/btree/.travis.yml
@ -1 +0,0 @@
-language: go
--- a/vendor/github.com/tidwall/btree/README.md
+++ b/vendor/github.com/tidwall/btree/README.md
@ -1,5 +1,199 @@
-# B-tree for Go
+# btree

-![Travis CI Build Status](https://api.travis-ci.org/tidwall/btree.svg?branch=master)
 [![GoDoc](https://godoc.org/github.com/tidwall/btree?status.svg)](https://godoc.org/github.com/tidwall/btree)

+An [efficient](#performance) [B-tree](https://en.wikipedia.org/wiki/B-tree) implementation in Go. 
+
+## Features
+
+- `Copy()` method with copy-on-write support.
+- Fast bulk loading for pre-ordered data using the `Load()` method.
+- All operations are thread-safe.
+- Path hinting optimization for operations with nearby keys.
+
+## Installing
+
+To start using btree, install Go and run `go get`:
+
+```sh
+$ go get -u github.com/tidwall/btree
+```
+
+## Usage
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"github.com/tidwall/btree"
+)
+
+type Item struct {
+	Key, Val string
+}
+
+// byKeys is a comparison function that compares item keys and returns true
+// when a is less than b.
+func byKeys(a, b interface{}) bool {
+	i1, i2 := a.(*Item), b.(*Item)
+	return i1.Key < i2.Key
+}
+
+// byVals is a comparison function that compares item values and returns true
+// when a is less than b.
+func byVals(a, b interface{}) bool {
+	i1, i2 := a.(*Item), b.(*Item)
+	if i1.Val < i2.Val {
+		return true
+	}
+	if i1.Val > i2.Val {
+		return false
+	}
+	// Both vals are equal so we should fall though
+	// and let the key comparison take over.
+	return byKeys(a, b)
+}
+
+func main() {
+	// Create a tree for keys and a tree for values.
+	// The "keys" tree will be sorted on the Keys field.
+	// The "values" tree will be sorted on the Values field.
+	keys := btree.New(byKeys)
+	vals := btree.New(byVals)
+
+	// Create some items.
+	users := []*Item{
+		&Item{Key: "user:1", Val: "Jane"},
+		&Item{Key: "user:2", Val: "Andy"},
+		&Item{Key: "user:3", Val: "Steve"},
+		&Item{Key: "user:4", Val: "Andrea"},
+		&Item{Key: "user:5", Val: "Janet"},
+		&Item{Key: "user:6", Val: "Andy"},
+	}
+
+	// Insert each user into both trees
+	for _, user := range users {
+		keys.Set(user)
+		vals.Set(user)
+	}
+
+	// Iterate over each user in the key tree
+	keys.Ascend(nil, func(item interface{}) bool {
+		kvi := item.(*Item)
+		fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
+		return true
+	})
+
+	fmt.Printf("\n")
+	// Iterate over each user in the val tree
+	vals.Ascend(nil, func(item interface{}) bool {
+		kvi := item.(*Item)
+		fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
+		return true
+	})
+
+	// Output:
+	// user:1 Jane
+	// user:2 Andy
+	// user:3 Steve
+	// user:4 Andrea
+	// user:5 Janet
+	// user:6 Andy
+	//
+	// user:4 Andrea
+	// user:2 Andy
+	// user:6 Andy
+	// user:1 Jane
+	// user:5 Janet
+	// user:3 Steve
+}
+```
+
+## Operations
+
+### Basic
+
+```
+Len()                   # return the number of items in the btree
+Set(item)               # insert or replace an existing item
+Get(item)               # get an existing item
+Delete(item)            # delete an item
+```
+
+### Iteration
+
+```
+Ascend(pivot, iter)     # scan items in ascending order starting at pivot.
+Descend(pivot, iter)    # scan items in descending order starting at pivot.
+```
+
+### Queues
+
+```
+Min()                   # return the first item in the btree
+Max()                   # return the last item in the btree
+PopMin()                # remove and return the first item in the btree
+PopMax()                # remove and return the last item in the btree
+```
+### Bulk loading
+
+```
+Load(item)              # load presorted items into tree
+```
+
+### Path hints
+
+```
+SetHint(item, *hint)    # insert or replace an existing item
+GetHint(item, *hint)    # get an existing item
+DeleteHint(item, *hint) # delete an item
+```
+
+## Performance
+
+This implementation was designed with performance in mind. 
+
+The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel Core i9) using Go 1.15.3. The items are simple 8-byte ints. 
+
+- `google`: The [google/btree](https://github.com/google/btree) package
+- `tidwall`: The [tidwall/btree](https://github.com/tidwall/btree) package
+- `go-arr`: Just a simple Go array
+
+```
+** sequential set **
+google:  set-seq        1,000,000 ops in 160ms, 6,262,097/sec, 159 ns/op, 31.0 MB, 32 bytes/op
+tidwall: set-seq        1,000,000 ops in 142ms, 7,020,721/sec, 142 ns/op, 36.6 MB, 38 bytes/op
+tidwall: set-seq-hint   1,000,000 ops in 87ms, 11,503,315/sec, 86 ns/op, 36.6 MB, 38 bytes/op
+tidwall: load-seq       1,000,000 ops in 37ms, 27,177,242/sec, 36 ns/op, 36.6 MB, 38 bytes/op
+go-arr:  append         1,000,000 ops in 49ms, 20,574,760/sec, 48 ns/op
+
+** random set **
+google:  set-rand       1,000,000 ops in 606ms, 1,649,921/sec, 606 ns/op, 21.5 MB, 22 bytes/op
+tidwall: set-rand       1,000,000 ops in 543ms, 1,841,590/sec, 543 ns/op, 26.7 MB, 27 bytes/op
+tidwall: set-rand-hint  1,000,000 ops in 573ms, 1,745,624/sec, 572 ns/op, 26.4 MB, 27 bytes/op
+tidwall: set-again      1,000,000 ops in 452ms, 2,212,581/sec, 451 ns/op, 27.1 MB, 28 bytes/op
+tidwall: set-after-copy 1,000,000 ops in 472ms, 2,117,457/sec, 472 ns/op, 27.9 MB, 29 bytes/op
+tidwall: load-rand      1,000,000 ops in 551ms, 1,816,498/sec, 550 ns/op, 26.1 MB, 27 bytes/op
+
+** sequential get **
+google:  get-seq        1,000,000 ops in 133ms, 7,497,604/sec, 133 ns/op
+tidwall: get-seq        1,000,000 ops in 110ms, 9,082,972/sec, 110 ns/op
+tidwall: get-seq-hint   1,000,000 ops in 55ms, 18,289,945/sec, 54 ns/op
+
+** random get **
+google:  get-rand       1,000,000 ops in 149ms, 6,704,337/sec, 149 ns/op
+tidwall: get-rand       1,000,000 ops in 131ms, 7,616,296/sec, 131 ns/op
+tidwall: get-rand-hint  1,000,000 ops in 216ms, 4,632,532/sec, 215 ns/op
+```
+
+*You can find the benchmark utility at [tidwall/btree-benchmark](https://github.com/tidwall/btree-benchmark)*
+
+## Contact
+
+Josh Baker [@tidwall](http://twitter.com/tidwall)
+
+## License
+
+Source code is available under the MIT [License](/LICENSE).
--- a/vendor/github.com/tidwall/btree/btree.go
+++ b/vendor/github.com/tidwall/btree/btree.go
@ -4,35 +4,39 @@

 package btree

+import "sync"
+
 const maxItems = 255
 const minItems = maxItems * 40 / 100

+type cow struct {
+	_ int // it cannot be an empty struct
+}
+
 type node struct {
+	cow      *cow
 	leaf     bool
 	numItems int16
 	items    [maxItems]interface{}
 	children *[maxItems + 1]*node
 }

-type justaLeaf struct {
-	leaf     bool
-	numItems int16
-	items    [maxItems]interface{}
-}
-
 // BTree is an ordered set items
 type BTree struct {
+	mu     *sync.RWMutex
+	cow    *cow
 	root   *node
 	length int
 	less   func(a, b interface{}) bool
 	lnode  *node
 }

-func newNode(leaf bool) *node {
+func (tr *BTree) newNode(leaf bool) *node {
 	n := &node{leaf: leaf}
 	if !leaf {
 		n.children = new([maxItems + 1]*node)
 	}
+	n.cow = tr.cow
 	return n
 }

@ -48,6 +52,7 @@ func New(less func(a, b interface{}) bool) *BTree {
 		panic("nil less")
 	}
 	tr := new(BTree)
+	tr.mu = new(sync.RWMutex)
 	tr.less = less
 	return tr
 }
@ -85,8 +90,7 @@ func (n *node) find(key interface{}, less func(a, b interface{}) bool,
 			high = mid - 1
 		}
 	}
-	if low > 0 && !less(n.items[low-1], key) &&
-		!less(key, n.items[low-1]) {
+	if low > 0 && !less(n.items[low-1], key) {
 		index = low - 1
 		found = true
 	} else {
@ -109,22 +113,29 @@ func (tr *BTree) SetHint(item interface{}, hint *PathHint) (prev interface{}) {
 	if item == nil {
 		panic("nil item")
 	}
+	tr.mu.Lock()
+	prev = tr.setHint(item, hint)
+	tr.mu.Unlock()
+	return prev
+}
+
+func (tr *BTree) setHint(item interface{}, hint *PathHint) (prev interface{}) {
 	if tr.root == nil {
-		tr.root = newNode(true)
+		tr.root = tr.newNode(true)
 		tr.root.items[0] = item
 		tr.root.numItems = 1
 		tr.length = 1
 		return
 	}
-	prev = tr.root.set(item, tr.less, hint, 0)
+	prev = tr.nodeSet(&tr.root, item, tr.less, hint, 0)
 	if prev != nil {
 		return prev
 	}
 	tr.lnode = nil
 	if tr.root.numItems == maxItems {
-		n := tr.root
-		right, median := n.split()
-		tr.root = newNode(false)
+		n := tr.cowLoad(&tr.root)
+		right, median := tr.nodeSplit(n)
+		tr.root = tr.newNode(false)
 		tr.root.children[0] = n
 		tr.root.items[0] = median
 		tr.root.children[1] = right
@ -139,8 +150,8 @@ func (tr *BTree) Set(item interface{}) (prev interface{}) {
 	return tr.SetHint(item, nil)
 }

-func (n *node) split() (right *node, median interface{}) {
-	right = newNode(n.leaf)
+func (tr *BTree) nodeSplit(n *node) (right *node, median interface{}) {
+	right = tr.newNode(n.leaf)
 	median = n.items[maxItems/2]
 	copy(right.items[:maxItems/2], n.items[maxItems/2+1:])
 	if !n.leaf {
@ -159,9 +170,30 @@ func (n *node) split() (right *node, median interface{}) {
 	return right, median
 }

-func (n *node) set(item interface{}, less func(a, b interface{}) bool,
-	hint *PathHint, depth int,
+//go:noinline
+func (tr *BTree) copy(n *node) *node {
+	n2 := *n
+	n2.cow = tr.cow
+	copy(n2.items[:], n.items[:])
+	if n.children != nil {
+		n2.children = new([maxItems + 1]*node)
+		copy(n2.children[:], n.children[:])
+	}
+	return &n2
+}
+
+// cowLoad loaded the provide node and, if needed, performs a copy-on-write.
+func (tr *BTree) cowLoad(cn **node) *node {
+	if (*cn).cow != tr.cow {
+		*cn = tr.copy(*cn)
+	}
+	return *cn
+}
+
+func (tr *BTree) nodeSet(cn **node, item interface{},
+	less func(a, b interface{}) bool, hint *PathHint, depth int,
 ) (prev interface{}) {
+	n := tr.cowLoad(cn)
 	i, found := n.find(item, less, hint, depth)
 	if found {
 		prev = n.items[i]
@ -174,12 +206,12 @@ func (n *node) set(item interface{}, less func(a, b interface{}) bool,
 		n.numItems++
 		return nil
 	}
-	prev = n.children[i].set(item, less, hint, depth+1)
+	prev = tr.nodeSet(&n.children[i], item, less, hint, depth+1)
 	if prev != nil {
 		return prev
 	}
 	if n.children[i].numItems == maxItems {
-		right, median := n.children[i].split()
+		right, median := tr.nodeSplit(n.children[i])
 		copy(n.children[i+1:], n.children[i:])
 		copy(n.items[i+1:], n.items[i:])
 		n.items[i] = median
@ -216,6 +248,8 @@ func (tr *BTree) Get(key interface{}) interface{} {

 // GetHint gets a value for key using a path hint
 func (tr *BTree) GetHint(key interface{}, hint *PathHint) interface{} {
+	tr.mu.RLock()
+	defer tr.mu.RUnlock()
 	if tr.root == nil || key == nil {
 		return nil
 	}
@ -246,10 +280,17 @@ func (tr *BTree) Delete(key interface{}) interface{} {

 // DeleteHint deletes a value for a key using a path hint
 func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} {
+	tr.mu.Lock()
+	prev := tr.deleteHint(key, hint)
+	tr.mu.Unlock()
+	return prev
+}
+
+func (tr *BTree) deleteHint(key interface{}, hint *PathHint) interface{} {
 	if tr.root == nil || key == nil {
 		return nil
 	}
-	prev := tr.root.delete(false, key, tr.less, hint, 0)
+	prev := tr.delete(&tr.root, false, key, tr.less, hint, 0)
 	if prev == nil {
 		return nil
 	}
@ -264,9 +305,10 @@ func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} {
 	return prev
 }

-func (n *node) delete(max bool, key interface{},
+func (tr *BTree) delete(cn **node, max bool, key interface{},
 	less func(a, b interface{}) bool, hint *PathHint, depth int,
 ) interface{} {
+	n := tr.cowLoad(cn)
 	var i int16
 	var found bool
 	if max {
@ -290,22 +332,28 @@ func (n *node) delete(max bool, key interface{},
 	if found {
 		if max {
 			i++
-			prev = n.children[i].delete(true, "", less, nil, 0)
+			prev = tr.delete(&n.children[i], true, "", less, nil, 0)
 		} else {
 			prev = n.items[i]
-			maxItem := n.children[i].delete(true, "", less, nil, 0)
+			maxItem := tr.delete(&n.children[i], true, "", less, nil, 0)
 			n.items[i] = maxItem
 		}
 	} else {
-		prev = n.children[i].delete(max, key, less, hint, depth+1)
+		prev = tr.delete(&n.children[i], max, key, less, hint, depth+1)
 	}
 	if prev == nil {
 		return nil
 	}
-	if n.children[i].numItems < minItems {
+	if n.children[i].numItems >= minItems {
+		return prev
+	}
+
+	// merge / rebalance nodes
 	if i == n.numItems {
 		i--
 	}
+	n.children[i] = tr.cowLoad(&n.children[i])
+	n.children[i+1] = tr.cowLoad(&n.children[i+1])
 	if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems {
 		// merge left + item + right
 		n.children[i].items[n.children[i].numItems] = n.items[i]
@ -358,7 +406,6 @@ func (n *node) delete(max bool, key interface{},
 		}
 		n.children[i+1].numItems--
 	}
-	}
 	return prev
 }

@ -366,6 +413,8 @@ func (n *node) delete(max bool, key interface{},
 // Pass nil for pivot to scan all item in ascending order
 // Return false to stop iterating
 func (tr *BTree) Ascend(pivot interface{}, iter func(item interface{}) bool) {
+	tr.mu.RLock()
+	defer tr.mu.RUnlock()
 	if tr.root == nil {
 		return
 	}
@ -427,6 +476,8 @@ func (n *node) reverse(iter func(item interface{}) bool) bool {
 // Pass nil for pivot to scan all item in descending order
 // Return false to stop iterating
 func (tr *BTree) Descend(pivot interface{}, iter func(item interface{}) bool) {
+	tr.mu.RLock()
+	defer tr.mu.RUnlock()
 	if tr.root == nil {
 		return
 	}
@ -467,6 +518,12 @@ func (tr *BTree) Load(item interface{}) interface{} {
 	if item == nil {
 		panic("nil item")
 	}
+	tr.mu.Lock()
+	defer tr.mu.Unlock()
+
+	// Load does not need a cowGrid because the Copy operation sets the
+	// lnode to nil.
+
 	if tr.lnode != nil && tr.lnode.numItems < maxItems-2 {
 		if tr.less(tr.lnode.items[tr.lnode.numItems-1], item) {
 			tr.lnode.items[tr.lnode.numItems] = item
@ -475,7 +532,7 @@ func (tr *BTree) Load(item interface{}) interface{} {
 			return nil
 		}
 	}
-	prev := tr.Set(item)
+	prev := tr.setHint(item, nil)
 	if prev != nil {
 		return prev
 	}
@ -493,6 +550,8 @@ func (tr *BTree) Load(item interface{}) interface{} {
 // Min returns the minimum item in tree.
 // Returns nil if the tree has no items.
 func (tr *BTree) Min() interface{} {
+	tr.mu.RLock()
+	defer tr.mu.RUnlock()
 	if tr.root == nil {
 		return nil
 	}
@ -508,6 +567,8 @@ func (tr *BTree) Min() interface{} {
 // Max returns the maximum item in tree.
 // Returns nil if the tree has no items.
 func (tr *BTree) Max() interface{} {
+	tr.mu.RLock()
+	defer tr.mu.RUnlock()
 	if tr.root == nil {
 		return nil
 	}
@ -523,53 +584,65 @@ func (tr *BTree) Max() interface{} {
 // PopMin removes the minimum item in tree and returns it.
 // Returns nil if the tree has no items.
 func (tr *BTree) PopMin() interface{} {
+	tr.mu.Lock()
+	defer tr.mu.Unlock()
 	if tr.root == nil {
 		return nil
 	}
 	tr.lnode = nil
-	n := tr.root
+	n := tr.cowLoad(&tr.root)
 	for {
 		if n.leaf {
 			item := n.items[0]
 			if n.numItems == minItems {
-				return tr.Delete(item)
+				return tr.deleteHint(item, nil)
 			}
 			copy(n.items[:], n.items[1:])
 			n.items[n.numItems-1] = nil
 			n.numItems--
 			tr.length--
+			if tr.length == 0 {
+				tr.root = nil
+			}
 			return item
 		}
-		n = n.children[0]
+		n = tr.cowLoad(&n.children[0])
 	}
 }

 // PopMax removes the minimum item in tree and returns it.
 // Returns nil if the tree has no items.
 func (tr *BTree) PopMax() interface{} {
+	tr.mu.Lock()
+	defer tr.mu.Unlock()
 	if tr.root == nil {
 		return nil
 	}
 	tr.lnode = nil
-	n := tr.root
+	n := tr.cowLoad(&tr.root)
 	for {
 		if n.leaf {
 			item := n.items[n.numItems-1]
 			if n.numItems == minItems {
-				return tr.Delete(item)
+				return tr.deleteHint(item, nil)
 			}
 			n.items[n.numItems-1] = nil
 			n.numItems--
 			tr.length--
+			if tr.length == 0 {
+				tr.root = nil
+			}
 			return item
 		}
-		n = n.children[n.numItems]
+		n = tr.cowLoad(&n.children[n.numItems])
 	}
 }

 // Height returns the height of the tree.
 // Returns zero if tree has no items.
 func (tr *BTree) Height() int {
+	tr.mu.RLock()
+	defer tr.mu.RUnlock()
 	var height int
 	if tr.root != nil {
 		n := tr.root
@ -587,6 +660,8 @@ func (tr *BTree) Height() int {
 // Walk iterates over all items in tree, in order.
 // The items param will contain one or more items.
 func (tr *BTree) Walk(iter func(item []interface{})) {
+	tr.mu.RLock()
+	defer tr.mu.RUnlock()
 	if tr.root != nil {
 		tr.root.walk(iter)
 	}
@ -603,3 +678,16 @@ func (n *node) walk(iter func(item []interface{})) {
 		n.children[n.numItems].walk(iter)
 	}
 }
+
+// Copy the tree. This operation is very fast because it only performs a
+// shadowed copy.
+func (tr *BTree) Copy() *BTree {
+	tr.mu.Lock()
+	tr.lnode = nil
+	tr.cow = new(cow)
+	tr2 := *tr
+	tr2.mu = new(sync.RWMutex)
+	tr2.cow = new(cow)
+	tr.mu.Unlock()
+	return &tr2
+}
--- a/vendor/github.com/tidwall/buntdb/.travis.yml
+++ b/vendor/github.com/tidwall/buntdb/.travis.yml
@ -1,4 +0,0 @@
-language: go
-
-go:
-    - 1.15.x
--- a/vendor/github.com/tidwall/buntdb/README.md
+++ b/vendor/github.com/tidwall/buntdb/README.md
@ -3,10 +3,9 @@
    src="logo.png"
    width="307" height="150" border="0" alt="BuntDB">
 <br>
-<a href="https://travis-ci.org/tidwall/buntdb"><img src="https://img.shields.io/travis/tidwall/buntdb.svg?style=flat-square" alt="Build Status"></a>
-<a href="http://gocover.io/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/coverage-95%25-brightgreen.svg?style=flat-square" alt="Code Coverage"></a>
 <a href="https://goreportcard.com/report/github.com/tidwall/buntdb"><img src="https://goreportcard.com/badge/github.com/tidwall/buntdb?style=flat-square" alt="Go Report Card"></a>
-<a href="https://godoc.org/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
+<a href="https://godoc.org/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square" alt="Godoc"></a>
+<a href="https://github.com/tidwall/buntdb/blob/master/LICENSE"><img src="https://img.shields.io/github/license/tidwall/buntdb.svg?style=flat-square" alt="LICENSE"></a>
 </p>

 BuntDB is a low-level, in-memory, key/value store in pure Go.
--- a/vendor/github.com/tidwall/buntdb/buntdb.go
+++ b/vendor/github.com/tidwall/buntdb/buntdb.go
@ -19,7 +19,7 @@ import (
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/grect"
 	"github.com/tidwall/match"
-	"github.com/tidwall/rtree"
+	"github.com/tidwall/rtred"
 )

 var (
@ -69,7 +69,6 @@ type DB struct {
 	keys      *btree.BTree      // a tree of all item ordered by key
 	exps      *btree.BTree      // a tree of items ordered by expiration
 	idxs      map[string]*index // the index trees.
-	exmgr     bool              // indicates that expires manager is running.
 	flushes   int               // a count of the number of disk flushes
 	closed    bool              // set when the database has been closed
 	config    Config            // the database configuration
@ -135,9 +134,6 @@ type exctx struct {
 	db *DB
 }

-// Default number of btree degrees
-const btreeDegrees = 64
-
 // Open opens a database at the provided path.
 // If the file does not exist then it will be created automatically.
 func Open(path string) (*DB, error) {
@ -241,14 +237,15 @@ func (db *DB) Load(rd io.Reader) error {
 		// cannot load into databases that persist to disk
 		return ErrPersistenceActive
 	}
-	return db.readLoad(rd, time.Now())
+	_, err := db.readLoad(rd, time.Now())
+	return err
 }

 // index represents a b-tree or r-tree index and also acts as the
 // b-tree/r-tree context for itself.
 type index struct {
 	btr     *btree.BTree                           // contains the items
-	rtr     *rtree.RTree                           // contains the items
+	rtr     *rtred.RTree                           // contains the items
 	name    string                                 // name of the index
 	pattern string                                 // a required key pattern
 	less    func(a, b string) bool                 // less comparison function
@ -289,7 +286,7 @@ func (idx *index) clearCopy() *index {
 		nidx.btr = btree.New(lessCtx(nidx))
 	}
 	if nidx.rect != nil {
-		nidx.rtr = rtree.New(nidx)
+		nidx.rtr = rtred.New(nidx)
 	}
 	return nidx
 }
@ -301,7 +298,7 @@ func (idx *index) rebuild() {
 		idx.btr = btree.New(lessCtx(idx))
 	}
 	if idx.rect != nil {
-		idx.rtr = rtree.New(idx)
+		idx.rtr = rtred.New(idx)
 	}
 	// iterate through all keys and fill the index
 	btreeAscend(idx.db.keys, func(item interface{}) bool {
@ -755,46 +752,65 @@ func (db *DB) Shrink() error {
 	}()
 }

-var errValidEOF = errors.New("valid eof")
-
 // readLoad reads from the reader and loads commands into the database.
 // modTime is the modified time of the reader, should be no greater than
 // the current time.Now().
-func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
+// Returns the number of bytes of the last command read and the error if any.
+func (db *DB) readLoad(rd io.Reader, modTime time.Time) (n int64, err error) {
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+	totalSize := int64(0)
 	data := make([]byte, 4096)
 	parts := make([]string, 0, 8)
 	r := bufio.NewReader(rd)
 	for {
+		// peek at the first byte. If it's a 'nul' control character then
+		// ignore it and move to the next byte.
+		c, err := r.ReadByte()
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return totalSize, err
+		}
+		if c == 0 {
+			// ignore nul control characters
+			n += 1
+			continue
+		}
+		if err := r.UnreadByte(); err != nil {
+			return totalSize, err
+		}
+
 		// read a single command.
 		// first we should read the number of parts that the of the command
+		cmdByteSize := int64(0)
 		line, err := r.ReadBytes('\n')
 		if err != nil {
-			if len(line) > 0 {
-				// got an eof but also data. this should be an unexpected eof.
-				return io.ErrUnexpectedEOF
-			}
-			if err == io.EOF {
-				break
-			}
-			return err
+			return totalSize, err
 		}
 		if line[0] != '*' {
-			return ErrInvalid
+			return totalSize, ErrInvalid
 		}
+		cmdByteSize += int64(len(line))
+
 		// convert the string number to and int
 		var n int
 		if len(line) == 4 && line[len(line)-2] == '\r' {
 			if line[1] < '0' || line[1] > '9' {
-				return ErrInvalid
+				return totalSize, ErrInvalid
 			}
 			n = int(line[1] - '0')
 		} else {
 			if len(line) < 5 || line[len(line)-2] != '\r' {
-				return ErrInvalid
+				return totalSize, ErrInvalid
 			}
 			for i := 1; i < len(line)-2; i++ {
 				if line[i] < '0' || line[i] > '9' {
-					return ErrInvalid
+					return totalSize, ErrInvalid
 				}
 				n = n*10 + int(line[i]-'0')
 			}
@ -805,25 +821,26 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
 			// read the number of bytes of the part.
 			line, err := r.ReadBytes('\n')
 			if err != nil {
-				return err
+				return totalSize, err
 			}
 			if line[0] != '$' {
-				return ErrInvalid
+				return totalSize, ErrInvalid
 			}
+			cmdByteSize += int64(len(line))
 			// convert the string number to and int
 			var n int
 			if len(line) == 4 && line[len(line)-2] == '\r' {
 				if line[1] < '0' || line[1] > '9' {
-					return ErrInvalid
+					return totalSize, ErrInvalid
 				}
 				n = int(line[1] - '0')
 			} else {
 				if len(line) < 5 || line[len(line)-2] != '\r' {
-					return ErrInvalid
+					return totalSize, ErrInvalid
 				}
 				for i := 1; i < len(line)-2; i++ {
 					if line[i] < '0' || line[i] > '9' {
-						return ErrInvalid
+						return totalSize, ErrInvalid
 					}
 					n = n*10 + int(line[i]-'0')
 				}
@ -837,13 +854,14 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
 				data = make([]byte, dataln)
 			}
 			if _, err = io.ReadFull(r, data[:n+2]); err != nil {
-				return err
+				return totalSize, err
 			}
 			if data[n] != '\r' || data[n+1] != '\n' {
-				return ErrInvalid
+				return totalSize, ErrInvalid
 			}
 			// copy string
 			parts = append(parts, string(data[:n]))
+			cmdByteSize += int64(n + 2)
 		}
 		// finished reading the command

@ -855,15 +873,15 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
 			(parts[0][2] == 't' || parts[0][2] == 'T') {
 			// SET
 			if len(parts) < 3 || len(parts) == 4 || len(parts) > 5 {
-				return ErrInvalid
+				return totalSize, ErrInvalid
 			}
 			if len(parts) == 5 {
 				if strings.ToLower(parts[3]) != "ex" {
-					return ErrInvalid
+					return totalSize, ErrInvalid
 				}
-				ex, err := strconv.ParseInt(parts[4], 10, 64)
+				ex, err := strconv.ParseUint(parts[4], 10, 64)
 				if err != nil {
-					return err
+					return totalSize, err
 				}
 				now := time.Now()
 				dur := (time.Duration(ex) * time.Second) - now.Sub(modTime)
@ -885,7 +903,7 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
 			(parts[0][2] == 'l' || parts[0][2] == 'L') {
 			// DEL
 			if len(parts) != 2 {
-				return ErrInvalid
+				return totalSize, ErrInvalid
 			}
 			db.deleteFromDatabase(&dbItem{key: parts[1]})
 		} else if (parts[0][0] == 'f' || parts[0][0] == 'F') &&
@ -894,10 +912,10 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
 			db.exps = btree.New(lessCtx(&exctx{db}))
 			db.idxs = make(map[string]*index)
 		} else {
-			return ErrInvalid
+			return totalSize, ErrInvalid
 		}
+		totalSize += cmdByteSize
 	}
-	return nil
 }

 // load reads entries from the append only database file and fills the database.
@ -910,10 +928,20 @@ func (db *DB) load() error {
 	if err != nil {
 		return err
 	}
-	if err := db.readLoad(db.file, fi.ModTime()); err != nil {
+	n, err := db.readLoad(db.file, fi.ModTime())
+	if err != nil {
+		if err == io.ErrUnexpectedEOF {
+			// The db file has ended mid-command, which is allowed but the
+			// data file should be truncated to the end of the last valid
+			// command
+			if err := db.file.Truncate(n); err != nil {
 				return err
 			}
-	pos, err := db.file.Seek(0, 2)
+		} else {
+			return err
+		}
+	}
+	pos, err := db.file.Seek(n, 0)
 	if err != nil {
 		return err
 	}
@ -1148,7 +1176,25 @@ func (tx *Tx) Commit() error {
 		// Flushing the buffer only once per transaction.
 		// If this operation fails then the write did failed and we must
 		// rollback.
-		if _, err = tx.db.file.Write(tx.db.buf); err != nil {
+		var n int
+		n, err = tx.db.file.Write(tx.db.buf)
+		if err != nil {
+			if n > 0 {
+				// There was a partial write to disk.
+				// We are possibly out of disk space.
+				// Delete the partially written bytes from the data file by
+				// seeking to the previously known position and performing
+				// a truncate operation.
+				// At this point a syscall failure is fatal and the process
+				// should be killed to avoid corrupting the file.
+				pos, err := tx.db.file.Seek(-int64(n), 1)
+				if err != nil {
+					panic(err)
+				}
+				if err := tx.db.file.Truncate(pos); err != nil {
+					panic(err)
+				}
+			}
 			tx.rollbackInner()
 		}
 		if tx.db.config.SyncPolicy == Always {
@ -1216,7 +1262,7 @@ func appendBulkString(buf []byte, s string) []byte {
 // writeSetTo writes an item as a single SET record to the a bufio Writer.
 func (dbi *dbItem) writeSetTo(buf []byte) []byte {
 	if dbi.opts != nil && dbi.opts.ex {
-		ex := dbi.opts.exat.Sub(time.Now()) / time.Second
+		ex := time.Until(dbi.opts.exat) / time.Second
 		buf = appendArray(buf, 5)
 		buf = appendBulkString(buf, "set")
 		buf = appendBulkString(buf, dbi.key)
@ -1390,7 +1436,9 @@ func (tx *Tx) Set(key, value string, opts *SetOptions) (previousValue string,
 			// create a rollback entry with a nil value. A nil value indicates
 			// that the entry should be deleted on rollback. When the value is
 			// *not* nil, that means the entry should be reverted.
+			if _, ok := tx.wc.rollbackItems[key]; !ok {
 				tx.wc.rollbackItems[key] = nil
+			}
 		} else {
 			// A previous item already exists in the database. Let's create a
 			// rollback entry with the item as the value. We need to check the
@ -1481,7 +1529,7 @@ func (tx *Tx) TTL(key string) (time.Duration, error) {
 	} else if item.opts == nil || !item.opts.ex {
 		return -1, nil
 	}
-	dur := item.opts.exat.Sub(time.Now())
+	dur := time.Until(item.opts.exat)
 	if dur < 0 {
 		return 0, ErrNotFound
 	}
@ -1822,7 +1870,7 @@ func (tx *Tx) Nearby(index, bounds string,
 		return nil
 	}
 	// // wrap a rtree specific iterator around the user-defined iterator.
-	iter := func(item rtree.Item, dist float64) bool {
+	iter := func(item rtred.Item, dist float64) bool {
 		dbi := item.(*dbItem)
 		return iterator(dbi.key, dbi.val, dist)
 	}
@ -1860,7 +1908,7 @@ func (tx *Tx) Intersects(index, bounds string,
 		return nil
 	}
 	// wrap a rtree specific iterator around the user-defined iterator.
-	iter := func(item rtree.Item) bool {
+	iter := func(item rtred.Item) bool {
 		dbi := item.(*dbItem)
 		return iterator(dbi.key, dbi.val)
 	}
--- a/vendor/github.com/tidwall/buntdb/go.mod
+++ b/vendor/github.com/tidwall/buntdb/go.mod
@ -1,12 +1,12 @@
 module github.com/tidwall/buntdb

-go 1.15
+go 1.16

 require (
-	github.com/tidwall/btree v0.2.2
-	github.com/tidwall/gjson v1.6.1
-	github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb
-	github.com/tidwall/match v1.0.1
-	github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8
-	github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect
+	github.com/tidwall/btree v0.4.2
+	github.com/tidwall/gjson v1.7.4
+	github.com/tidwall/grect v0.1.1
+	github.com/tidwall/lotsa v1.0.2
+	github.com/tidwall/match v1.0.3
+	github.com/tidwall/rtred v0.1.2
 )
--- a/vendor/github.com/tidwall/buntdb/go.sum
+++ b/vendor/github.com/tidwall/buntdb/go.sum
@ -1,14 +1,16 @@
-github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY=
-github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
-github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
-github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
-github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
-github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
-github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
-github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
-github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
-github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8SutN9KQHc6vqHEXJTcAFwyGNius=
-github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
-github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
-github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
+github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs=
+github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8=
+github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
+github.com/tidwall/grect v0.1.1 h1:+kMEkxhoqB7rniVXzMEIA66XwU07STgINqxh+qVIndY=
+github.com/tidwall/grect v0.1.1/go.mod h1:CzvbGiFbWUwiJ1JohXLb28McpyBsI00TK9Y6pDWLGRQ=
+github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
+github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
+github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
+github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
+github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
+github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
+github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
+github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
--- a/vendor/github.com/tidwall/gjson/.travis.yml
+++ b/vendor/github.com/tidwall/gjson/.travis.yml
@ -1 +0,0 @@
-language: go
--- a/vendor/github.com/tidwall/gjson/README.md
+++ b/vendor/github.com/tidwall/gjson/README.md
@ -3,13 +3,10 @@
    src="logo.png" 
    width="240" height="78" border="0" alt="GJSON">
 <br>
-<a href="https://travis-ci.org/tidwall/gjson"><img src="https://img.shields.io/travis/tidwall/gjson.svg?style=flat-square" alt="Build Status"></a>
 <a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
 <a href="http://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a>
 </p>

-
-
 <p align="center">get json values quickly</a></p>

 GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
@ -17,6 +14,8 @@ It has features such as [one line retrieval](#get-a-value), [dot notation paths]

 Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool.

+For the Rust version go to [gjson.rs](https://github.com/tidwall/gjson.rs).
+
 Getting Started
 ===============

@ -476,7 +475,7 @@ JSON document used:
 }    
 ```

-Each operation was rotated though one of the following search paths:
+Each operation was rotated through one of the following search paths:

 ```
 widget.window.name
@ -484,12 +483,4 @@ widget.image.hOffset
 widget.text.onMouseUp
 ```

-*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).*
-
-
-## Contact
-Josh Baker [@tidwall](http://twitter.com/tidwall)
-
-## License
-
-GJSON source code is available under the MIT [License](/LICENSE).
+*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be found [here](https://github.com/tidwall/gjson-benchmarks).*
--- a/vendor/github.com/tidwall/gjson/SYNTAX.md
+++ b/vendor/github.com/tidwall/gjson/SYNTAX.md
@ -77,14 +77,21 @@ Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`.
 fav\.movie             "Deer Hunter"
 ```

-You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in source code.
+You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in you source code.

 ```go
-res := gjson.Get(json, "fav\\.movie")  // must escape the slash
-res := gjson.Get(json, `fav\.movie`)   // no need to escape the slash 
-
+// Go
+val := gjson.Get(json, "fav\\.movie")  // must escape the slash
+val := gjson.Get(json, `fav\.movie`)   // no need to escape the slash 
 ```

+```rust
+// Rust
+let val = gjson::get(json, "fav\\.movie")     // must escape the slash
+let val = gjson::get(json, r#"fav\.movie"#)   // no need to escape the slash 
+```
+
+
 ### Arrays

 The `#` character allows for digging into JSON Arrays.
@ -248,6 +255,8 @@ gjson.AddModifier("case", func(json, arg string) string {
 "children.@case:lower.@reverse"    ["jack","alex","sara"]
 ```

+*Note: Custom modifiers are not yet available in the Rust version*
+
 ### Multipaths

 Starting with v1.3.0, GJSON added the ability to join multiple paths together
--- a/vendor/github.com/tidwall/gjson/gjson.go
+++ b/vendor/github.com/tidwall/gjson/gjson.go
@ -3,7 +3,6 @@ package gjson

 import (
 	"encoding/json"
-	"reflect"
 	"strconv"
 	"strings"
 	"time"
@ -106,7 +105,8 @@ func (t Result) Bool() bool {
 	case True:
 		return true
 	case String:
-		return t.Str != "" && t.Str != "0" && t.Str != "false"
+		b, _ := strconv.ParseBool(strings.ToLower(t.Str))
+		return b
 	case Number:
 		return t.Num != 0
 	}
@ -124,17 +124,18 @@ func (t Result) Int() int64 {
 		return n
 	case Number:
 		// try to directly convert the float64 to int64
-		n, ok := floatToInt(t.Num)
-		if !ok {
+		i, ok := safeInt(t.Num)
+		if ok {
+			return i
+		}
 		// now try to parse the raw string
-			n, ok = parseInt(t.Raw)
-			if !ok {
+		i, ok = parseInt(t.Raw)
+		if ok {
+			return i
+		}
 		// fallback to a standard conversion
 		return int64(t.Num)
 	}
-		}
-		return n
-	}
 }

 // Uint returns an unsigned integer representation.
@ -149,17 +150,18 @@ func (t Result) Uint() uint64 {
 		return n
 	case Number:
 		// try to directly convert the float64 to uint64
-		n, ok := floatToUint(t.Num)
-		if !ok {
+		i, ok := safeInt(t.Num)
+		if ok && i >= 0 {
+			return uint64(i)
+		}
 		// now try to parse the raw string
-			n, ok = parseUint(t.Raw)
-			if !ok {
+		u, ok := parseUint(t.Raw)
+		if ok {
+			return u
+		}
 		// fallback to a standard conversion
 		return uint64(t.Num)
 	}
-		}
-		return n
-	}
 }

 // Float returns an float64 representation.
@ -495,6 +497,9 @@ func squash(json string) string {
 					}
 				}
 				if depth == 0 {
+					if i >= len(json) {
+						return json
+					}
 					return json[:i+1]
 				}
 			case '{', '[', '(':
@ -726,8 +731,13 @@ func parseArrayPath(path string) (r arrayPathResult) {
 		}
 		if path[i] == '.' {
 			r.part = path[:i]
+			if !r.arrch && i < len(path)-1 && isDotPiperChar(path[i+1]) {
+				r.pipe = path[i+1:]
+				r.piped = true
+			} else {
 				r.path = path[i+1:]
 				r.more = true
+			}
 			return
 		}
 		if path[i] == '#' {
@ -973,6 +983,11 @@ right:
 	return s
 }

+// peek at the next byte and see if it's a '@', '[', or '{'.
+func isDotPiperChar(c byte) bool {
+	return !DisableModifiers && (c == '@' || c == '[' || c == '{')
+}
+
 type objectPathResult struct {
 	part  string
 	path  string
@ -991,12 +1006,8 @@ func parseObjectPath(path string) (r objectPathResult) {
 			return
 		}
 		if path[i] == '.' {
-			// peek at the next byte and see if it's a '@', '[', or '{'.
 			r.part = path[:i]
-			if !DisableModifiers &&
-				i < len(path)-1 &&
-				(path[i+1] == '@' ||
-					path[i+1] == '[' || path[i+1] == '{') {
+			if i < len(path)-1 && isDotPiperChar(path[i+1]) {
 				r.pipe = path[i+1:]
 				r.piped = true
 			} else {
@ -1026,14 +1037,11 @@ func parseObjectPath(path string) (r objectPathResult) {
 						continue
 					} else if path[i] == '.' {
 						r.part = string(epart)
-						// peek at the next byte and see if it's a '@' modifier
-						if !DisableModifiers &&
-							i < len(path)-1 && path[i+1] == '@' {
+						if i < len(path)-1 && isDotPiperChar(path[i+1]) {
 							r.pipe = path[i+1:]
 							r.piped = true
 						} else {
 							r.path = path[i+1:]
-							r.more = true
 						}
 						r.more = true
 						return
@ -1398,7 +1406,6 @@ func parseArray(c *parseContext, i int, path string) (int, bool) {
 		}
 		return false
 	}
-
 	for i < len(c.json)+1 {
 		if !rp.arrch {
 			pmatch = partidx == h
@ -1600,11 +1607,18 @@ func parseArray(c *parseContext, i int, path string) (int, bool) {
 					c.calcd = true
 					return i + 1, true
 				}
-				if len(multires) > 0 && !c.value.Exists() {
+				if !c.value.Exists() {
+					if len(multires) > 0 {
 						c.value = Result{
 							Raw:  string(append(multires, ']')),
 							Type: JSON,
 						}
+					} else if rp.query.all {
+						c.value = Result{
+							Raw:  "[]",
+							Type: JSON,
+						}
+					}
 				}
 				return i + 1, false
 			}
@ -1835,7 +1849,7 @@ type parseContext struct {
 // A path is in dot syntax, such as "name.last" or "age".
 // When the value is found it's returned immediately.
 //
-// A path is a series of keys searated by a dot.
+// A path is a series of keys separated by a dot.
 // A key may contain special wildcard characters '*' and '?'.
 // To access an array value use the index as the key.
 // To get the number of elements in an array or to access a child path, use
@ -1944,7 +1958,6 @@ func Get(json, path string) Result {
 			}
 		}
 	}
-
 	var i int
 	var c = &parseContext{json: json}
 	if len(path) >= 2 && path[0] == '.' && path[1] == '.' {
@ -2169,11 +2182,6 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) {
 	return i, res, false
 }

-var ( // used for testing
-	testWatchForFallback bool
-	testLastWasFallback  bool
-)
-
 // GetMany searches json for the multiple paths.
 // The return value is a Result array where the number of items
 // will be equal to the number of input paths.
@ -2374,6 +2382,12 @@ func validnumber(data []byte, i int) (outi int, ok bool) {
 	// sign
 	if data[i] == '-' {
 		i++
+		if i == len(data) {
+			return i, false
+		}
+		if data[i] < '0' || data[i] > '9' {
+			return i, false
+		}
 	}
 	// int
 	if i == len(data) {
@ -2524,25 +2538,14 @@ func parseInt(s string) (n int64, ok bool) {
 	return n, true
 }

-const minUint53 = 0
-const maxUint53 = 4503599627370495
-const minInt53 = -2251799813685248
-const maxInt53 = 2251799813685247
-
-func floatToUint(f float64) (n uint64, ok bool) {
-	n = uint64(f)
-	if float64(n) == f && n >= minUint53 && n <= maxUint53 {
-		return n, true
-	}
+// safeInt validates a given JSON number
+// ensures it lies within the minimum and maximum representable JSON numbers
+func safeInt(f float64) (n int64, ok bool) {
+	//  https://tc39.es/ecma262/#sec-number.min_safe_integer || https://tc39.es/ecma262/#sec-number.max_safe_integer
+	if f < -9007199254740991 || f > 9007199254740991 {
 		return 0, false
-}
-
-func floatToInt(f float64) (n int64, ok bool) {
-	n = int64(f)
-	if float64(n) == f && n >= minInt53 && n <= maxInt53 {
-		return n, true
 	}
-	return 0, false
+	return int64(f), true
 }

 // execModifier parses the path to find a matching modifier function.
@ -2600,7 +2603,7 @@ func execModifier(json, path string) (pathOut, res string, ok bool) {
 // unwrap removes the '[]' or '{}' characters around json
 func unwrap(json string) string {
 	json = trim(json)
-	if len(json) >= 2 && json[0] == '[' || json[0] == '{' {
+	if len(json) >= 2 && (json[0] == '[' || json[0] == '{') {
 		json = json[1 : len(json)-1]
 	}
 	return json
@ -2632,6 +2635,26 @@ func ModifierExists(name string, fn func(json, arg string) string) bool {
 	return ok
 }

+// cleanWS remove any non-whitespace from string
+func cleanWS(s string) string {
+	for i := 0; i < len(s); i++ {
+		switch s[i] {
+		case ' ', '\t', '\n', '\r':
+			continue
+		default:
+			var s2 []byte
+			for i := 0; i < len(s); i++ {
+				switch s[i] {
+				case ' ', '\t', '\n', '\r':
+					s2 = append(s2, s[i])
+				}
+			}
+			return string(s2)
+		}
+	}
+	return s
+}
+
 // @pretty modifier makes the json look nice.
 func modPretty(json, arg string) string {
 	if len(arg) > 0 {
@ -2641,9 +2664,9 @@ func modPretty(json, arg string) string {
 			case "sortKeys":
 				opts.SortKeys = value.Bool()
 			case "indent":
-				opts.Indent = value.String()
+				opts.Indent = cleanWS(value.String())
 			case "prefix":
-				opts.Prefix = value.String()
+				opts.Prefix = cleanWS(value.String())
 			case "width":
 				opts.Width = int(value.Int())
 			}
@ -2729,19 +2752,24 @@ func modFlatten(json, arg string) string {
 	out = append(out, '[')
 	var idx int
 	res.ForEach(func(_, value Result) bool {
+		var raw string
+		if value.IsArray() {
+			if deep {
+				raw = unwrap(modFlatten(value.Raw, arg))
+			} else {
+				raw = unwrap(value.Raw)
+			}
+		} else {
+			raw = value.Raw
+		}
+		raw = strings.TrimSpace(raw)
+		if len(raw) > 0 {
 			if idx > 0 {
 				out = append(out, ',')
 			}
-		if value.IsArray() {
-			if deep {
-				out = append(out, unwrap(modFlatten(value.Raw, arg))...)
-			} else {
-				out = append(out, unwrap(value.Raw)...)
-			}
-		} else {
-			out = append(out, value.Raw...)
-		}
+			out = append(out, raw...)
 			idx++
+		}
 		return true
 	})
 	out = append(out, ']')
@ -2825,6 +2853,19 @@ func modValid(json, arg string) string {
 	return json
 }

+// stringHeader instead of reflect.StringHeader
+type stringHeader struct {
+	data unsafe.Pointer
+	len  int
+}
+
+// sliceHeader instead of reflect.SliceHeader
+type sliceHeader struct {
+	data unsafe.Pointer
+	len  int
+	cap  int
+}
+
 // getBytes casts the input json bytes to a string and safely returns the
 // results as uniquely allocated data. This operation is intended to minimize
 // copies and allocations for the large json string->[]byte.
@ -2834,14 +2875,14 @@ func getBytes(json []byte, path string) Result {
 		// unsafe cast to string
 		result = Get(*(*string)(unsafe.Pointer(&json)), path)
 		// safely get the string headers
-		rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw))
-		strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str))
+		rawhi := *(*stringHeader)(unsafe.Pointer(&result.Raw))
+		strhi := *(*stringHeader)(unsafe.Pointer(&result.Str))
 		// create byte slice headers
-		rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len}
-		strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len}
-		if strh.Data == 0 {
+		rawh := sliceHeader{data: rawhi.data, len: rawhi.len, cap: rawhi.len}
+		strh := sliceHeader{data: strhi.data, len: strhi.len, cap: rawhi.len}
+		if strh.data == nil {
 			// str is nil
-			if rawh.Data == 0 {
+			if rawh.data == nil {
 				// raw is nil
 				result.Raw = ""
 			} else {
@ -2849,19 +2890,20 @@ func getBytes(json []byte, path string) Result {
 				result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
 			}
 			result.Str = ""
-		} else if rawh.Data == 0 {
+		} else if rawh.data == nil {
 			// raw is nil
 			result.Raw = ""
 			// str has data, safely copy the slice header to a string
 			result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
-		} else if strh.Data >= rawh.Data &&
-			int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len {
+		} else if uintptr(strh.data) >= uintptr(rawh.data) &&
+			uintptr(strh.data)+uintptr(strh.len) <=
+				uintptr(rawh.data)+uintptr(rawh.len) {
 			// Str is a substring of Raw.
-			start := int(strh.Data - rawh.Data)
+			start := uintptr(strh.data) - uintptr(rawh.data)
 			// safely copy the raw slice header
 			result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
 			// substring the raw
-			result.Str = result.Raw[start : start+strh.Len]
+			result.Str = result.Raw[start : start+uintptr(strh.len)]
 		} else {
 			// safely copy both the raw and str slice headers to strings
 			result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
@ -2876,9 +2918,9 @@ func getBytes(json []byte, path string) Result {
 // used instead.
 func fillIndex(json string, c *parseContext) {
 	if len(c.value.Raw) > 0 && !c.calcd {
-		jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json))
-		rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw)))
-		c.value.Index = int(rhdr.Data - jhdr.Data)
+		jhdr := *(*stringHeader)(unsafe.Pointer(&json))
+		rhdr := *(*stringHeader)(unsafe.Pointer(&(c.value.Raw)))
+		c.value.Index = int(uintptr(rhdr.data) - uintptr(jhdr.data))
 		if c.value.Index < 0 || c.value.Index >= len(json) {
 			c.value.Index = 0
 		}
@ -2886,10 +2928,10 @@ func fillIndex(json string, c *parseContext) {
 }

 func stringBytes(s string) []byte {
-	return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
-		Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data,
-		Len:  len(s),
-		Cap:  len(s),
+	return *(*[]byte)(unsafe.Pointer(&sliceHeader{
+		data: (*stringHeader)(unsafe.Pointer(&s)).data,
+		len:  len(s),
+		cap:  len(s),
 	}))
 }

--- a/vendor/github.com/tidwall/gjson/go.mod
+++ b/vendor/github.com/tidwall/gjson/go.mod
@ -3,6 +3,6 @@ module github.com/tidwall/gjson
 go 1.12

 require (
-	github.com/tidwall/match v1.0.1
-	github.com/tidwall/pretty v1.0.2
+	github.com/tidwall/match v1.0.3
+	github.com/tidwall/pretty v1.1.0
 )
--- a/vendor/github.com/tidwall/gjson/go.sum
+++ b/vendor/github.com/tidwall/gjson/go.sum
@ -1,4 +1,4 @@
-github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
-github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
-github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
-github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
+github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
+github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
--- a/vendor/github.com/tidwall/grect/go.mod
+++ b/vendor/github.com/tidwall/grect/go.mod
@ -0,0 +1,5 @@
+module github.com/tidwall/grect
+
+go 1.15
+
+require github.com/tidwall/gjson v1.7.4
--- a/vendor/github.com/tidwall/grect/go.sum
+++ b/vendor/github.com/tidwall/grect/go.sum
@ -0,0 +1,6 @@
+github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8=
+github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
+github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
+github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
+github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
--- a/vendor/github.com/tidwall/match/.travis.yml
+++ b/vendor/github.com/tidwall/match/.travis.yml
@ -1 +0,0 @@
-language: go
--- a/vendor/github.com/tidwall/match/README.md
+++ b/vendor/github.com/tidwall/match/README.md
@ -1,20 +1,17 @@
-Match
-=====
-<a href="https://travis-ci.org/tidwall/match"><img src="https://img.shields.io/travis/tidwall/match.svg?style=flat-square" alt="Build Status"></a>
-<a href="https://godoc.org/github.com/tidwall/match"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
+# Match
+
+[![GoDoc](https://godoc.org/github.com/tidwall/match?status.svg)](https://godoc.org/github.com/tidwall/match)

 Match is a very simple pattern matcher where '*' matches on any 
 number characters and '?' matches on any one character.

-Installing
----------
+## Installing

 ```
 go get -u github.com/tidwall/match
 ```

-Example
-------
+## Example

 ```go
 match.Match("hello", "*llo") 
@ -23,10 +20,10 @@ match.Match("hello", "h*o")
 ```


-Contact
-------
+## Contact
+
 Josh Baker [@tidwall](http://twitter.com/tidwall)

-License
-------
+## License
+
 Redcon source code is available under the MIT [License](/LICENSE).
--- a/vendor/github.com/tidwall/match/go.mod
+++ b/vendor/github.com/tidwall/match/go.mod
@ -0,0 +1,3 @@
+module github.com/tidwall/match
+
+go 1.15
--- a/vendor/github.com/tidwall/match/match.go
+++ b/vendor/github.com/tidwall/match/match.go
@ -1,4 +1,4 @@
-// Match provides a simple pattern matcher with unicode support.
+// Package match provides a simple pattern matcher with unicode support.
 package match

 import "unicode/utf8"
@ -6,7 +6,7 @@ import "unicode/utf8"
 // Match returns true if str matches pattern. This is a very
 // simple wildcard match where '*' matches on any number characters
 // and '?' matches on any one character.
-
+//
 // pattern:
 // 	{ term }
 // term:
@ -16,12 +16,16 @@ import "unicode/utf8"
 // 	'\\' c      matches character c
 //
 func Match(str, pattern string) bool {
+	return deepMatch(str, pattern)
+}
+
+func deepMatch(str, pattern string) bool {
 	if pattern == "*" {
 		return true
 	}
-	return deepMatch(str, pattern)
-}
-func deepMatch(str, pattern string) bool {
+	for len(pattern) > 1 && pattern[0] == '*' && pattern[1] == '*' {
+		pattern = pattern[1:]
+	}
 	for len(pattern) > 0 {
 		if pattern[0] > 0x7f {
 			return deepMatchRune(str, pattern)
@ -52,6 +56,13 @@ func deepMatch(str, pattern string) bool {
 }

 func deepMatchRune(str, pattern string) bool {
+	if pattern == "*" {
+		return true
+	}
+	for len(pattern) > 1 && pattern[0] == '*' && pattern[1] == '*' {
+		pattern = pattern[1:]
+	}
+
 	var sr, pr rune
 	var srsz, prsz int

--- a/vendor/github.com/tidwall/pretty/.travis.yml
+++ b/vendor/github.com/tidwall/pretty/.travis.yml
@ -1 +0,0 @@
-language: go
--- a/vendor/github.com/tidwall/pretty/README.md
+++ b/vendor/github.com/tidwall/pretty/README.md
@ -1,8 +1,6 @@
 # Pretty
-[![Build Status](https://img.shields.io/travis/tidwall/pretty.svg?style=flat-square)](https://travis-ci.org/tidwall/prettty)
-[![Coverage Status](https://img.shields.io/badge/coverage-100%25-brightgreen.svg?style=flat-square)](http://gocover.io/github.com/tidwall/pretty)
-[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty) 

+[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty) 

 Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads.

@ -81,6 +79,45 @@ Will format the json to:
 {"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}```
 ```

+## Spec
+
+Spec cleans comments and trailing commas from input JSON, converting it to
+valid JSON per the official spec: https://tools.ietf.org/html/rfc8259
+
+The resulting JSON will always be the same length as the input and it will
+include all of the same line breaks at matching offsets. This is to ensure
+the result can be later processed by a external parser and that that
+parser will report messages or errors with the correct offsets.
+
+The following example uses a JSON document that has comments and trailing
+commas and converts it prior to unmarshalling to using the standard Go
+JSON library.
+
+```go
+
+data := `
+{
+  /* Dev Machine */
+  "dbInfo": {
+    "host": "localhost",
+    "port": 5432,          // use full email address
+    "username": "josh",
+    "password": "pass123", // use a hashed password
+  }
+  /* Only SMTP Allowed */
+  "emailInfo": {
+    "email": "josh@example.com",
+    "password": "pass123",
+    "smtp": "smpt.example.com",
+  }
+}
+`
+
+err := json.Unmarshal(pretty.Spec(data), &config)
+
+```
+
+

 ## Customized output

--- a/vendor/github.com/tidwall/pretty/pretty.go
+++ b/vendor/github.com/tidwall/pretty/pretty.go
@ -118,21 +118,27 @@ type pair struct {
 	vstart, vend int
 }

-type byKey struct {
+type byKeyVal struct {
 	sorted bool
 	json   []byte
 	pairs  []pair
 }

-func (arr *byKey) Len() int {
+func (arr *byKeyVal) Len() int {
 	return len(arr.pairs)
 }
-func (arr *byKey) Less(i, j int) bool {
+func (arr *byKeyVal) Less(i, j int) bool {
 	key1 := arr.json[arr.pairs[i].kstart+1 : arr.pairs[i].kend-1]
 	key2 := arr.json[arr.pairs[j].kstart+1 : arr.pairs[j].kend-1]
-	return string(key1) < string(key2)
+	if string(key1) < string(key2) {
+		return true
+	}
+	if string(key1) > string(key2) {
+		return false
+	}
+	return arr.pairs[i].vstart < arr.pairs[j].vstart
 }
-func (arr *byKey) Swap(i, j int) {
+func (arr *byKeyVal) Swap(i, j int) {
 	arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i]
 	arr.sorted = true
 }
@ -174,8 +180,12 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool,
 				}
 				if n > 0 {
 					nl = len(buf)
+					if buf[nl-1] == ' ' {
+						buf[nl-1] = '\n'
+					} else {
 						buf = append(buf, '\n')
 					}
+				}
 				if buf[len(buf)-1] != open {
 					buf = appendTabs(buf, prefix, indent, tabs)
 				}
@ -193,7 +203,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool,
 			var p pair
 			if pretty {
 				nl = len(buf)
+				if buf[nl-1] == ' ' {
+					buf[nl-1] = '\n'
+				} else {
 					buf = append(buf, '\n')
+				}
 				if open == '{' && sortkeys {
 					p.kstart = i
 					p.vstart = len(buf)
@ -235,8 +249,8 @@ func sortPairs(json, buf []byte, pairs []pair) []byte {
 	}
 	vstart := pairs[0].vstart
 	vend := pairs[len(pairs)-1].vend
-	arr := byKey{false, json, pairs}
-	sort.Sort(&arr)
+	arr := byKeyVal{false, json, pairs}
+	sort.Stable(&arr)
 	if !arr.sorted {
 		return buf
 	}
@ -305,6 +319,7 @@ func appendTabs(buf []byte, prefix, indent string, tabs int) []byte {
 type Style struct {
 	Key, String, Number [2]string
 	True, False, Null   [2]string
+	Escape              [2]string
 	Append              func(dst []byte, c byte) []byte
 }

@ -328,6 +343,7 @@ func init() {
 		True:   [2]string{"\x1B[96m", "\x1B[0m"},
 		False:  [2]string{"\x1B[96m", "\x1B[0m"},
 		Null:   [2]string{"\x1B[91m", "\x1B[0m"},
+		Escape: [2]string{"\x1B[35m", "\x1B[0m"},
 		Append: func(dst []byte, c byte) []byte {
 			if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
 				dst = append(dst, "\\u00"...)
@ -367,8 +383,39 @@ func Color(src []byte, style *Style) []byte {
 				dst = append(dst, style.String[0]...)
 			}
 			dst = apnd(dst, '"')
+			esc := false
+			uesc := 0
 			for i = i + 1; i < len(src); i++ {
+				if src[i] == '\\' {
+					if key {
+						dst = append(dst, style.Key[1]...)
+					} else {
+						dst = append(dst, style.String[1]...)
+					}
+					dst = append(dst, style.Escape[0]...)
 					dst = apnd(dst, src[i])
+					esc = true
+					if i+1 < len(src) && src[i+1] == 'u' {
+						uesc = 5
+					} else {
+						uesc = 1
+					}
+				} else if esc {
+					dst = apnd(dst, src[i])
+					if uesc == 1 {
+						esc = false
+						dst = append(dst, style.Escape[1]...)
+						if key {
+							dst = append(dst, style.Key[0]...)
+						} else {
+							dst = append(dst, style.String[0]...)
+						}
+					} else {
+						uesc--
+					}
+				} else {
+					dst = apnd(dst, src[i])
+				}
 				if src[i] == '"' {
 					j := i - 1
 					for ; ; j-- {
@ -381,7 +428,9 @@ func Color(src []byte, style *Style) []byte {
 					}
 				}
 			}
-			if key {
+			if esc {
+				dst = append(dst, style.Escape[1]...)
+			} else if key {
 				dst = append(dst, style.Key[1]...)
 			} else {
 				dst = append(dst, style.String[1]...)
@ -434,3 +483,90 @@ func Color(src []byte, style *Style) []byte {
 	}
 	return dst
 }
+
+// Spec strips out comments and trailing commas and convert the input to a
+// valid JSON per the official spec: https://tools.ietf.org/html/rfc8259
+//
+// The resulting JSON will always be the same length as the input and it will
+// include all of the same line breaks at matching offsets. This is to ensure
+// the result can be later processed by a external parser and that that
+// parser will report messages or errors with the correct offsets.
+func Spec(src []byte) []byte {
+	return spec(src, nil)
+}
+
+// SpecInPlace is the same as Spec, but this method reuses the input json
+// buffer to avoid allocations. Do not use the original bytes slice upon return.
+func SpecInPlace(src []byte) []byte {
+	return spec(src, src)
+}
+
+func spec(src, dst []byte) []byte {
+	dst = dst[:0]
+	for i := 0; i < len(src); i++ {
+		if src[i] == '/' {
+			if i < len(src)-1 {
+				if src[i+1] == '/' {
+					dst = append(dst, ' ', ' ')
+					i += 2
+					for ; i < len(src); i++ {
+						if src[i] == '\n' {
+							dst = append(dst, '\n')
+							break
+						} else if src[i] == '\t' || src[i] == '\r' {
+							dst = append(dst, src[i])
+						} else {
+							dst = append(dst, ' ')
+						}
+					}
+					continue
+				}
+				if src[i+1] == '*' {
+					dst = append(dst, ' ', ' ')
+					i += 2
+					for ; i < len(src)-1; i++ {
+						if src[i] == '*' && src[i+1] == '/' {
+							dst = append(dst, ' ', ' ')
+							i++
+							break
+						} else if src[i] == '\n' || src[i] == '\t' ||
+							src[i] == '\r' {
+							dst = append(dst, src[i])
+						} else {
+							dst = append(dst, ' ')
+						}
+					}
+					continue
+				}
+			}
+		}
+		dst = append(dst, src[i])
+		if src[i] == '"' {
+			for i = i + 1; i < len(src); i++ {
+				dst = append(dst, src[i])
+				if src[i] == '"' {
+					j := i - 1
+					for ; ; j-- {
+						if src[j] != '\\' {
+							break
+						}
+					}
+					if (j-i)%2 != 0 {
+						break
+					}
+				}
+			}
+		} else if src[i] == '}' || src[i] == ']' {
+			for j := len(dst) - 2; j >= 0; j-- {
+				if dst[j] <= ' ' {
+					continue
+				}
+				if dst[j] == ',' {
+					dst[j] = ' '
+				}
+				break
+			}
+		}
+	}
+	return dst
+}
--- a/vendor/github.com/tidwall/rtred/LICENSE
+++ b/vendor/github.com/tidwall/rtred/LICENSE
--- a/vendor/github.com/tidwall/rtred/README.md
+++ b/vendor/github.com/tidwall/rtred/README.md
@ -1,7 +1,6 @@
 RTree implementation for Go
 ===========================

-[![Build Status](https://travis-ci.org/tidwall/rtree.svg?branch=master)](https://travis-ci.org/tidwall/rtree)
 [![GoDoc](https://godoc.org/github.com/tidwall/rtree?status.svg)](https://godoc.org/github.com/tidwall/rtree)

 This package provides an in-memory R-Tree implementation for Go, useful as a spatial data structure.
--- a/vendor/github.com/tidwall/rtred/base/knn.go
+++ b/vendor/github.com/tidwall/rtred/base/knn.go
--- a/vendor/github.com/tidwall/rtred/base/load.go
+++ b/vendor/github.com/tidwall/rtred/base/load.go
--- a/vendor/github.com/tidwall/rtred/base/rtree.go
+++ b/vendor/github.com/tidwall/rtred/base/rtree.go
--- a/vendor/github.com/tidwall/rtred/go.mod
+++ b/vendor/github.com/tidwall/rtred/go.mod
@ -0,0 +1,5 @@
+module github.com/tidwall/rtred
+
+go 1.15
+
+require github.com/tidwall/tinyqueue v0.1.1
--- a/vendor/github.com/tidwall/rtred/go.sum
+++ b/vendor/github.com/tidwall/rtred/go.sum
@ -0,0 +1,2 @@
+github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
+github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
--- a/vendor/github.com/tidwall/rtred/rtree.go
+++ b/vendor/github.com/tidwall/rtred/rtree.go
@ -1,10 +1,10 @@
-package rtree
+package rtred

 import (
 	"math"
 	"sync"

-	"github.com/tidwall/rtree/base"
+	"github.com/tidwall/rtred/base"
 )

 type Iterator func(item Item) bool
--- a/vendor/github.com/tidwall/rtree/.travis.yml
+++ b/vendor/github.com/tidwall/rtree/.travis.yml
@ -1 +0,0 @@
-language: go
--- a/vendor/github.com/tidwall/tinyqueue/go.mod
+++ b/vendor/github.com/tidwall/tinyqueue/go.mod
@ -0,0 +1,3 @@
+module github.com/tidwall/tinyqueue
+
+go 1.15
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@ -39,23 +39,25 @@ github.com/oragono/confusables
 github.com/oragono/go-ident
 # github.com/stretchr/testify v1.4.0
 ## explicit
-# github.com/tidwall/btree v0.2.2
+# github.com/tidwall/btree v0.4.2
 github.com/tidwall/btree
-# github.com/tidwall/buntdb v1.1.4
+# github.com/tidwall/buntdb v1.2.3
 ## explicit
 github.com/tidwall/buntdb
-# github.com/tidwall/gjson v1.6.1
+# github.com/tidwall/gjson v1.7.4
 github.com/tidwall/gjson
-# github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb
+# github.com/tidwall/grect v0.1.1
 github.com/tidwall/grect
-# github.com/tidwall/match v1.0.1
+# github.com/tidwall/match v1.0.3
 github.com/tidwall/match
-# github.com/tidwall/pretty v1.0.2
+# github.com/tidwall/pretty v1.1.0
 github.com/tidwall/pretty
+# github.com/tidwall/rtred v0.1.2
+github.com/tidwall/rtred
+github.com/tidwall/rtred/base
 # github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8
-github.com/tidwall/rtree
-github.com/tidwall/rtree/base
-# github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563
+## explicit
+# github.com/tidwall/tinyqueue v0.1.1
 github.com/tidwall/tinyqueue
 # github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208
 ## explicit