From b022c34a2316d60991bab2e8100aef16314637ee Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Wed, 31 Mar 2021 07:06:58 -0400 Subject: [PATCH 1/2] try to record buntdb errors from persisting lastSeen See #1603 --- irc/accounts.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/irc/accounts.go b/irc/accounts.go index 413e1780..cfa12977 100644 --- a/irc/accounts.go +++ b/irc/accounts.go @@ -614,7 +614,7 @@ func (am *AccountManager) saveLastSeen(account string, lastSeen map[string]time. text, _ := json.Marshal(lastSeen) val = string(text) } - am.server.store.Update(func(tx *buntdb.Tx) error { + err := am.server.store.Update(func(tx *buntdb.Tx) error { if val != "" { tx.Set(key, val, nil) } else { @@ -622,6 +622,9 @@ func (am *AccountManager) saveLastSeen(account string, lastSeen map[string]time. } return nil }) + if err != nil { + am.server.logger.Error("internal", "error persisting lastSeen", account, err.Error()) + } } func (am *AccountManager) loadLastSeen(account string) (lastSeen map[string]time.Time) { From fd3cbab6eee260a0889d55c6e311701c827a41f7 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Thu, 1 Apr 2021 20:45:15 -0400 Subject: [PATCH 2/2] bump buntdb to v1.2.3 Potentially fixes the database corruption seen on #1603 --- go.mod | 3 +- go.sum | 17 ++ vendor/github.com/tidwall/btree/.travis.yml | 1 - vendor/github.com/tidwall/btree/README.md | 198 ++++++++++++- vendor/github.com/tidwall/btree/btree.go | 260 ++++++++++++------ vendor/github.com/tidwall/buntdb/.travis.yml | 4 - vendor/github.com/tidwall/buntdb/README.md | 5 +- vendor/github.com/tidwall/buntdb/buntdb.go | 142 ++++++---- vendor/github.com/tidwall/buntdb/go.mod | 14 +- vendor/github.com/tidwall/buntdb/go.sum | 30 +- vendor/github.com/tidwall/gjson/.travis.yml | 1 - vendor/github.com/tidwall/gjson/README.md | 17 +- vendor/github.com/tidwall/gjson/SYNTAX.md | 17 +- vendor/github.com/tidwall/gjson/gjson.go | 220 +++++++++------ vendor/github.com/tidwall/gjson/go.mod | 4 +- vendor/github.com/tidwall/gjson/go.sum | 8 +- vendor/github.com/tidwall/grect/go.mod | 5 + vendor/github.com/tidwall/grect/go.sum | 6 + vendor/github.com/tidwall/match/.travis.yml | 1 - vendor/github.com/tidwall/match/README.md | 21 +- vendor/github.com/tidwall/match/go.mod | 3 + vendor/github.com/tidwall/match/match.go | 21 +- vendor/github.com/tidwall/pretty/.travis.yml | 1 - vendor/github.com/tidwall/pretty/README.md | 43 ++- vendor/github.com/tidwall/pretty/pretty.go | 158 ++++++++++- .../tidwall/{rtree => rtred}/LICENSE | 0 .../tidwall/{rtree => rtred}/README.md | 1 - .../tidwall/{rtree => rtred}/base/knn.go | 0 .../tidwall/{rtree => rtred}/base/load.go | 0 .../tidwall/{rtree => rtred}/base/rtree.go | 0 vendor/github.com/tidwall/rtred/go.mod | 5 + vendor/github.com/tidwall/rtred/go.sum | 2 + .../tidwall/{rtree => rtred}/rtree.go | 4 +- vendor/github.com/tidwall/rtree/.travis.yml | 1 - vendor/github.com/tidwall/tinyqueue/go.mod | 3 + vendor/modules.txt | 20 +- 36 files changed, 912 insertions(+), 324 deletions(-) delete mode 100644 vendor/github.com/tidwall/btree/.travis.yml delete mode 100644 vendor/github.com/tidwall/buntdb/.travis.yml delete mode 100644 vendor/github.com/tidwall/gjson/.travis.yml create mode 100644 vendor/github.com/tidwall/grect/go.mod create mode 100644 vendor/github.com/tidwall/grect/go.sum delete mode 100644 vendor/github.com/tidwall/match/.travis.yml create mode 100644 vendor/github.com/tidwall/match/go.mod delete mode 100644 vendor/github.com/tidwall/pretty/.travis.yml rename vendor/github.com/tidwall/{rtree => rtred}/LICENSE (100%) rename vendor/github.com/tidwall/{rtree => rtred}/README.md (88%) rename vendor/github.com/tidwall/{rtree => rtred}/base/knn.go (100%) rename vendor/github.com/tidwall/{rtree => rtred}/base/load.go (100%) rename vendor/github.com/tidwall/{rtree => rtred}/base/rtree.go (100%) create mode 100644 vendor/github.com/tidwall/rtred/go.mod create mode 100644 vendor/github.com/tidwall/rtred/go.sum rename vendor/github.com/tidwall/{rtree => rtred}/rtree.go (99%) delete mode 100644 vendor/github.com/tidwall/rtree/.travis.yml create mode 100644 vendor/github.com/tidwall/tinyqueue/go.mod diff --git a/go.mod b/go.mod index 2141c4bb..15bbb505 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,8 @@ require ( github.com/oragono/confusables v0.0.0-20201108231250-4ab98ab61fb1 github.com/oragono/go-ident v0.0.0-20200511222032-830550b1d775 github.com/stretchr/testify v1.4.0 // indirect - github.com/tidwall/buntdb v1.1.4 + github.com/tidwall/buntdb v1.2.3 + github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 // indirect github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf // indirect diff --git a/go.sum b/go.sum index 3d0d6ced..1109e4bb 100644 --- a/go.sum +++ b/go.sum @@ -67,20 +67,37 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY= github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= +github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs= +github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= github.com/tidwall/buntdb v1.1.4 h1:W7y9+2dM3GOswU0t3pz6+BcwZXjj/tVOhPcO6EHufME= github.com/tidwall/buntdb v1.1.4/go.mod h1:06+/n7EFf6uUaIG5r9xZcExYN3H0Lnc+g/Kqx0fZFkI= +github.com/tidwall/buntdb v1.2.3 h1:AoGVe4yrhKmnEPHrPrW5EUOATHOCIk4VtFvd8xn/ZtU= +github.com/tidwall/buntdb v1.2.3/go.mod h1:+i/gBwYOHWG19wLgwMXFLkl00twh9+VWkkaOhuNQ4PA= github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= +github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8= +github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE= github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M= +github.com/tidwall/grect v0.1.1 h1:+kMEkxhoqB7rniVXzMEIA66XwU07STgINqxh+qVIndY= +github.com/tidwall/grect v0.1.1/go.mod h1:CzvbGiFbWUwiJ1JohXLb28McpyBsI00TK9Y6pDWLGRQ= +github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= +github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8= +github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= +github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8SutN9KQHc6vqHEXJTcAFwyGNius= github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao= github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE= github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ= +github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= +github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM= github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/vendor/github.com/tidwall/btree/.travis.yml b/vendor/github.com/tidwall/btree/.travis.yml deleted file mode 100644 index 4f2ee4d9..00000000 --- a/vendor/github.com/tidwall/btree/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/tidwall/btree/README.md b/vendor/github.com/tidwall/btree/README.md index 10b55297..aa2c53a0 100644 --- a/vendor/github.com/tidwall/btree/README.md +++ b/vendor/github.com/tidwall/btree/README.md @@ -1,5 +1,199 @@ -# B-tree for Go +# btree -![Travis CI Build Status](https://api.travis-ci.org/tidwall/btree.svg?branch=master) [![GoDoc](https://godoc.org/github.com/tidwall/btree?status.svg)](https://godoc.org/github.com/tidwall/btree) +An [efficient](#performance) [B-tree](https://en.wikipedia.org/wiki/B-tree) implementation in Go. + +## Features + +- `Copy()` method with copy-on-write support. +- Fast bulk loading for pre-ordered data using the `Load()` method. +- All operations are thread-safe. +- Path hinting optimization for operations with nearby keys. + +## Installing + +To start using btree, install Go and run `go get`: + +```sh +$ go get -u github.com/tidwall/btree +``` + +## Usage + +```go +package main + +import ( + "fmt" + + "github.com/tidwall/btree" +) + +type Item struct { + Key, Val string +} + +// byKeys is a comparison function that compares item keys and returns true +// when a is less than b. +func byKeys(a, b interface{}) bool { + i1, i2 := a.(*Item), b.(*Item) + return i1.Key < i2.Key +} + +// byVals is a comparison function that compares item values and returns true +// when a is less than b. +func byVals(a, b interface{}) bool { + i1, i2 := a.(*Item), b.(*Item) + if i1.Val < i2.Val { + return true + } + if i1.Val > i2.Val { + return false + } + // Both vals are equal so we should fall though + // and let the key comparison take over. + return byKeys(a, b) +} + +func main() { + // Create a tree for keys and a tree for values. + // The "keys" tree will be sorted on the Keys field. + // The "values" tree will be sorted on the Values field. + keys := btree.New(byKeys) + vals := btree.New(byVals) + + // Create some items. + users := []*Item{ + &Item{Key: "user:1", Val: "Jane"}, + &Item{Key: "user:2", Val: "Andy"}, + &Item{Key: "user:3", Val: "Steve"}, + &Item{Key: "user:4", Val: "Andrea"}, + &Item{Key: "user:5", Val: "Janet"}, + &Item{Key: "user:6", Val: "Andy"}, + } + + // Insert each user into both trees + for _, user := range users { + keys.Set(user) + vals.Set(user) + } + + // Iterate over each user in the key tree + keys.Ascend(nil, func(item interface{}) bool { + kvi := item.(*Item) + fmt.Printf("%s %s\n", kvi.Key, kvi.Val) + return true + }) + + fmt.Printf("\n") + // Iterate over each user in the val tree + vals.Ascend(nil, func(item interface{}) bool { + kvi := item.(*Item) + fmt.Printf("%s %s\n", kvi.Key, kvi.Val) + return true + }) + + // Output: + // user:1 Jane + // user:2 Andy + // user:3 Steve + // user:4 Andrea + // user:5 Janet + // user:6 Andy + // + // user:4 Andrea + // user:2 Andy + // user:6 Andy + // user:1 Jane + // user:5 Janet + // user:3 Steve +} +``` + +## Operations + +### Basic + +``` +Len() # return the number of items in the btree +Set(item) # insert or replace an existing item +Get(item) # get an existing item +Delete(item) # delete an item +``` + +### Iteration + +``` +Ascend(pivot, iter) # scan items in ascending order starting at pivot. +Descend(pivot, iter) # scan items in descending order starting at pivot. +``` + +### Queues + +``` +Min() # return the first item in the btree +Max() # return the last item in the btree +PopMin() # remove and return the first item in the btree +PopMax() # remove and return the last item in the btree +``` +### Bulk loading + +``` +Load(item) # load presorted items into tree +``` + +### Path hints + +``` +SetHint(item, *hint) # insert or replace an existing item +GetHint(item, *hint) # get an existing item +DeleteHint(item, *hint) # delete an item +``` + +## Performance + +This implementation was designed with performance in mind. + +The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel Core i9) using Go 1.15.3. The items are simple 8-byte ints. + +- `google`: The [google/btree](https://github.com/google/btree) package +- `tidwall`: The [tidwall/btree](https://github.com/tidwall/btree) package +- `go-arr`: Just a simple Go array + +``` +** sequential set ** +google: set-seq 1,000,000 ops in 160ms, 6,262,097/sec, 159 ns/op, 31.0 MB, 32 bytes/op +tidwall: set-seq 1,000,000 ops in 142ms, 7,020,721/sec, 142 ns/op, 36.6 MB, 38 bytes/op +tidwall: set-seq-hint 1,000,000 ops in 87ms, 11,503,315/sec, 86 ns/op, 36.6 MB, 38 bytes/op +tidwall: load-seq 1,000,000 ops in 37ms, 27,177,242/sec, 36 ns/op, 36.6 MB, 38 bytes/op +go-arr: append 1,000,000 ops in 49ms, 20,574,760/sec, 48 ns/op + +** random set ** +google: set-rand 1,000,000 ops in 606ms, 1,649,921/sec, 606 ns/op, 21.5 MB, 22 bytes/op +tidwall: set-rand 1,000,000 ops in 543ms, 1,841,590/sec, 543 ns/op, 26.7 MB, 27 bytes/op +tidwall: set-rand-hint 1,000,000 ops in 573ms, 1,745,624/sec, 572 ns/op, 26.4 MB, 27 bytes/op +tidwall: set-again 1,000,000 ops in 452ms, 2,212,581/sec, 451 ns/op, 27.1 MB, 28 bytes/op +tidwall: set-after-copy 1,000,000 ops in 472ms, 2,117,457/sec, 472 ns/op, 27.9 MB, 29 bytes/op +tidwall: load-rand 1,000,000 ops in 551ms, 1,816,498/sec, 550 ns/op, 26.1 MB, 27 bytes/op + +** sequential get ** +google: get-seq 1,000,000 ops in 133ms, 7,497,604/sec, 133 ns/op +tidwall: get-seq 1,000,000 ops in 110ms, 9,082,972/sec, 110 ns/op +tidwall: get-seq-hint 1,000,000 ops in 55ms, 18,289,945/sec, 54 ns/op + +** random get ** +google: get-rand 1,000,000 ops in 149ms, 6,704,337/sec, 149 ns/op +tidwall: get-rand 1,000,000 ops in 131ms, 7,616,296/sec, 131 ns/op +tidwall: get-rand-hint 1,000,000 ops in 216ms, 4,632,532/sec, 215 ns/op +``` + +*You can find the benchmark utility at [tidwall/btree-benchmark](https://github.com/tidwall/btree-benchmark)* + +## Contact + +Josh Baker [@tidwall](http://twitter.com/tidwall) + +## License + +Source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/btree/btree.go b/vendor/github.com/tidwall/btree/btree.go index 8e38bb06..9a2f2968 100644 --- a/vendor/github.com/tidwall/btree/btree.go +++ b/vendor/github.com/tidwall/btree/btree.go @@ -4,35 +4,39 @@ package btree +import "sync" + const maxItems = 255 const minItems = maxItems * 40 / 100 +type cow struct { + _ int // it cannot be an empty struct +} + type node struct { + cow *cow leaf bool numItems int16 items [maxItems]interface{} children *[maxItems + 1]*node } -type justaLeaf struct { - leaf bool - numItems int16 - items [maxItems]interface{} -} - // BTree is an ordered set items type BTree struct { + mu *sync.RWMutex + cow *cow root *node length int less func(a, b interface{}) bool lnode *node } -func newNode(leaf bool) *node { +func (tr *BTree) newNode(leaf bool) *node { n := &node{leaf: leaf} if !leaf { n.children = new([maxItems + 1]*node) } + n.cow = tr.cow return n } @@ -48,6 +52,7 @@ func New(less func(a, b interface{}) bool) *BTree { panic("nil less") } tr := new(BTree) + tr.mu = new(sync.RWMutex) tr.less = less return tr } @@ -85,8 +90,7 @@ func (n *node) find(key interface{}, less func(a, b interface{}) bool, high = mid - 1 } } - if low > 0 && !less(n.items[low-1], key) && - !less(key, n.items[low-1]) { + if low > 0 && !less(n.items[low-1], key) { index = low - 1 found = true } else { @@ -109,22 +113,29 @@ func (tr *BTree) SetHint(item interface{}, hint *PathHint) (prev interface{}) { if item == nil { panic("nil item") } + tr.mu.Lock() + prev = tr.setHint(item, hint) + tr.mu.Unlock() + return prev +} + +func (tr *BTree) setHint(item interface{}, hint *PathHint) (prev interface{}) { if tr.root == nil { - tr.root = newNode(true) + tr.root = tr.newNode(true) tr.root.items[0] = item tr.root.numItems = 1 tr.length = 1 return } - prev = tr.root.set(item, tr.less, hint, 0) + prev = tr.nodeSet(&tr.root, item, tr.less, hint, 0) if prev != nil { return prev } tr.lnode = nil if tr.root.numItems == maxItems { - n := tr.root - right, median := n.split() - tr.root = newNode(false) + n := tr.cowLoad(&tr.root) + right, median := tr.nodeSplit(n) + tr.root = tr.newNode(false) tr.root.children[0] = n tr.root.items[0] = median tr.root.children[1] = right @@ -139,8 +150,8 @@ func (tr *BTree) Set(item interface{}) (prev interface{}) { return tr.SetHint(item, nil) } -func (n *node) split() (right *node, median interface{}) { - right = newNode(n.leaf) +func (tr *BTree) nodeSplit(n *node) (right *node, median interface{}) { + right = tr.newNode(n.leaf) median = n.items[maxItems/2] copy(right.items[:maxItems/2], n.items[maxItems/2+1:]) if !n.leaf { @@ -159,9 +170,30 @@ func (n *node) split() (right *node, median interface{}) { return right, median } -func (n *node) set(item interface{}, less func(a, b interface{}) bool, - hint *PathHint, depth int, +//go:noinline +func (tr *BTree) copy(n *node) *node { + n2 := *n + n2.cow = tr.cow + copy(n2.items[:], n.items[:]) + if n.children != nil { + n2.children = new([maxItems + 1]*node) + copy(n2.children[:], n.children[:]) + } + return &n2 +} + +// cowLoad loaded the provide node and, if needed, performs a copy-on-write. +func (tr *BTree) cowLoad(cn **node) *node { + if (*cn).cow != tr.cow { + *cn = tr.copy(*cn) + } + return *cn +} + +func (tr *BTree) nodeSet(cn **node, item interface{}, + less func(a, b interface{}) bool, hint *PathHint, depth int, ) (prev interface{}) { + n := tr.cowLoad(cn) i, found := n.find(item, less, hint, depth) if found { prev = n.items[i] @@ -174,12 +206,12 @@ func (n *node) set(item interface{}, less func(a, b interface{}) bool, n.numItems++ return nil } - prev = n.children[i].set(item, less, hint, depth+1) + prev = tr.nodeSet(&n.children[i], item, less, hint, depth+1) if prev != nil { return prev } if n.children[i].numItems == maxItems { - right, median := n.children[i].split() + right, median := tr.nodeSplit(n.children[i]) copy(n.children[i+1:], n.children[i:]) copy(n.items[i+1:], n.items[i:]) n.items[i] = median @@ -216,6 +248,8 @@ func (tr *BTree) Get(key interface{}) interface{} { // GetHint gets a value for key using a path hint func (tr *BTree) GetHint(key interface{}, hint *PathHint) interface{} { + tr.mu.RLock() + defer tr.mu.RUnlock() if tr.root == nil || key == nil { return nil } @@ -246,10 +280,17 @@ func (tr *BTree) Delete(key interface{}) interface{} { // DeleteHint deletes a value for a key using a path hint func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} { + tr.mu.Lock() + prev := tr.deleteHint(key, hint) + tr.mu.Unlock() + return prev +} + +func (tr *BTree) deleteHint(key interface{}, hint *PathHint) interface{} { if tr.root == nil || key == nil { return nil } - prev := tr.root.delete(false, key, tr.less, hint, 0) + prev := tr.delete(&tr.root, false, key, tr.less, hint, 0) if prev == nil { return nil } @@ -264,9 +305,10 @@ func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} { return prev } -func (n *node) delete(max bool, key interface{}, +func (tr *BTree) delete(cn **node, max bool, key interface{}, less func(a, b interface{}) bool, hint *PathHint, depth int, ) interface{} { + n := tr.cowLoad(cn) var i int16 var found bool if max { @@ -290,74 +332,79 @@ func (n *node) delete(max bool, key interface{}, if found { if max { i++ - prev = n.children[i].delete(true, "", less, nil, 0) + prev = tr.delete(&n.children[i], true, "", less, nil, 0) } else { prev = n.items[i] - maxItem := n.children[i].delete(true, "", less, nil, 0) + maxItem := tr.delete(&n.children[i], true, "", less, nil, 0) n.items[i] = maxItem } } else { - prev = n.children[i].delete(max, key, less, hint, depth+1) + prev = tr.delete(&n.children[i], max, key, less, hint, depth+1) } if prev == nil { return nil } - if n.children[i].numItems < minItems { - if i == n.numItems { - i-- + if n.children[i].numItems >= minItems { + return prev + } + + // merge / rebalance nodes + if i == n.numItems { + i-- + } + n.children[i] = tr.cowLoad(&n.children[i]) + n.children[i+1] = tr.cowLoad(&n.children[i+1]) + if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems { + // merge left + item + right + n.children[i].items[n.children[i].numItems] = n.items[i] + copy(n.children[i].items[n.children[i].numItems+1:], + n.children[i+1].items[:n.children[i+1].numItems]) + if !n.children[0].leaf { + copy(n.children[i].children[n.children[i].numItems+1:], + n.children[i+1].children[:n.children[i+1].numItems+1]) } - if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems { - // merge left + item + right - n.children[i].items[n.children[i].numItems] = n.items[i] - copy(n.children[i].items[n.children[i].numItems+1:], - n.children[i+1].items[:n.children[i+1].numItems]) - if !n.children[0].leaf { - copy(n.children[i].children[n.children[i].numItems+1:], - n.children[i+1].children[:n.children[i+1].numItems+1]) - } - n.children[i].numItems += n.children[i+1].numItems + 1 - copy(n.items[i:], n.items[i+1:n.numItems]) - copy(n.children[i+1:], n.children[i+2:n.numItems+1]) - n.items[n.numItems] = nil - n.children[n.numItems+1] = nil - n.numItems-- - } else if n.children[i].numItems > n.children[i+1].numItems { - // move left -> right - copy(n.children[i+1].items[1:], - n.children[i+1].items[:n.children[i+1].numItems]) - if !n.children[0].leaf { - copy(n.children[i+1].children[1:], - n.children[i+1].children[:n.children[i+1].numItems+1]) - } - n.children[i+1].items[0] = n.items[i] - if !n.children[0].leaf { - n.children[i+1].children[0] = - n.children[i].children[n.children[i].numItems] - } - n.children[i+1].numItems++ - n.items[i] = n.children[i].items[n.children[i].numItems-1] - n.children[i].items[n.children[i].numItems-1] = nil - if !n.children[0].leaf { - n.children[i].children[n.children[i].numItems] = nil - } - n.children[i].numItems-- - } else { - // move right -> left - n.children[i].items[n.children[i].numItems] = n.items[i] - if !n.children[0].leaf { - n.children[i].children[n.children[i].numItems+1] = - n.children[i+1].children[0] - } - n.children[i].numItems++ - n.items[i] = n.children[i+1].items[0] - copy(n.children[i+1].items[:], - n.children[i+1].items[1:n.children[i+1].numItems]) - if !n.children[0].leaf { - copy(n.children[i+1].children[:], - n.children[i+1].children[1:n.children[i+1].numItems+1]) - } - n.children[i+1].numItems-- + n.children[i].numItems += n.children[i+1].numItems + 1 + copy(n.items[i:], n.items[i+1:n.numItems]) + copy(n.children[i+1:], n.children[i+2:n.numItems+1]) + n.items[n.numItems] = nil + n.children[n.numItems+1] = nil + n.numItems-- + } else if n.children[i].numItems > n.children[i+1].numItems { + // move left -> right + copy(n.children[i+1].items[1:], + n.children[i+1].items[:n.children[i+1].numItems]) + if !n.children[0].leaf { + copy(n.children[i+1].children[1:], + n.children[i+1].children[:n.children[i+1].numItems+1]) } + n.children[i+1].items[0] = n.items[i] + if !n.children[0].leaf { + n.children[i+1].children[0] = + n.children[i].children[n.children[i].numItems] + } + n.children[i+1].numItems++ + n.items[i] = n.children[i].items[n.children[i].numItems-1] + n.children[i].items[n.children[i].numItems-1] = nil + if !n.children[0].leaf { + n.children[i].children[n.children[i].numItems] = nil + } + n.children[i].numItems-- + } else { + // move right -> left + n.children[i].items[n.children[i].numItems] = n.items[i] + if !n.children[0].leaf { + n.children[i].children[n.children[i].numItems+1] = + n.children[i+1].children[0] + } + n.children[i].numItems++ + n.items[i] = n.children[i+1].items[0] + copy(n.children[i+1].items[:], + n.children[i+1].items[1:n.children[i+1].numItems]) + if !n.children[0].leaf { + copy(n.children[i+1].children[:], + n.children[i+1].children[1:n.children[i+1].numItems+1]) + } + n.children[i+1].numItems-- } return prev } @@ -366,6 +413,8 @@ func (n *node) delete(max bool, key interface{}, // Pass nil for pivot to scan all item in ascending order // Return false to stop iterating func (tr *BTree) Ascend(pivot interface{}, iter func(item interface{}) bool) { + tr.mu.RLock() + defer tr.mu.RUnlock() if tr.root == nil { return } @@ -427,6 +476,8 @@ func (n *node) reverse(iter func(item interface{}) bool) bool { // Pass nil for pivot to scan all item in descending order // Return false to stop iterating func (tr *BTree) Descend(pivot interface{}, iter func(item interface{}) bool) { + tr.mu.RLock() + defer tr.mu.RUnlock() if tr.root == nil { return } @@ -467,6 +518,12 @@ func (tr *BTree) Load(item interface{}) interface{} { if item == nil { panic("nil item") } + tr.mu.Lock() + defer tr.mu.Unlock() + + // Load does not need a cowGrid because the Copy operation sets the + // lnode to nil. + if tr.lnode != nil && tr.lnode.numItems < maxItems-2 { if tr.less(tr.lnode.items[tr.lnode.numItems-1], item) { tr.lnode.items[tr.lnode.numItems] = item @@ -475,7 +532,7 @@ func (tr *BTree) Load(item interface{}) interface{} { return nil } } - prev := tr.Set(item) + prev := tr.setHint(item, nil) if prev != nil { return prev } @@ -493,6 +550,8 @@ func (tr *BTree) Load(item interface{}) interface{} { // Min returns the minimum item in tree. // Returns nil if the tree has no items. func (tr *BTree) Min() interface{} { + tr.mu.RLock() + defer tr.mu.RUnlock() if tr.root == nil { return nil } @@ -508,6 +567,8 @@ func (tr *BTree) Min() interface{} { // Max returns the maximum item in tree. // Returns nil if the tree has no items. func (tr *BTree) Max() interface{} { + tr.mu.RLock() + defer tr.mu.RUnlock() if tr.root == nil { return nil } @@ -523,53 +584,65 @@ func (tr *BTree) Max() interface{} { // PopMin removes the minimum item in tree and returns it. // Returns nil if the tree has no items. func (tr *BTree) PopMin() interface{} { + tr.mu.Lock() + defer tr.mu.Unlock() if tr.root == nil { return nil } tr.lnode = nil - n := tr.root + n := tr.cowLoad(&tr.root) for { if n.leaf { item := n.items[0] if n.numItems == minItems { - return tr.Delete(item) + return tr.deleteHint(item, nil) } copy(n.items[:], n.items[1:]) n.items[n.numItems-1] = nil n.numItems-- tr.length-- + if tr.length == 0 { + tr.root = nil + } return item } - n = n.children[0] + n = tr.cowLoad(&n.children[0]) } } // PopMax removes the minimum item in tree and returns it. // Returns nil if the tree has no items. func (tr *BTree) PopMax() interface{} { + tr.mu.Lock() + defer tr.mu.Unlock() if tr.root == nil { return nil } tr.lnode = nil - n := tr.root + n := tr.cowLoad(&tr.root) for { if n.leaf { item := n.items[n.numItems-1] if n.numItems == minItems { - return tr.Delete(item) + return tr.deleteHint(item, nil) } n.items[n.numItems-1] = nil n.numItems-- tr.length-- + if tr.length == 0 { + tr.root = nil + } return item } - n = n.children[n.numItems] + n = tr.cowLoad(&n.children[n.numItems]) } } // Height returns the height of the tree. // Returns zero if tree has no items. func (tr *BTree) Height() int { + tr.mu.RLock() + defer tr.mu.RUnlock() var height int if tr.root != nil { n := tr.root @@ -587,6 +660,8 @@ func (tr *BTree) Height() int { // Walk iterates over all items in tree, in order. // The items param will contain one or more items. func (tr *BTree) Walk(iter func(item []interface{})) { + tr.mu.RLock() + defer tr.mu.RUnlock() if tr.root != nil { tr.root.walk(iter) } @@ -603,3 +678,16 @@ func (n *node) walk(iter func(item []interface{})) { n.children[n.numItems].walk(iter) } } + +// Copy the tree. This operation is very fast because it only performs a +// shadowed copy. +func (tr *BTree) Copy() *BTree { + tr.mu.Lock() + tr.lnode = nil + tr.cow = new(cow) + tr2 := *tr + tr2.mu = new(sync.RWMutex) + tr2.cow = new(cow) + tr.mu.Unlock() + return &tr2 +} diff --git a/vendor/github.com/tidwall/buntdb/.travis.yml b/vendor/github.com/tidwall/buntdb/.travis.yml deleted file mode 100644 index 10f5fa48..00000000 --- a/vendor/github.com/tidwall/buntdb/.travis.yml +++ /dev/null @@ -1,4 +0,0 @@ -language: go - -go: - - 1.15.x diff --git a/vendor/github.com/tidwall/buntdb/README.md b/vendor/github.com/tidwall/buntdb/README.md index 88f6d19b..cdb199e1 100644 --- a/vendor/github.com/tidwall/buntdb/README.md +++ b/vendor/github.com/tidwall/buntdb/README.md @@ -3,10 +3,9 @@ src="logo.png" width="307" height="150" border="0" alt="BuntDB">
-Build Status -Code Coverage Go Report Card -GoDoc +Godoc +LICENSE

BuntDB is a low-level, in-memory, key/value store in pure Go. diff --git a/vendor/github.com/tidwall/buntdb/buntdb.go b/vendor/github.com/tidwall/buntdb/buntdb.go index ca2152eb..6e612a7b 100644 --- a/vendor/github.com/tidwall/buntdb/buntdb.go +++ b/vendor/github.com/tidwall/buntdb/buntdb.go @@ -19,7 +19,7 @@ import ( "github.com/tidwall/gjson" "github.com/tidwall/grect" "github.com/tidwall/match" - "github.com/tidwall/rtree" + "github.com/tidwall/rtred" ) var ( @@ -69,7 +69,6 @@ type DB struct { keys *btree.BTree // a tree of all item ordered by key exps *btree.BTree // a tree of items ordered by expiration idxs map[string]*index // the index trees. - exmgr bool // indicates that expires manager is running. flushes int // a count of the number of disk flushes closed bool // set when the database has been closed config Config // the database configuration @@ -135,9 +134,6 @@ type exctx struct { db *DB } -// Default number of btree degrees -const btreeDegrees = 64 - // Open opens a database at the provided path. // If the file does not exist then it will be created automatically. func Open(path string) (*DB, error) { @@ -241,14 +237,15 @@ func (db *DB) Load(rd io.Reader) error { // cannot load into databases that persist to disk return ErrPersistenceActive } - return db.readLoad(rd, time.Now()) + _, err := db.readLoad(rd, time.Now()) + return err } // index represents a b-tree or r-tree index and also acts as the // b-tree/r-tree context for itself. type index struct { btr *btree.BTree // contains the items - rtr *rtree.RTree // contains the items + rtr *rtred.RTree // contains the items name string // name of the index pattern string // a required key pattern less func(a, b string) bool // less comparison function @@ -289,7 +286,7 @@ func (idx *index) clearCopy() *index { nidx.btr = btree.New(lessCtx(nidx)) } if nidx.rect != nil { - nidx.rtr = rtree.New(nidx) + nidx.rtr = rtred.New(nidx) } return nidx } @@ -301,7 +298,7 @@ func (idx *index) rebuild() { idx.btr = btree.New(lessCtx(idx)) } if idx.rect != nil { - idx.rtr = rtree.New(idx) + idx.rtr = rtred.New(idx) } // iterate through all keys and fill the index btreeAscend(idx.db.keys, func(item interface{}) bool { @@ -755,46 +752,65 @@ func (db *DB) Shrink() error { }() } -var errValidEOF = errors.New("valid eof") - // readLoad reads from the reader and loads commands into the database. // modTime is the modified time of the reader, should be no greater than // the current time.Now(). -func (db *DB) readLoad(rd io.Reader, modTime time.Time) error { +// Returns the number of bytes of the last command read and the error if any. +func (db *DB) readLoad(rd io.Reader, modTime time.Time) (n int64, err error) { + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + totalSize := int64(0) data := make([]byte, 4096) parts := make([]string, 0, 8) r := bufio.NewReader(rd) for { + // peek at the first byte. If it's a 'nul' control character then + // ignore it and move to the next byte. + c, err := r.ReadByte() + if err != nil { + if err == io.EOF { + err = nil + } + return totalSize, err + } + if c == 0 { + // ignore nul control characters + n += 1 + continue + } + if err := r.UnreadByte(); err != nil { + return totalSize, err + } + // read a single command. // first we should read the number of parts that the of the command + cmdByteSize := int64(0) line, err := r.ReadBytes('\n') if err != nil { - if len(line) > 0 { - // got an eof but also data. this should be an unexpected eof. - return io.ErrUnexpectedEOF - } - if err == io.EOF { - break - } - return err + return totalSize, err } if line[0] != '*' { - return ErrInvalid + return totalSize, ErrInvalid } + cmdByteSize += int64(len(line)) + // convert the string number to and int var n int if len(line) == 4 && line[len(line)-2] == '\r' { if line[1] < '0' || line[1] > '9' { - return ErrInvalid + return totalSize, ErrInvalid } n = int(line[1] - '0') } else { if len(line) < 5 || line[len(line)-2] != '\r' { - return ErrInvalid + return totalSize, ErrInvalid } for i := 1; i < len(line)-2; i++ { if line[i] < '0' || line[i] > '9' { - return ErrInvalid + return totalSize, ErrInvalid } n = n*10 + int(line[i]-'0') } @@ -805,25 +821,26 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error { // read the number of bytes of the part. line, err := r.ReadBytes('\n') if err != nil { - return err + return totalSize, err } if line[0] != '$' { - return ErrInvalid + return totalSize, ErrInvalid } + cmdByteSize += int64(len(line)) // convert the string number to and int var n int if len(line) == 4 && line[len(line)-2] == '\r' { if line[1] < '0' || line[1] > '9' { - return ErrInvalid + return totalSize, ErrInvalid } n = int(line[1] - '0') } else { if len(line) < 5 || line[len(line)-2] != '\r' { - return ErrInvalid + return totalSize, ErrInvalid } for i := 1; i < len(line)-2; i++ { if line[i] < '0' || line[i] > '9' { - return ErrInvalid + return totalSize, ErrInvalid } n = n*10 + int(line[i]-'0') } @@ -837,13 +854,14 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error { data = make([]byte, dataln) } if _, err = io.ReadFull(r, data[:n+2]); err != nil { - return err + return totalSize, err } if data[n] != '\r' || data[n+1] != '\n' { - return ErrInvalid + return totalSize, ErrInvalid } // copy string parts = append(parts, string(data[:n])) + cmdByteSize += int64(n + 2) } // finished reading the command @@ -855,15 +873,15 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error { (parts[0][2] == 't' || parts[0][2] == 'T') { // SET if len(parts) < 3 || len(parts) == 4 || len(parts) > 5 { - return ErrInvalid + return totalSize, ErrInvalid } if len(parts) == 5 { if strings.ToLower(parts[3]) != "ex" { - return ErrInvalid + return totalSize, ErrInvalid } - ex, err := strconv.ParseInt(parts[4], 10, 64) + ex, err := strconv.ParseUint(parts[4], 10, 64) if err != nil { - return err + return totalSize, err } now := time.Now() dur := (time.Duration(ex) * time.Second) - now.Sub(modTime) @@ -885,7 +903,7 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error { (parts[0][2] == 'l' || parts[0][2] == 'L') { // DEL if len(parts) != 2 { - return ErrInvalid + return totalSize, ErrInvalid } db.deleteFromDatabase(&dbItem{key: parts[1]}) } else if (parts[0][0] == 'f' || parts[0][0] == 'F') && @@ -894,10 +912,10 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error { db.exps = btree.New(lessCtx(&exctx{db})) db.idxs = make(map[string]*index) } else { - return ErrInvalid + return totalSize, ErrInvalid } + totalSize += cmdByteSize } - return nil } // load reads entries from the append only database file and fills the database. @@ -910,10 +928,20 @@ func (db *DB) load() error { if err != nil { return err } - if err := db.readLoad(db.file, fi.ModTime()); err != nil { - return err + n, err := db.readLoad(db.file, fi.ModTime()) + if err != nil { + if err == io.ErrUnexpectedEOF { + // The db file has ended mid-command, which is allowed but the + // data file should be truncated to the end of the last valid + // command + if err := db.file.Truncate(n); err != nil { + return err + } + } else { + return err + } } - pos, err := db.file.Seek(0, 2) + pos, err := db.file.Seek(n, 0) if err != nil { return err } @@ -1148,7 +1176,25 @@ func (tx *Tx) Commit() error { // Flushing the buffer only once per transaction. // If this operation fails then the write did failed and we must // rollback. - if _, err = tx.db.file.Write(tx.db.buf); err != nil { + var n int + n, err = tx.db.file.Write(tx.db.buf) + if err != nil { + if n > 0 { + // There was a partial write to disk. + // We are possibly out of disk space. + // Delete the partially written bytes from the data file by + // seeking to the previously known position and performing + // a truncate operation. + // At this point a syscall failure is fatal and the process + // should be killed to avoid corrupting the file. + pos, err := tx.db.file.Seek(-int64(n), 1) + if err != nil { + panic(err) + } + if err := tx.db.file.Truncate(pos); err != nil { + panic(err) + } + } tx.rollbackInner() } if tx.db.config.SyncPolicy == Always { @@ -1216,7 +1262,7 @@ func appendBulkString(buf []byte, s string) []byte { // writeSetTo writes an item as a single SET record to the a bufio Writer. func (dbi *dbItem) writeSetTo(buf []byte) []byte { if dbi.opts != nil && dbi.opts.ex { - ex := dbi.opts.exat.Sub(time.Now()) / time.Second + ex := time.Until(dbi.opts.exat) / time.Second buf = appendArray(buf, 5) buf = appendBulkString(buf, "set") buf = appendBulkString(buf, dbi.key) @@ -1390,7 +1436,9 @@ func (tx *Tx) Set(key, value string, opts *SetOptions) (previousValue string, // create a rollback entry with a nil value. A nil value indicates // that the entry should be deleted on rollback. When the value is // *not* nil, that means the entry should be reverted. - tx.wc.rollbackItems[key] = nil + if _, ok := tx.wc.rollbackItems[key]; !ok { + tx.wc.rollbackItems[key] = nil + } } else { // A previous item already exists in the database. Let's create a // rollback entry with the item as the value. We need to check the @@ -1481,7 +1529,7 @@ func (tx *Tx) TTL(key string) (time.Duration, error) { } else if item.opts == nil || !item.opts.ex { return -1, nil } - dur := item.opts.exat.Sub(time.Now()) + dur := time.Until(item.opts.exat) if dur < 0 { return 0, ErrNotFound } @@ -1822,7 +1870,7 @@ func (tx *Tx) Nearby(index, bounds string, return nil } // // wrap a rtree specific iterator around the user-defined iterator. - iter := func(item rtree.Item, dist float64) bool { + iter := func(item rtred.Item, dist float64) bool { dbi := item.(*dbItem) return iterator(dbi.key, dbi.val, dist) } @@ -1860,7 +1908,7 @@ func (tx *Tx) Intersects(index, bounds string, return nil } // wrap a rtree specific iterator around the user-defined iterator. - iter := func(item rtree.Item) bool { + iter := func(item rtred.Item) bool { dbi := item.(*dbItem) return iterator(dbi.key, dbi.val) } diff --git a/vendor/github.com/tidwall/buntdb/go.mod b/vendor/github.com/tidwall/buntdb/go.mod index 225704cd..cd77790c 100644 --- a/vendor/github.com/tidwall/buntdb/go.mod +++ b/vendor/github.com/tidwall/buntdb/go.mod @@ -1,12 +1,12 @@ module github.com/tidwall/buntdb -go 1.15 +go 1.16 require ( - github.com/tidwall/btree v0.2.2 - github.com/tidwall/gjson v1.6.1 - github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb - github.com/tidwall/match v1.0.1 - github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 - github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect + github.com/tidwall/btree v0.4.2 + github.com/tidwall/gjson v1.7.4 + github.com/tidwall/grect v0.1.1 + github.com/tidwall/lotsa v1.0.2 + github.com/tidwall/match v1.0.3 + github.com/tidwall/rtred v0.1.2 ) diff --git a/vendor/github.com/tidwall/buntdb/go.sum b/vendor/github.com/tidwall/buntdb/go.sum index 6c801950..8cf5f843 100644 --- a/vendor/github.com/tidwall/buntdb/go.sum +++ b/vendor/github.com/tidwall/buntdb/go.sum @@ -1,14 +1,16 @@ -github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY= -github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= -github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws= -github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0= -github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE= -github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M= -github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= -github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8SutN9KQHc6vqHEXJTcAFwyGNius= -github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao= -github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE= -github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ= +github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs= +github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= +github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8= +github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= +github.com/tidwall/grect v0.1.1 h1:+kMEkxhoqB7rniVXzMEIA66XwU07STgINqxh+qVIndY= +github.com/tidwall/grect v0.1.1/go.mod h1:CzvbGiFbWUwiJ1JohXLb28McpyBsI00TK9Y6pDWLGRQ= +github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= +github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= +github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= +github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8= +github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= +github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= +github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= +github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= diff --git a/vendor/github.com/tidwall/gjson/.travis.yml b/vendor/github.com/tidwall/gjson/.travis.yml deleted file mode 100644 index 4f2ee4d9..00000000 --- a/vendor/github.com/tidwall/gjson/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index 4108deb3..55620142 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -3,13 +3,10 @@ src="logo.png" width="240" height="78" border="0" alt="GJSON">
-Build Status GoDoc GJSON Playground

- -

get json values quickly

GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document. @@ -17,6 +14,8 @@ It has features such as [one line retrieval](#get-a-value), [dot notation paths] Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool. +For the Rust version go to [gjson.rs](https://github.com/tidwall/gjson.rs). + Getting Started =============== @@ -476,7 +475,7 @@ JSON document used: } ``` -Each operation was rotated though one of the following search paths: +Each operation was rotated through one of the following search paths: ``` widget.window.name @@ -484,12 +483,4 @@ widget.image.hOffset widget.text.onMouseUp ``` -*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).* - - -## Contact -Josh Baker [@tidwall](http://twitter.com/tidwall) - -## License - -GJSON source code is available under the MIT [License](/LICENSE). +*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be found [here](https://github.com/tidwall/gjson-benchmarks).* diff --git a/vendor/github.com/tidwall/gjson/SYNTAX.md b/vendor/github.com/tidwall/gjson/SYNTAX.md index 5ea0407f..86235b9b 100644 --- a/vendor/github.com/tidwall/gjson/SYNTAX.md +++ b/vendor/github.com/tidwall/gjson/SYNTAX.md @@ -77,14 +77,21 @@ Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`. fav\.movie "Deer Hunter" ``` -You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in source code. +You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in you source code. ```go -res := gjson.Get(json, "fav\\.movie") // must escape the slash -res := gjson.Get(json, `fav\.movie`) // no need to escape the slash - +// Go +val := gjson.Get(json, "fav\\.movie") // must escape the slash +val := gjson.Get(json, `fav\.movie`) // no need to escape the slash ``` +```rust +// Rust +let val = gjson::get(json, "fav\\.movie") // must escape the slash +let val = gjson::get(json, r#"fav\.movie"#) // no need to escape the slash +``` + + ### Arrays The `#` character allows for digging into JSON Arrays. @@ -248,6 +255,8 @@ gjson.AddModifier("case", func(json, arg string) string { "children.@case:lower.@reverse" ["jack","alex","sara"] ``` +*Note: Custom modifiers are not yet available in the Rust version* + ### Multipaths Starting with v1.3.0, GJSON added the ability to join multiple paths together diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index 0b6dcb09..7bb82e95 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -3,7 +3,6 @@ package gjson import ( "encoding/json" - "reflect" "strconv" "strings" "time" @@ -106,7 +105,8 @@ func (t Result) Bool() bool { case True: return true case String: - return t.Str != "" && t.Str != "0" && t.Str != "false" + b, _ := strconv.ParseBool(strings.ToLower(t.Str)) + return b case Number: return t.Num != 0 } @@ -124,16 +124,17 @@ func (t Result) Int() int64 { return n case Number: // try to directly convert the float64 to int64 - n, ok := floatToInt(t.Num) - if !ok { - // now try to parse the raw string - n, ok = parseInt(t.Raw) - if !ok { - // fallback to a standard conversion - return int64(t.Num) - } + i, ok := safeInt(t.Num) + if ok { + return i } - return n + // now try to parse the raw string + i, ok = parseInt(t.Raw) + if ok { + return i + } + // fallback to a standard conversion + return int64(t.Num) } } @@ -149,16 +150,17 @@ func (t Result) Uint() uint64 { return n case Number: // try to directly convert the float64 to uint64 - n, ok := floatToUint(t.Num) - if !ok { - // now try to parse the raw string - n, ok = parseUint(t.Raw) - if !ok { - // fallback to a standard conversion - return uint64(t.Num) - } + i, ok := safeInt(t.Num) + if ok && i >= 0 { + return uint64(i) } - return n + // now try to parse the raw string + u, ok := parseUint(t.Raw) + if ok { + return u + } + // fallback to a standard conversion + return uint64(t.Num) } } @@ -495,6 +497,9 @@ func squash(json string) string { } } if depth == 0 { + if i >= len(json) { + return json + } return json[:i+1] } case '{', '[', '(': @@ -726,8 +731,13 @@ func parseArrayPath(path string) (r arrayPathResult) { } if path[i] == '.' { r.part = path[:i] - r.path = path[i+1:] - r.more = true + if !r.arrch && i < len(path)-1 && isDotPiperChar(path[i+1]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + r.more = true + } return } if path[i] == '#' { @@ -973,6 +983,11 @@ right: return s } +// peek at the next byte and see if it's a '@', '[', or '{'. +func isDotPiperChar(c byte) bool { + return !DisableModifiers && (c == '@' || c == '[' || c == '{') +} + type objectPathResult struct { part string path string @@ -991,12 +1006,8 @@ func parseObjectPath(path string) (r objectPathResult) { return } if path[i] == '.' { - // peek at the next byte and see if it's a '@', '[', or '{'. r.part = path[:i] - if !DisableModifiers && - i < len(path)-1 && - (path[i+1] == '@' || - path[i+1] == '[' || path[i+1] == '{') { + if i < len(path)-1 && isDotPiperChar(path[i+1]) { r.pipe = path[i+1:] r.piped = true } else { @@ -1026,14 +1037,11 @@ func parseObjectPath(path string) (r objectPathResult) { continue } else if path[i] == '.' { r.part = string(epart) - // peek at the next byte and see if it's a '@' modifier - if !DisableModifiers && - i < len(path)-1 && path[i+1] == '@' { + if i < len(path)-1 && isDotPiperChar(path[i+1]) { r.pipe = path[i+1:] r.piped = true } else { r.path = path[i+1:] - r.more = true } r.more = true return @@ -1398,7 +1406,6 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } return false } - for i < len(c.json)+1 { if !rp.arrch { pmatch = partidx == h @@ -1600,10 +1607,17 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { c.calcd = true return i + 1, true } - if len(multires) > 0 && !c.value.Exists() { - c.value = Result{ - Raw: string(append(multires, ']')), - Type: JSON, + if !c.value.Exists() { + if len(multires) > 0 { + c.value = Result{ + Raw: string(append(multires, ']')), + Type: JSON, + } + } else if rp.query.all { + c.value = Result{ + Raw: "[]", + Type: JSON, + } } } return i + 1, false @@ -1835,7 +1849,7 @@ type parseContext struct { // A path is in dot syntax, such as "name.last" or "age". // When the value is found it's returned immediately. // -// A path is a series of keys searated by a dot. +// A path is a series of keys separated by a dot. // A key may contain special wildcard characters '*' and '?'. // To access an array value use the index as the key. // To get the number of elements in an array or to access a child path, use @@ -1944,7 +1958,6 @@ func Get(json, path string) Result { } } } - var i int var c = &parseContext{json: json} if len(path) >= 2 && path[0] == '.' && path[1] == '.' { @@ -2169,11 +2182,6 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) { return i, res, false } -var ( // used for testing - testWatchForFallback bool - testLastWasFallback bool -) - // GetMany searches json for the multiple paths. // The return value is a Result array where the number of items // will be equal to the number of input paths. @@ -2374,6 +2382,12 @@ func validnumber(data []byte, i int) (outi int, ok bool) { // sign if data[i] == '-' { i++ + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } } // int if i == len(data) { @@ -2524,25 +2538,14 @@ func parseInt(s string) (n int64, ok bool) { return n, true } -const minUint53 = 0 -const maxUint53 = 4503599627370495 -const minInt53 = -2251799813685248 -const maxInt53 = 2251799813685247 - -func floatToUint(f float64) (n uint64, ok bool) { - n = uint64(f) - if float64(n) == f && n >= minUint53 && n <= maxUint53 { - return n, true +// safeInt validates a given JSON number +// ensures it lies within the minimum and maximum representable JSON numbers +func safeInt(f float64) (n int64, ok bool) { + // https://tc39.es/ecma262/#sec-number.min_safe_integer || https://tc39.es/ecma262/#sec-number.max_safe_integer + if f < -9007199254740991 || f > 9007199254740991 { + return 0, false } - return 0, false -} - -func floatToInt(f float64) (n int64, ok bool) { - n = int64(f) - if float64(n) == f && n >= minInt53 && n <= maxInt53 { - return n, true - } - return 0, false + return int64(f), true } // execModifier parses the path to find a matching modifier function. @@ -2600,7 +2603,7 @@ func execModifier(json, path string) (pathOut, res string, ok bool) { // unwrap removes the '[]' or '{}' characters around json func unwrap(json string) string { json = trim(json) - if len(json) >= 2 && json[0] == '[' || json[0] == '{' { + if len(json) >= 2 && (json[0] == '[' || json[0] == '{') { json = json[1 : len(json)-1] } return json @@ -2632,6 +2635,26 @@ func ModifierExists(name string, fn func(json, arg string) string) bool { return ok } +// cleanWS remove any non-whitespace from string +func cleanWS(s string) string { + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + continue + default: + var s2 []byte + for i := 0; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + s2 = append(s2, s[i]) + } + } + return string(s2) + } + } + return s +} + // @pretty modifier makes the json look nice. func modPretty(json, arg string) string { if len(arg) > 0 { @@ -2641,9 +2664,9 @@ func modPretty(json, arg string) string { case "sortKeys": opts.SortKeys = value.Bool() case "indent": - opts.Indent = value.String() + opts.Indent = cleanWS(value.String()) case "prefix": - opts.Prefix = value.String() + opts.Prefix = cleanWS(value.String()) case "width": opts.Width = int(value.Int()) } @@ -2729,19 +2752,24 @@ func modFlatten(json, arg string) string { out = append(out, '[') var idx int res.ForEach(func(_, value Result) bool { - if idx > 0 { - out = append(out, ',') - } + var raw string if value.IsArray() { if deep { - out = append(out, unwrap(modFlatten(value.Raw, arg))...) + raw = unwrap(modFlatten(value.Raw, arg)) } else { - out = append(out, unwrap(value.Raw)...) + raw = unwrap(value.Raw) } } else { - out = append(out, value.Raw...) + raw = value.Raw + } + raw = strings.TrimSpace(raw) + if len(raw) > 0 { + if idx > 0 { + out = append(out, ',') + } + out = append(out, raw...) + idx++ } - idx++ return true }) out = append(out, ']') @@ -2825,6 +2853,19 @@ func modValid(json, arg string) string { return json } +// stringHeader instead of reflect.StringHeader +type stringHeader struct { + data unsafe.Pointer + len int +} + +// sliceHeader instead of reflect.SliceHeader +type sliceHeader struct { + data unsafe.Pointer + len int + cap int +} + // getBytes casts the input json bytes to a string and safely returns the // results as uniquely allocated data. This operation is intended to minimize // copies and allocations for the large json string->[]byte. @@ -2834,14 +2875,14 @@ func getBytes(json []byte, path string) Result { // unsafe cast to string result = Get(*(*string)(unsafe.Pointer(&json)), path) // safely get the string headers - rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) - strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) + rawhi := *(*stringHeader)(unsafe.Pointer(&result.Raw)) + strhi := *(*stringHeader)(unsafe.Pointer(&result.Str)) // create byte slice headers - rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} - strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} - if strh.Data == 0 { + rawh := sliceHeader{data: rawhi.data, len: rawhi.len, cap: rawhi.len} + strh := sliceHeader{data: strhi.data, len: strhi.len, cap: rawhi.len} + if strh.data == nil { // str is nil - if rawh.Data == 0 { + if rawh.data == nil { // raw is nil result.Raw = "" } else { @@ -2849,19 +2890,20 @@ func getBytes(json []byte, path string) Result { result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) } result.Str = "" - } else if rawh.Data == 0 { + } else if rawh.data == nil { // raw is nil result.Raw = "" // str has data, safely copy the slice header to a string result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } else if strh.Data >= rawh.Data && - int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { + } else if uintptr(strh.data) >= uintptr(rawh.data) && + uintptr(strh.data)+uintptr(strh.len) <= + uintptr(rawh.data)+uintptr(rawh.len) { // Str is a substring of Raw. - start := int(strh.Data - rawh.Data) + start := uintptr(strh.data) - uintptr(rawh.data) // safely copy the raw slice header result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) // substring the raw - result.Str = result.Raw[start : start+strh.Len] + result.Str = result.Raw[start : start+uintptr(strh.len)] } else { // safely copy both the raw and str slice headers to strings result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) @@ -2876,9 +2918,9 @@ func getBytes(json []byte, path string) Result { // used instead. func fillIndex(json string, c *parseContext) { if len(c.value.Raw) > 0 && !c.calcd { - jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json)) - rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw))) - c.value.Index = int(rhdr.Data - jhdr.Data) + jhdr := *(*stringHeader)(unsafe.Pointer(&json)) + rhdr := *(*stringHeader)(unsafe.Pointer(&(c.value.Raw))) + c.value.Index = int(uintptr(rhdr.data) - uintptr(jhdr.data)) if c.value.Index < 0 || c.value.Index >= len(json) { c.value.Index = 0 } @@ -2886,10 +2928,10 @@ func fillIndex(json string, c *parseContext) { } func stringBytes(s string) []byte { - return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data, - Len: len(s), - Cap: len(s), + return *(*[]byte)(unsafe.Pointer(&sliceHeader{ + data: (*stringHeader)(unsafe.Pointer(&s)).data, + len: len(s), + cap: len(s), })) } diff --git a/vendor/github.com/tidwall/gjson/go.mod b/vendor/github.com/tidwall/gjson/go.mod index c287095f..9853a309 100644 --- a/vendor/github.com/tidwall/gjson/go.mod +++ b/vendor/github.com/tidwall/gjson/go.mod @@ -3,6 +3,6 @@ module github.com/tidwall/gjson go 1.12 require ( - github.com/tidwall/match v1.0.1 - github.com/tidwall/pretty v1.0.2 + github.com/tidwall/match v1.0.3 + github.com/tidwall/pretty v1.1.0 ) diff --git a/vendor/github.com/tidwall/gjson/go.sum b/vendor/github.com/tidwall/gjson/go.sum index 75e1dd7b..739d1327 100644 --- a/vendor/github.com/tidwall/gjson/go.sum +++ b/vendor/github.com/tidwall/gjson/go.sum @@ -1,4 +1,4 @@ -github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= -github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= +github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8= +github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= diff --git a/vendor/github.com/tidwall/grect/go.mod b/vendor/github.com/tidwall/grect/go.mod new file mode 100644 index 00000000..9399750d --- /dev/null +++ b/vendor/github.com/tidwall/grect/go.mod @@ -0,0 +1,5 @@ +module github.com/tidwall/grect + +go 1.15 + +require github.com/tidwall/gjson v1.7.4 diff --git a/vendor/github.com/tidwall/grect/go.sum b/vendor/github.com/tidwall/grect/go.sum new file mode 100644 index 00000000..7dd8de53 --- /dev/null +++ b/vendor/github.com/tidwall/grect/go.sum @@ -0,0 +1,6 @@ +github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8= +github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= +github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= +github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8= +github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= diff --git a/vendor/github.com/tidwall/match/.travis.yml b/vendor/github.com/tidwall/match/.travis.yml deleted file mode 100644 index 4f2ee4d9..00000000 --- a/vendor/github.com/tidwall/match/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/tidwall/match/README.md b/vendor/github.com/tidwall/match/README.md index 2aa5bc38..5fdd4cf6 100644 --- a/vendor/github.com/tidwall/match/README.md +++ b/vendor/github.com/tidwall/match/README.md @@ -1,20 +1,17 @@ -Match -===== -Build Status -GoDoc +# Match + +[![GoDoc](https://godoc.org/github.com/tidwall/match?status.svg)](https://godoc.org/github.com/tidwall/match) Match is a very simple pattern matcher where '*' matches on any number characters and '?' matches on any one character. -Installing ----------- +## Installing ``` go get -u github.com/tidwall/match ``` -Example -------- +## Example ```go match.Match("hello", "*llo") @@ -23,10 +20,10 @@ match.Match("hello", "h*o") ``` -Contact -------- +## Contact + Josh Baker [@tidwall](http://twitter.com/tidwall) -License -------- +## License + Redcon source code is available under the MIT [License](/LICENSE). diff --git a/vendor/github.com/tidwall/match/go.mod b/vendor/github.com/tidwall/match/go.mod new file mode 100644 index 00000000..df19b5f7 --- /dev/null +++ b/vendor/github.com/tidwall/match/go.mod @@ -0,0 +1,3 @@ +module github.com/tidwall/match + +go 1.15 diff --git a/vendor/github.com/tidwall/match/match.go b/vendor/github.com/tidwall/match/match.go index fcfe998b..57aabf42 100644 --- a/vendor/github.com/tidwall/match/match.go +++ b/vendor/github.com/tidwall/match/match.go @@ -1,4 +1,4 @@ -// Match provides a simple pattern matcher with unicode support. +// Package match provides a simple pattern matcher with unicode support. package match import "unicode/utf8" @@ -6,7 +6,7 @@ import "unicode/utf8" // Match returns true if str matches pattern. This is a very // simple wildcard match where '*' matches on any number characters // and '?' matches on any one character. - +// // pattern: // { term } // term: @@ -16,12 +16,16 @@ import "unicode/utf8" // '\\' c matches character c // func Match(str, pattern string) bool { + return deepMatch(str, pattern) +} + +func deepMatch(str, pattern string) bool { if pattern == "*" { return true } - return deepMatch(str, pattern) -} -func deepMatch(str, pattern string) bool { + for len(pattern) > 1 && pattern[0] == '*' && pattern[1] == '*' { + pattern = pattern[1:] + } for len(pattern) > 0 { if pattern[0] > 0x7f { return deepMatchRune(str, pattern) @@ -52,6 +56,13 @@ func deepMatch(str, pattern string) bool { } func deepMatchRune(str, pattern string) bool { + if pattern == "*" { + return true + } + for len(pattern) > 1 && pattern[0] == '*' && pattern[1] == '*' { + pattern = pattern[1:] + } + var sr, pr rune var srsz, prsz int diff --git a/vendor/github.com/tidwall/pretty/.travis.yml b/vendor/github.com/tidwall/pretty/.travis.yml deleted file mode 100644 index 4f2ee4d9..00000000 --- a/vendor/github.com/tidwall/pretty/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/tidwall/pretty/README.md b/vendor/github.com/tidwall/pretty/README.md index 09884692..7a614223 100644 --- a/vendor/github.com/tidwall/pretty/README.md +++ b/vendor/github.com/tidwall/pretty/README.md @@ -1,8 +1,6 @@ # Pretty -[![Build Status](https://img.shields.io/travis/tidwall/pretty.svg?style=flat-square)](https://travis-ci.org/tidwall/prettty) -[![Coverage Status](https://img.shields.io/badge/coverage-100%25-brightgreen.svg?style=flat-square)](http://gocover.io/github.com/tidwall/pretty) -[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty) +[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty) Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads. @@ -81,6 +79,45 @@ Will format the json to: {"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}``` ``` +## Spec + +Spec cleans comments and trailing commas from input JSON, converting it to +valid JSON per the official spec: https://tools.ietf.org/html/rfc8259 + +The resulting JSON will always be the same length as the input and it will +include all of the same line breaks at matching offsets. This is to ensure +the result can be later processed by a external parser and that that +parser will report messages or errors with the correct offsets. + +The following example uses a JSON document that has comments and trailing +commas and converts it prior to unmarshalling to using the standard Go +JSON library. + +```go + +data := ` +{ + /* Dev Machine */ + "dbInfo": { + "host": "localhost", + "port": 5432, // use full email address + "username": "josh", + "password": "pass123", // use a hashed password + } + /* Only SMTP Allowed */ + "emailInfo": { + "email": "josh@example.com", + "password": "pass123", + "smtp": "smpt.example.com", + } +} +` + +err := json.Unmarshal(pretty.Spec(data), &config) + +``` + + ## Customized output diff --git a/vendor/github.com/tidwall/pretty/pretty.go b/vendor/github.com/tidwall/pretty/pretty.go index 2951c610..7fa9d639 100644 --- a/vendor/github.com/tidwall/pretty/pretty.go +++ b/vendor/github.com/tidwall/pretty/pretty.go @@ -118,21 +118,27 @@ type pair struct { vstart, vend int } -type byKey struct { +type byKeyVal struct { sorted bool json []byte pairs []pair } -func (arr *byKey) Len() int { +func (arr *byKeyVal) Len() int { return len(arr.pairs) } -func (arr *byKey) Less(i, j int) bool { +func (arr *byKeyVal) Less(i, j int) bool { key1 := arr.json[arr.pairs[i].kstart+1 : arr.pairs[i].kend-1] key2 := arr.json[arr.pairs[j].kstart+1 : arr.pairs[j].kend-1] - return string(key1) < string(key2) + if string(key1) < string(key2) { + return true + } + if string(key1) > string(key2) { + return false + } + return arr.pairs[i].vstart < arr.pairs[j].vstart } -func (arr *byKey) Swap(i, j int) { +func (arr *byKeyVal) Swap(i, j int) { arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i] arr.sorted = true } @@ -174,7 +180,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, } if n > 0 { nl = len(buf) - buf = append(buf, '\n') + if buf[nl-1] == ' ' { + buf[nl-1] = '\n' + } else { + buf = append(buf, '\n') + } } if buf[len(buf)-1] != open { buf = appendTabs(buf, prefix, indent, tabs) @@ -193,7 +203,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, var p pair if pretty { nl = len(buf) - buf = append(buf, '\n') + if buf[nl-1] == ' ' { + buf[nl-1] = '\n' + } else { + buf = append(buf, '\n') + } if open == '{' && sortkeys { p.kstart = i p.vstart = len(buf) @@ -235,8 +249,8 @@ func sortPairs(json, buf []byte, pairs []pair) []byte { } vstart := pairs[0].vstart vend := pairs[len(pairs)-1].vend - arr := byKey{false, json, pairs} - sort.Sort(&arr) + arr := byKeyVal{false, json, pairs} + sort.Stable(&arr) if !arr.sorted { return buf } @@ -305,6 +319,7 @@ func appendTabs(buf []byte, prefix, indent string, tabs int) []byte { type Style struct { Key, String, Number [2]string True, False, Null [2]string + Escape [2]string Append func(dst []byte, c byte) []byte } @@ -328,6 +343,7 @@ func init() { True: [2]string{"\x1B[96m", "\x1B[0m"}, False: [2]string{"\x1B[96m", "\x1B[0m"}, Null: [2]string{"\x1B[91m", "\x1B[0m"}, + Escape: [2]string{"\x1B[35m", "\x1B[0m"}, Append: func(dst []byte, c byte) []byte { if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') { dst = append(dst, "\\u00"...) @@ -367,8 +383,39 @@ func Color(src []byte, style *Style) []byte { dst = append(dst, style.String[0]...) } dst = apnd(dst, '"') + esc := false + uesc := 0 for i = i + 1; i < len(src); i++ { - dst = apnd(dst, src[i]) + if src[i] == '\\' { + if key { + dst = append(dst, style.Key[1]...) + } else { + dst = append(dst, style.String[1]...) + } + dst = append(dst, style.Escape[0]...) + dst = apnd(dst, src[i]) + esc = true + if i+1 < len(src) && src[i+1] == 'u' { + uesc = 5 + } else { + uesc = 1 + } + } else if esc { + dst = apnd(dst, src[i]) + if uesc == 1 { + esc = false + dst = append(dst, style.Escape[1]...) + if key { + dst = append(dst, style.Key[0]...) + } else { + dst = append(dst, style.String[0]...) + } + } else { + uesc-- + } + } else { + dst = apnd(dst, src[i]) + } if src[i] == '"' { j := i - 1 for ; ; j-- { @@ -381,7 +428,9 @@ func Color(src []byte, style *Style) []byte { } } } - if key { + if esc { + dst = append(dst, style.Escape[1]...) + } else if key { dst = append(dst, style.Key[1]...) } else { dst = append(dst, style.String[1]...) @@ -434,3 +483,90 @@ func Color(src []byte, style *Style) []byte { } return dst } + +// Spec strips out comments and trailing commas and convert the input to a +// valid JSON per the official spec: https://tools.ietf.org/html/rfc8259 +// +// The resulting JSON will always be the same length as the input and it will +// include all of the same line breaks at matching offsets. This is to ensure +// the result can be later processed by a external parser and that that +// parser will report messages or errors with the correct offsets. +func Spec(src []byte) []byte { + return spec(src, nil) +} + +// SpecInPlace is the same as Spec, but this method reuses the input json +// buffer to avoid allocations. Do not use the original bytes slice upon return. +func SpecInPlace(src []byte) []byte { + return spec(src, src) +} + +func spec(src, dst []byte) []byte { + dst = dst[:0] + for i := 0; i < len(src); i++ { + if src[i] == '/' { + if i < len(src)-1 { + if src[i+1] == '/' { + dst = append(dst, ' ', ' ') + i += 2 + for ; i < len(src); i++ { + if src[i] == '\n' { + dst = append(dst, '\n') + break + } else if src[i] == '\t' || src[i] == '\r' { + dst = append(dst, src[i]) + } else { + dst = append(dst, ' ') + } + } + continue + } + if src[i+1] == '*' { + dst = append(dst, ' ', ' ') + i += 2 + for ; i < len(src)-1; i++ { + if src[i] == '*' && src[i+1] == '/' { + dst = append(dst, ' ', ' ') + i++ + break + } else if src[i] == '\n' || src[i] == '\t' || + src[i] == '\r' { + dst = append(dst, src[i]) + } else { + dst = append(dst, ' ') + } + } + continue + } + } + } + dst = append(dst, src[i]) + if src[i] == '"' { + for i = i + 1; i < len(src); i++ { + dst = append(dst, src[i]) + if src[i] == '"' { + j := i - 1 + for ; ; j-- { + if src[j] != '\\' { + break + } + } + if (j-i)%2 != 0 { + break + } + } + } + } else if src[i] == '}' || src[i] == ']' { + for j := len(dst) - 2; j >= 0; j-- { + if dst[j] <= ' ' { + continue + } + if dst[j] == ',' { + dst[j] = ' ' + } + break + } + } + } + return dst +} diff --git a/vendor/github.com/tidwall/rtree/LICENSE b/vendor/github.com/tidwall/rtred/LICENSE similarity index 100% rename from vendor/github.com/tidwall/rtree/LICENSE rename to vendor/github.com/tidwall/rtred/LICENSE diff --git a/vendor/github.com/tidwall/rtree/README.md b/vendor/github.com/tidwall/rtred/README.md similarity index 88% rename from vendor/github.com/tidwall/rtree/README.md rename to vendor/github.com/tidwall/rtred/README.md index 53a845d2..361d0412 100644 --- a/vendor/github.com/tidwall/rtree/README.md +++ b/vendor/github.com/tidwall/rtred/README.md @@ -1,7 +1,6 @@ RTree implementation for Go =========================== -[![Build Status](https://travis-ci.org/tidwall/rtree.svg?branch=master)](https://travis-ci.org/tidwall/rtree) [![GoDoc](https://godoc.org/github.com/tidwall/rtree?status.svg)](https://godoc.org/github.com/tidwall/rtree) This package provides an in-memory R-Tree implementation for Go, useful as a spatial data structure. diff --git a/vendor/github.com/tidwall/rtree/base/knn.go b/vendor/github.com/tidwall/rtred/base/knn.go similarity index 100% rename from vendor/github.com/tidwall/rtree/base/knn.go rename to vendor/github.com/tidwall/rtred/base/knn.go diff --git a/vendor/github.com/tidwall/rtree/base/load.go b/vendor/github.com/tidwall/rtred/base/load.go similarity index 100% rename from vendor/github.com/tidwall/rtree/base/load.go rename to vendor/github.com/tidwall/rtred/base/load.go diff --git a/vendor/github.com/tidwall/rtree/base/rtree.go b/vendor/github.com/tidwall/rtred/base/rtree.go similarity index 100% rename from vendor/github.com/tidwall/rtree/base/rtree.go rename to vendor/github.com/tidwall/rtred/base/rtree.go diff --git a/vendor/github.com/tidwall/rtred/go.mod b/vendor/github.com/tidwall/rtred/go.mod new file mode 100644 index 00000000..9fa4967c --- /dev/null +++ b/vendor/github.com/tidwall/rtred/go.mod @@ -0,0 +1,5 @@ +module github.com/tidwall/rtred + +go 1.15 + +require github.com/tidwall/tinyqueue v0.1.1 diff --git a/vendor/github.com/tidwall/rtred/go.sum b/vendor/github.com/tidwall/rtred/go.sum new file mode 100644 index 00000000..0fbfb4da --- /dev/null +++ b/vendor/github.com/tidwall/rtred/go.sum @@ -0,0 +1,2 @@ +github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= +github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= diff --git a/vendor/github.com/tidwall/rtree/rtree.go b/vendor/github.com/tidwall/rtred/rtree.go similarity index 99% rename from vendor/github.com/tidwall/rtree/rtree.go rename to vendor/github.com/tidwall/rtred/rtree.go index bbbb1ffe..89522cac 100644 --- a/vendor/github.com/tidwall/rtree/rtree.go +++ b/vendor/github.com/tidwall/rtred/rtree.go @@ -1,10 +1,10 @@ -package rtree +package rtred import ( "math" "sync" - "github.com/tidwall/rtree/base" + "github.com/tidwall/rtred/base" ) type Iterator func(item Item) bool diff --git a/vendor/github.com/tidwall/rtree/.travis.yml b/vendor/github.com/tidwall/rtree/.travis.yml deleted file mode 100644 index 4f2ee4d9..00000000 --- a/vendor/github.com/tidwall/rtree/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/tidwall/tinyqueue/go.mod b/vendor/github.com/tidwall/tinyqueue/go.mod new file mode 100644 index 00000000..6ba78df6 --- /dev/null +++ b/vendor/github.com/tidwall/tinyqueue/go.mod @@ -0,0 +1,3 @@ +module github.com/tidwall/tinyqueue + +go 1.15 diff --git a/vendor/modules.txt b/vendor/modules.txt index d7fd49d2..27a4ed2e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -39,23 +39,25 @@ github.com/oragono/confusables github.com/oragono/go-ident # github.com/stretchr/testify v1.4.0 ## explicit -# github.com/tidwall/btree v0.2.2 +# github.com/tidwall/btree v0.4.2 github.com/tidwall/btree -# github.com/tidwall/buntdb v1.1.4 +# github.com/tidwall/buntdb v1.2.3 ## explicit github.com/tidwall/buntdb -# github.com/tidwall/gjson v1.6.1 +# github.com/tidwall/gjson v1.7.4 github.com/tidwall/gjson -# github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb +# github.com/tidwall/grect v0.1.1 github.com/tidwall/grect -# github.com/tidwall/match v1.0.1 +# github.com/tidwall/match v1.0.3 github.com/tidwall/match -# github.com/tidwall/pretty v1.0.2 +# github.com/tidwall/pretty v1.1.0 github.com/tidwall/pretty +# github.com/tidwall/rtred v0.1.2 +github.com/tidwall/rtred +github.com/tidwall/rtred/base # github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 -github.com/tidwall/rtree -github.com/tidwall/rtree/base -# github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 +## explicit +# github.com/tidwall/tinyqueue v0.1.1 github.com/tidwall/tinyqueue # github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 ## explicit