3
0
mirror of https://github.com/ergochat/ergo.git synced 2024-11-11 06:29:29 +01:00

Merge pull request #1605 from slingamn/bunt_error

try to record buntdb errors from persisting lastSeen
This commit is contained in:
Shivaram Lingamneni 2021-04-02 04:49:52 -04:00 committed by GitHub
commit 2e9a0d4b2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 916 additions and 325 deletions

3
go.mod
View File

@ -16,7 +16,8 @@ require (
github.com/oragono/confusables v0.0.0-20201108231250-4ab98ab61fb1
github.com/oragono/go-ident v0.0.0-20200511222032-830550b1d775
github.com/stretchr/testify v1.4.0 // indirect
github.com/tidwall/buntdb v1.1.4
github.com/tidwall/buntdb v1.2.3
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 // indirect
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf // indirect

17
go.sum
View File

@ -67,20 +67,37 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY=
github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs=
github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/buntdb v1.1.4 h1:W7y9+2dM3GOswU0t3pz6+BcwZXjj/tVOhPcO6EHufME=
github.com/tidwall/buntdb v1.1.4/go.mod h1:06+/n7EFf6uUaIG5r9xZcExYN3H0Lnc+g/Kqx0fZFkI=
github.com/tidwall/buntdb v1.2.3 h1:AoGVe4yrhKmnEPHrPrW5EUOATHOCIk4VtFvd8xn/ZtU=
github.com/tidwall/buntdb v1.2.3/go.mod h1:+i/gBwYOHWG19wLgwMXFLkl00twh9+VWkkaOhuNQ4PA=
github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8=
github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
github.com/tidwall/grect v0.1.1 h1:+kMEkxhoqB7rniVXzMEIA66XwU07STgINqxh+qVIndY=
github.com/tidwall/grect v0.1.1/go.mod h1:CzvbGiFbWUwiJ1JohXLb28McpyBsI00TK9Y6pDWLGRQ=
github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8SutN9KQHc6vqHEXJTcAFwyGNius=
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM=
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

View File

@ -614,7 +614,7 @@ func (am *AccountManager) saveLastSeen(account string, lastSeen map[string]time.
text, _ := json.Marshal(lastSeen)
val = string(text)
}
am.server.store.Update(func(tx *buntdb.Tx) error {
err := am.server.store.Update(func(tx *buntdb.Tx) error {
if val != "" {
tx.Set(key, val, nil)
} else {
@ -622,6 +622,9 @@ func (am *AccountManager) saveLastSeen(account string, lastSeen map[string]time.
}
return nil
})
if err != nil {
am.server.logger.Error("internal", "error persisting lastSeen", account, err.Error())
}
}
func (am *AccountManager) loadLastSeen(account string) (lastSeen map[string]time.Time) {

View File

@ -1 +0,0 @@
language: go

View File

@ -1,5 +1,199 @@
# B-tree for Go
# btree
![Travis CI Build Status](https://api.travis-ci.org/tidwall/btree.svg?branch=master)
[![GoDoc](https://godoc.org/github.com/tidwall/btree?status.svg)](https://godoc.org/github.com/tidwall/btree)
An [efficient](#performance) [B-tree](https://en.wikipedia.org/wiki/B-tree) implementation in Go.
## Features
- `Copy()` method with copy-on-write support.
- Fast bulk loading for pre-ordered data using the `Load()` method.
- All operations are thread-safe.
- Path hinting optimization for operations with nearby keys.
## Installing
To start using btree, install Go and run `go get`:
```sh
$ go get -u github.com/tidwall/btree
```
## Usage
```go
package main
import (
"fmt"
"github.com/tidwall/btree"
)
type Item struct {
Key, Val string
}
// byKeys is a comparison function that compares item keys and returns true
// when a is less than b.
func byKeys(a, b interface{}) bool {
i1, i2 := a.(*Item), b.(*Item)
return i1.Key < i2.Key
}
// byVals is a comparison function that compares item values and returns true
// when a is less than b.
func byVals(a, b interface{}) bool {
i1, i2 := a.(*Item), b.(*Item)
if i1.Val < i2.Val {
return true
}
if i1.Val > i2.Val {
return false
}
// Both vals are equal so we should fall though
// and let the key comparison take over.
return byKeys(a, b)
}
func main() {
// Create a tree for keys and a tree for values.
// The "keys" tree will be sorted on the Keys field.
// The "values" tree will be sorted on the Values field.
keys := btree.New(byKeys)
vals := btree.New(byVals)
// Create some items.
users := []*Item{
&Item{Key: "user:1", Val: "Jane"},
&Item{Key: "user:2", Val: "Andy"},
&Item{Key: "user:3", Val: "Steve"},
&Item{Key: "user:4", Val: "Andrea"},
&Item{Key: "user:5", Val: "Janet"},
&Item{Key: "user:6", Val: "Andy"},
}
// Insert each user into both trees
for _, user := range users {
keys.Set(user)
vals.Set(user)
}
// Iterate over each user in the key tree
keys.Ascend(nil, func(item interface{}) bool {
kvi := item.(*Item)
fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
return true
})
fmt.Printf("\n")
// Iterate over each user in the val tree
vals.Ascend(nil, func(item interface{}) bool {
kvi := item.(*Item)
fmt.Printf("%s %s\n", kvi.Key, kvi.Val)
return true
})
// Output:
// user:1 Jane
// user:2 Andy
// user:3 Steve
// user:4 Andrea
// user:5 Janet
// user:6 Andy
//
// user:4 Andrea
// user:2 Andy
// user:6 Andy
// user:1 Jane
// user:5 Janet
// user:3 Steve
}
```
## Operations
### Basic
```
Len() # return the number of items in the btree
Set(item) # insert or replace an existing item
Get(item) # get an existing item
Delete(item) # delete an item
```
### Iteration
```
Ascend(pivot, iter) # scan items in ascending order starting at pivot.
Descend(pivot, iter) # scan items in descending order starting at pivot.
```
### Queues
```
Min() # return the first item in the btree
Max() # return the last item in the btree
PopMin() # remove and return the first item in the btree
PopMax() # remove and return the last item in the btree
```
### Bulk loading
```
Load(item) # load presorted items into tree
```
### Path hints
```
SetHint(item, *hint) # insert or replace an existing item
GetHint(item, *hint) # get an existing item
DeleteHint(item, *hint) # delete an item
```
## Performance
This implementation was designed with performance in mind.
The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel Core i9) using Go 1.15.3. The items are simple 8-byte ints.
- `google`: The [google/btree](https://github.com/google/btree) package
- `tidwall`: The [tidwall/btree](https://github.com/tidwall/btree) package
- `go-arr`: Just a simple Go array
```
** sequential set **
google: set-seq 1,000,000 ops in 160ms, 6,262,097/sec, 159 ns/op, 31.0 MB, 32 bytes/op
tidwall: set-seq 1,000,000 ops in 142ms, 7,020,721/sec, 142 ns/op, 36.6 MB, 38 bytes/op
tidwall: set-seq-hint 1,000,000 ops in 87ms, 11,503,315/sec, 86 ns/op, 36.6 MB, 38 bytes/op
tidwall: load-seq 1,000,000 ops in 37ms, 27,177,242/sec, 36 ns/op, 36.6 MB, 38 bytes/op
go-arr: append 1,000,000 ops in 49ms, 20,574,760/sec, 48 ns/op
** random set **
google: set-rand 1,000,000 ops in 606ms, 1,649,921/sec, 606 ns/op, 21.5 MB, 22 bytes/op
tidwall: set-rand 1,000,000 ops in 543ms, 1,841,590/sec, 543 ns/op, 26.7 MB, 27 bytes/op
tidwall: set-rand-hint 1,000,000 ops in 573ms, 1,745,624/sec, 572 ns/op, 26.4 MB, 27 bytes/op
tidwall: set-again 1,000,000 ops in 452ms, 2,212,581/sec, 451 ns/op, 27.1 MB, 28 bytes/op
tidwall: set-after-copy 1,000,000 ops in 472ms, 2,117,457/sec, 472 ns/op, 27.9 MB, 29 bytes/op
tidwall: load-rand 1,000,000 ops in 551ms, 1,816,498/sec, 550 ns/op, 26.1 MB, 27 bytes/op
** sequential get **
google: get-seq 1,000,000 ops in 133ms, 7,497,604/sec, 133 ns/op
tidwall: get-seq 1,000,000 ops in 110ms, 9,082,972/sec, 110 ns/op
tidwall: get-seq-hint 1,000,000 ops in 55ms, 18,289,945/sec, 54 ns/op
** random get **
google: get-rand 1,000,000 ops in 149ms, 6,704,337/sec, 149 ns/op
tidwall: get-rand 1,000,000 ops in 131ms, 7,616,296/sec, 131 ns/op
tidwall: get-rand-hint 1,000,000 ops in 216ms, 4,632,532/sec, 215 ns/op
```
*You can find the benchmark utility at [tidwall/btree-benchmark](https://github.com/tidwall/btree-benchmark)*
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
## License
Source code is available under the MIT [License](/LICENSE).

View File

@ -4,35 +4,39 @@
package btree
import "sync"
const maxItems = 255
const minItems = maxItems * 40 / 100
type cow struct {
_ int // it cannot be an empty struct
}
type node struct {
cow *cow
leaf bool
numItems int16
items [maxItems]interface{}
children *[maxItems + 1]*node
}
type justaLeaf struct {
leaf bool
numItems int16
items [maxItems]interface{}
}
// BTree is an ordered set items
type BTree struct {
mu *sync.RWMutex
cow *cow
root *node
length int
less func(a, b interface{}) bool
lnode *node
}
func newNode(leaf bool) *node {
func (tr *BTree) newNode(leaf bool) *node {
n := &node{leaf: leaf}
if !leaf {
n.children = new([maxItems + 1]*node)
}
n.cow = tr.cow
return n
}
@ -48,6 +52,7 @@ func New(less func(a, b interface{}) bool) *BTree {
panic("nil less")
}
tr := new(BTree)
tr.mu = new(sync.RWMutex)
tr.less = less
return tr
}
@ -85,8 +90,7 @@ func (n *node) find(key interface{}, less func(a, b interface{}) bool,
high = mid - 1
}
}
if low > 0 && !less(n.items[low-1], key) &&
!less(key, n.items[low-1]) {
if low > 0 && !less(n.items[low-1], key) {
index = low - 1
found = true
} else {
@ -109,22 +113,29 @@ func (tr *BTree) SetHint(item interface{}, hint *PathHint) (prev interface{}) {
if item == nil {
panic("nil item")
}
tr.mu.Lock()
prev = tr.setHint(item, hint)
tr.mu.Unlock()
return prev
}
func (tr *BTree) setHint(item interface{}, hint *PathHint) (prev interface{}) {
if tr.root == nil {
tr.root = newNode(true)
tr.root = tr.newNode(true)
tr.root.items[0] = item
tr.root.numItems = 1
tr.length = 1
return
}
prev = tr.root.set(item, tr.less, hint, 0)
prev = tr.nodeSet(&tr.root, item, tr.less, hint, 0)
if prev != nil {
return prev
}
tr.lnode = nil
if tr.root.numItems == maxItems {
n := tr.root
right, median := n.split()
tr.root = newNode(false)
n := tr.cowLoad(&tr.root)
right, median := tr.nodeSplit(n)
tr.root = tr.newNode(false)
tr.root.children[0] = n
tr.root.items[0] = median
tr.root.children[1] = right
@ -139,8 +150,8 @@ func (tr *BTree) Set(item interface{}) (prev interface{}) {
return tr.SetHint(item, nil)
}
func (n *node) split() (right *node, median interface{}) {
right = newNode(n.leaf)
func (tr *BTree) nodeSplit(n *node) (right *node, median interface{}) {
right = tr.newNode(n.leaf)
median = n.items[maxItems/2]
copy(right.items[:maxItems/2], n.items[maxItems/2+1:])
if !n.leaf {
@ -159,9 +170,30 @@ func (n *node) split() (right *node, median interface{}) {
return right, median
}
func (n *node) set(item interface{}, less func(a, b interface{}) bool,
hint *PathHint, depth int,
//go:noinline
func (tr *BTree) copy(n *node) *node {
n2 := *n
n2.cow = tr.cow
copy(n2.items[:], n.items[:])
if n.children != nil {
n2.children = new([maxItems + 1]*node)
copy(n2.children[:], n.children[:])
}
return &n2
}
// cowLoad loaded the provide node and, if needed, performs a copy-on-write.
func (tr *BTree) cowLoad(cn **node) *node {
if (*cn).cow != tr.cow {
*cn = tr.copy(*cn)
}
return *cn
}
func (tr *BTree) nodeSet(cn **node, item interface{},
less func(a, b interface{}) bool, hint *PathHint, depth int,
) (prev interface{}) {
n := tr.cowLoad(cn)
i, found := n.find(item, less, hint, depth)
if found {
prev = n.items[i]
@ -174,12 +206,12 @@ func (n *node) set(item interface{}, less func(a, b interface{}) bool,
n.numItems++
return nil
}
prev = n.children[i].set(item, less, hint, depth+1)
prev = tr.nodeSet(&n.children[i], item, less, hint, depth+1)
if prev != nil {
return prev
}
if n.children[i].numItems == maxItems {
right, median := n.children[i].split()
right, median := tr.nodeSplit(n.children[i])
copy(n.children[i+1:], n.children[i:])
copy(n.items[i+1:], n.items[i:])
n.items[i] = median
@ -216,6 +248,8 @@ func (tr *BTree) Get(key interface{}) interface{} {
// GetHint gets a value for key using a path hint
func (tr *BTree) GetHint(key interface{}, hint *PathHint) interface{} {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil || key == nil {
return nil
}
@ -246,10 +280,17 @@ func (tr *BTree) Delete(key interface{}) interface{} {
// DeleteHint deletes a value for a key using a path hint
func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} {
tr.mu.Lock()
prev := tr.deleteHint(key, hint)
tr.mu.Unlock()
return prev
}
func (tr *BTree) deleteHint(key interface{}, hint *PathHint) interface{} {
if tr.root == nil || key == nil {
return nil
}
prev := tr.root.delete(false, key, tr.less, hint, 0)
prev := tr.delete(&tr.root, false, key, tr.less, hint, 0)
if prev == nil {
return nil
}
@ -264,9 +305,10 @@ func (tr *BTree) DeleteHint(key interface{}, hint *PathHint) interface{} {
return prev
}
func (n *node) delete(max bool, key interface{},
func (tr *BTree) delete(cn **node, max bool, key interface{},
less func(a, b interface{}) bool, hint *PathHint, depth int,
) interface{} {
n := tr.cowLoad(cn)
var i int16
var found bool
if max {
@ -290,74 +332,79 @@ func (n *node) delete(max bool, key interface{},
if found {
if max {
i++
prev = n.children[i].delete(true, "", less, nil, 0)
prev = tr.delete(&n.children[i], true, "", less, nil, 0)
} else {
prev = n.items[i]
maxItem := n.children[i].delete(true, "", less, nil, 0)
maxItem := tr.delete(&n.children[i], true, "", less, nil, 0)
n.items[i] = maxItem
}
} else {
prev = n.children[i].delete(max, key, less, hint, depth+1)
prev = tr.delete(&n.children[i], max, key, less, hint, depth+1)
}
if prev == nil {
return nil
}
if n.children[i].numItems < minItems {
if i == n.numItems {
i--
if n.children[i].numItems >= minItems {
return prev
}
// merge / rebalance nodes
if i == n.numItems {
i--
}
n.children[i] = tr.cowLoad(&n.children[i])
n.children[i+1] = tr.cowLoad(&n.children[i+1])
if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems {
// merge left + item + right
n.children[i].items[n.children[i].numItems] = n.items[i]
copy(n.children[i].items[n.children[i].numItems+1:],
n.children[i+1].items[:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i].children[n.children[i].numItems+1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
if n.children[i].numItems+n.children[i+1].numItems+1 < maxItems {
// merge left + item + right
n.children[i].items[n.children[i].numItems] = n.items[i]
copy(n.children[i].items[n.children[i].numItems+1:],
n.children[i+1].items[:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i].children[n.children[i].numItems+1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
n.children[i].numItems += n.children[i+1].numItems + 1
copy(n.items[i:], n.items[i+1:n.numItems])
copy(n.children[i+1:], n.children[i+2:n.numItems+1])
n.items[n.numItems] = nil
n.children[n.numItems+1] = nil
n.numItems--
} else if n.children[i].numItems > n.children[i+1].numItems {
// move left -> right
copy(n.children[i+1].items[1:],
n.children[i+1].items[:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i+1].children[1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
n.children[i+1].items[0] = n.items[i]
if !n.children[0].leaf {
n.children[i+1].children[0] =
n.children[i].children[n.children[i].numItems]
}
n.children[i+1].numItems++
n.items[i] = n.children[i].items[n.children[i].numItems-1]
n.children[i].items[n.children[i].numItems-1] = nil
if !n.children[0].leaf {
n.children[i].children[n.children[i].numItems] = nil
}
n.children[i].numItems--
} else {
// move right -> left
n.children[i].items[n.children[i].numItems] = n.items[i]
if !n.children[0].leaf {
n.children[i].children[n.children[i].numItems+1] =
n.children[i+1].children[0]
}
n.children[i].numItems++
n.items[i] = n.children[i+1].items[0]
copy(n.children[i+1].items[:],
n.children[i+1].items[1:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i+1].children[:],
n.children[i+1].children[1:n.children[i+1].numItems+1])
}
n.children[i+1].numItems--
n.children[i].numItems += n.children[i+1].numItems + 1
copy(n.items[i:], n.items[i+1:n.numItems])
copy(n.children[i+1:], n.children[i+2:n.numItems+1])
n.items[n.numItems] = nil
n.children[n.numItems+1] = nil
n.numItems--
} else if n.children[i].numItems > n.children[i+1].numItems {
// move left -> right
copy(n.children[i+1].items[1:],
n.children[i+1].items[:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i+1].children[1:],
n.children[i+1].children[:n.children[i+1].numItems+1])
}
n.children[i+1].items[0] = n.items[i]
if !n.children[0].leaf {
n.children[i+1].children[0] =
n.children[i].children[n.children[i].numItems]
}
n.children[i+1].numItems++
n.items[i] = n.children[i].items[n.children[i].numItems-1]
n.children[i].items[n.children[i].numItems-1] = nil
if !n.children[0].leaf {
n.children[i].children[n.children[i].numItems] = nil
}
n.children[i].numItems--
} else {
// move right -> left
n.children[i].items[n.children[i].numItems] = n.items[i]
if !n.children[0].leaf {
n.children[i].children[n.children[i].numItems+1] =
n.children[i+1].children[0]
}
n.children[i].numItems++
n.items[i] = n.children[i+1].items[0]
copy(n.children[i+1].items[:],
n.children[i+1].items[1:n.children[i+1].numItems])
if !n.children[0].leaf {
copy(n.children[i+1].children[:],
n.children[i+1].children[1:n.children[i+1].numItems+1])
}
n.children[i+1].numItems--
}
return prev
}
@ -366,6 +413,8 @@ func (n *node) delete(max bool, key interface{},
// Pass nil for pivot to scan all item in ascending order
// Return false to stop iterating
func (tr *BTree) Ascend(pivot interface{}, iter func(item interface{}) bool) {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil {
return
}
@ -427,6 +476,8 @@ func (n *node) reverse(iter func(item interface{}) bool) bool {
// Pass nil for pivot to scan all item in descending order
// Return false to stop iterating
func (tr *BTree) Descend(pivot interface{}, iter func(item interface{}) bool) {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil {
return
}
@ -467,6 +518,12 @@ func (tr *BTree) Load(item interface{}) interface{} {
if item == nil {
panic("nil item")
}
tr.mu.Lock()
defer tr.mu.Unlock()
// Load does not need a cowGrid because the Copy operation sets the
// lnode to nil.
if tr.lnode != nil && tr.lnode.numItems < maxItems-2 {
if tr.less(tr.lnode.items[tr.lnode.numItems-1], item) {
tr.lnode.items[tr.lnode.numItems] = item
@ -475,7 +532,7 @@ func (tr *BTree) Load(item interface{}) interface{} {
return nil
}
}
prev := tr.Set(item)
prev := tr.setHint(item, nil)
if prev != nil {
return prev
}
@ -493,6 +550,8 @@ func (tr *BTree) Load(item interface{}) interface{} {
// Min returns the minimum item in tree.
// Returns nil if the tree has no items.
func (tr *BTree) Min() interface{} {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil {
return nil
}
@ -508,6 +567,8 @@ func (tr *BTree) Min() interface{} {
// Max returns the maximum item in tree.
// Returns nil if the tree has no items.
func (tr *BTree) Max() interface{} {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root == nil {
return nil
}
@ -523,53 +584,65 @@ func (tr *BTree) Max() interface{} {
// PopMin removes the minimum item in tree and returns it.
// Returns nil if the tree has no items.
func (tr *BTree) PopMin() interface{} {
tr.mu.Lock()
defer tr.mu.Unlock()
if tr.root == nil {
return nil
}
tr.lnode = nil
n := tr.root
n := tr.cowLoad(&tr.root)
for {
if n.leaf {
item := n.items[0]
if n.numItems == minItems {
return tr.Delete(item)
return tr.deleteHint(item, nil)
}
copy(n.items[:], n.items[1:])
n.items[n.numItems-1] = nil
n.numItems--
tr.length--
if tr.length == 0 {
tr.root = nil
}
return item
}
n = n.children[0]
n = tr.cowLoad(&n.children[0])
}
}
// PopMax removes the minimum item in tree and returns it.
// Returns nil if the tree has no items.
func (tr *BTree) PopMax() interface{} {
tr.mu.Lock()
defer tr.mu.Unlock()
if tr.root == nil {
return nil
}
tr.lnode = nil
n := tr.root
n := tr.cowLoad(&tr.root)
for {
if n.leaf {
item := n.items[n.numItems-1]
if n.numItems == minItems {
return tr.Delete(item)
return tr.deleteHint(item, nil)
}
n.items[n.numItems-1] = nil
n.numItems--
tr.length--
if tr.length == 0 {
tr.root = nil
}
return item
}
n = n.children[n.numItems]
n = tr.cowLoad(&n.children[n.numItems])
}
}
// Height returns the height of the tree.
// Returns zero if tree has no items.
func (tr *BTree) Height() int {
tr.mu.RLock()
defer tr.mu.RUnlock()
var height int
if tr.root != nil {
n := tr.root
@ -587,6 +660,8 @@ func (tr *BTree) Height() int {
// Walk iterates over all items in tree, in order.
// The items param will contain one or more items.
func (tr *BTree) Walk(iter func(item []interface{})) {
tr.mu.RLock()
defer tr.mu.RUnlock()
if tr.root != nil {
tr.root.walk(iter)
}
@ -603,3 +678,16 @@ func (n *node) walk(iter func(item []interface{})) {
n.children[n.numItems].walk(iter)
}
}
// Copy the tree. This operation is very fast because it only performs a
// shadowed copy.
func (tr *BTree) Copy() *BTree {
tr.mu.Lock()
tr.lnode = nil
tr.cow = new(cow)
tr2 := *tr
tr2.mu = new(sync.RWMutex)
tr2.cow = new(cow)
tr.mu.Unlock()
return &tr2
}

View File

@ -1,4 +0,0 @@
language: go
go:
- 1.15.x

View File

@ -3,10 +3,9 @@
src="logo.png"
width="307" height="150" border="0" alt="BuntDB">
<br>
<a href="https://travis-ci.org/tidwall/buntdb"><img src="https://img.shields.io/travis/tidwall/buntdb.svg?style=flat-square" alt="Build Status"></a>
<a href="http://gocover.io/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/coverage-95%25-brightgreen.svg?style=flat-square" alt="Code Coverage"></a>
<a href="https://goreportcard.com/report/github.com/tidwall/buntdb"><img src="https://goreportcard.com/badge/github.com/tidwall/buntdb?style=flat-square" alt="Go Report Card"></a>
<a href="https://godoc.org/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
<a href="https://godoc.org/github.com/tidwall/buntdb"><img src="https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square" alt="Godoc"></a>
<a href="https://github.com/tidwall/buntdb/blob/master/LICENSE"><img src="https://img.shields.io/github/license/tidwall/buntdb.svg?style=flat-square" alt="LICENSE"></a>
</p>
BuntDB is a low-level, in-memory, key/value store in pure Go.

View File

@ -19,7 +19,7 @@ import (
"github.com/tidwall/gjson"
"github.com/tidwall/grect"
"github.com/tidwall/match"
"github.com/tidwall/rtree"
"github.com/tidwall/rtred"
)
var (
@ -69,7 +69,6 @@ type DB struct {
keys *btree.BTree // a tree of all item ordered by key
exps *btree.BTree // a tree of items ordered by expiration
idxs map[string]*index // the index trees.
exmgr bool // indicates that expires manager is running.
flushes int // a count of the number of disk flushes
closed bool // set when the database has been closed
config Config // the database configuration
@ -135,9 +134,6 @@ type exctx struct {
db *DB
}
// Default number of btree degrees
const btreeDegrees = 64
// Open opens a database at the provided path.
// If the file does not exist then it will be created automatically.
func Open(path string) (*DB, error) {
@ -241,14 +237,15 @@ func (db *DB) Load(rd io.Reader) error {
// cannot load into databases that persist to disk
return ErrPersistenceActive
}
return db.readLoad(rd, time.Now())
_, err := db.readLoad(rd, time.Now())
return err
}
// index represents a b-tree or r-tree index and also acts as the
// b-tree/r-tree context for itself.
type index struct {
btr *btree.BTree // contains the items
rtr *rtree.RTree // contains the items
rtr *rtred.RTree // contains the items
name string // name of the index
pattern string // a required key pattern
less func(a, b string) bool // less comparison function
@ -289,7 +286,7 @@ func (idx *index) clearCopy() *index {
nidx.btr = btree.New(lessCtx(nidx))
}
if nidx.rect != nil {
nidx.rtr = rtree.New(nidx)
nidx.rtr = rtred.New(nidx)
}
return nidx
}
@ -301,7 +298,7 @@ func (idx *index) rebuild() {
idx.btr = btree.New(lessCtx(idx))
}
if idx.rect != nil {
idx.rtr = rtree.New(idx)
idx.rtr = rtred.New(idx)
}
// iterate through all keys and fill the index
btreeAscend(idx.db.keys, func(item interface{}) bool {
@ -755,46 +752,65 @@ func (db *DB) Shrink() error {
}()
}
var errValidEOF = errors.New("valid eof")
// readLoad reads from the reader and loads commands into the database.
// modTime is the modified time of the reader, should be no greater than
// the current time.Now().
func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
// Returns the number of bytes of the last command read and the error if any.
func (db *DB) readLoad(rd io.Reader, modTime time.Time) (n int64, err error) {
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
totalSize := int64(0)
data := make([]byte, 4096)
parts := make([]string, 0, 8)
r := bufio.NewReader(rd)
for {
// peek at the first byte. If it's a 'nul' control character then
// ignore it and move to the next byte.
c, err := r.ReadByte()
if err != nil {
if err == io.EOF {
err = nil
}
return totalSize, err
}
if c == 0 {
// ignore nul control characters
n += 1
continue
}
if err := r.UnreadByte(); err != nil {
return totalSize, err
}
// read a single command.
// first we should read the number of parts that the of the command
cmdByteSize := int64(0)
line, err := r.ReadBytes('\n')
if err != nil {
if len(line) > 0 {
// got an eof but also data. this should be an unexpected eof.
return io.ErrUnexpectedEOF
}
if err == io.EOF {
break
}
return err
return totalSize, err
}
if line[0] != '*' {
return ErrInvalid
return totalSize, ErrInvalid
}
cmdByteSize += int64(len(line))
// convert the string number to and int
var n int
if len(line) == 4 && line[len(line)-2] == '\r' {
if line[1] < '0' || line[1] > '9' {
return ErrInvalid
return totalSize, ErrInvalid
}
n = int(line[1] - '0')
} else {
if len(line) < 5 || line[len(line)-2] != '\r' {
return ErrInvalid
return totalSize, ErrInvalid
}
for i := 1; i < len(line)-2; i++ {
if line[i] < '0' || line[i] > '9' {
return ErrInvalid
return totalSize, ErrInvalid
}
n = n*10 + int(line[i]-'0')
}
@ -805,25 +821,26 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
// read the number of bytes of the part.
line, err := r.ReadBytes('\n')
if err != nil {
return err
return totalSize, err
}
if line[0] != '$' {
return ErrInvalid
return totalSize, ErrInvalid
}
cmdByteSize += int64(len(line))
// convert the string number to and int
var n int
if len(line) == 4 && line[len(line)-2] == '\r' {
if line[1] < '0' || line[1] > '9' {
return ErrInvalid
return totalSize, ErrInvalid
}
n = int(line[1] - '0')
} else {
if len(line) < 5 || line[len(line)-2] != '\r' {
return ErrInvalid
return totalSize, ErrInvalid
}
for i := 1; i < len(line)-2; i++ {
if line[i] < '0' || line[i] > '9' {
return ErrInvalid
return totalSize, ErrInvalid
}
n = n*10 + int(line[i]-'0')
}
@ -837,13 +854,14 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
data = make([]byte, dataln)
}
if _, err = io.ReadFull(r, data[:n+2]); err != nil {
return err
return totalSize, err
}
if data[n] != '\r' || data[n+1] != '\n' {
return ErrInvalid
return totalSize, ErrInvalid
}
// copy string
parts = append(parts, string(data[:n]))
cmdByteSize += int64(n + 2)
}
// finished reading the command
@ -855,15 +873,15 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
(parts[0][2] == 't' || parts[0][2] == 'T') {
// SET
if len(parts) < 3 || len(parts) == 4 || len(parts) > 5 {
return ErrInvalid
return totalSize, ErrInvalid
}
if len(parts) == 5 {
if strings.ToLower(parts[3]) != "ex" {
return ErrInvalid
return totalSize, ErrInvalid
}
ex, err := strconv.ParseInt(parts[4], 10, 64)
ex, err := strconv.ParseUint(parts[4], 10, 64)
if err != nil {
return err
return totalSize, err
}
now := time.Now()
dur := (time.Duration(ex) * time.Second) - now.Sub(modTime)
@ -885,7 +903,7 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
(parts[0][2] == 'l' || parts[0][2] == 'L') {
// DEL
if len(parts) != 2 {
return ErrInvalid
return totalSize, ErrInvalid
}
db.deleteFromDatabase(&dbItem{key: parts[1]})
} else if (parts[0][0] == 'f' || parts[0][0] == 'F') &&
@ -894,10 +912,10 @@ func (db *DB) readLoad(rd io.Reader, modTime time.Time) error {
db.exps = btree.New(lessCtx(&exctx{db}))
db.idxs = make(map[string]*index)
} else {
return ErrInvalid
return totalSize, ErrInvalid
}
totalSize += cmdByteSize
}
return nil
}
// load reads entries from the append only database file and fills the database.
@ -910,10 +928,20 @@ func (db *DB) load() error {
if err != nil {
return err
}
if err := db.readLoad(db.file, fi.ModTime()); err != nil {
return err
n, err := db.readLoad(db.file, fi.ModTime())
if err != nil {
if err == io.ErrUnexpectedEOF {
// The db file has ended mid-command, which is allowed but the
// data file should be truncated to the end of the last valid
// command
if err := db.file.Truncate(n); err != nil {
return err
}
} else {
return err
}
}
pos, err := db.file.Seek(0, 2)
pos, err := db.file.Seek(n, 0)
if err != nil {
return err
}
@ -1148,7 +1176,25 @@ func (tx *Tx) Commit() error {
// Flushing the buffer only once per transaction.
// If this operation fails then the write did failed and we must
// rollback.
if _, err = tx.db.file.Write(tx.db.buf); err != nil {
var n int
n, err = tx.db.file.Write(tx.db.buf)
if err != nil {
if n > 0 {
// There was a partial write to disk.
// We are possibly out of disk space.
// Delete the partially written bytes from the data file by
// seeking to the previously known position and performing
// a truncate operation.
// At this point a syscall failure is fatal and the process
// should be killed to avoid corrupting the file.
pos, err := tx.db.file.Seek(-int64(n), 1)
if err != nil {
panic(err)
}
if err := tx.db.file.Truncate(pos); err != nil {
panic(err)
}
}
tx.rollbackInner()
}
if tx.db.config.SyncPolicy == Always {
@ -1216,7 +1262,7 @@ func appendBulkString(buf []byte, s string) []byte {
// writeSetTo writes an item as a single SET record to the a bufio Writer.
func (dbi *dbItem) writeSetTo(buf []byte) []byte {
if dbi.opts != nil && dbi.opts.ex {
ex := dbi.opts.exat.Sub(time.Now()) / time.Second
ex := time.Until(dbi.opts.exat) / time.Second
buf = appendArray(buf, 5)
buf = appendBulkString(buf, "set")
buf = appendBulkString(buf, dbi.key)
@ -1390,7 +1436,9 @@ func (tx *Tx) Set(key, value string, opts *SetOptions) (previousValue string,
// create a rollback entry with a nil value. A nil value indicates
// that the entry should be deleted on rollback. When the value is
// *not* nil, that means the entry should be reverted.
tx.wc.rollbackItems[key] = nil
if _, ok := tx.wc.rollbackItems[key]; !ok {
tx.wc.rollbackItems[key] = nil
}
} else {
// A previous item already exists in the database. Let's create a
// rollback entry with the item as the value. We need to check the
@ -1481,7 +1529,7 @@ func (tx *Tx) TTL(key string) (time.Duration, error) {
} else if item.opts == nil || !item.opts.ex {
return -1, nil
}
dur := item.opts.exat.Sub(time.Now())
dur := time.Until(item.opts.exat)
if dur < 0 {
return 0, ErrNotFound
}
@ -1822,7 +1870,7 @@ func (tx *Tx) Nearby(index, bounds string,
return nil
}
// // wrap a rtree specific iterator around the user-defined iterator.
iter := func(item rtree.Item, dist float64) bool {
iter := func(item rtred.Item, dist float64) bool {
dbi := item.(*dbItem)
return iterator(dbi.key, dbi.val, dist)
}
@ -1860,7 +1908,7 @@ func (tx *Tx) Intersects(index, bounds string,
return nil
}
// wrap a rtree specific iterator around the user-defined iterator.
iter := func(item rtree.Item) bool {
iter := func(item rtred.Item) bool {
dbi := item.(*dbItem)
return iterator(dbi.key, dbi.val)
}

View File

@ -1,12 +1,12 @@
module github.com/tidwall/buntdb
go 1.15
go 1.16
require (
github.com/tidwall/btree v0.2.2
github.com/tidwall/gjson v1.6.1
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb
github.com/tidwall/match v1.0.1
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect
github.com/tidwall/btree v0.4.2
github.com/tidwall/gjson v1.7.4
github.com/tidwall/grect v0.1.1
github.com/tidwall/lotsa v1.0.2
github.com/tidwall/match v1.0.3
github.com/tidwall/rtred v0.1.2
)

View File

@ -1,14 +1,16 @@
github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY=
github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb h1:5NSYaAdrnblKByzd7XByQEJVT8+9v0W/tIY0Oo4OwrE=
github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb/go.mod h1:lKYYLFIr9OIgdgrtgkZ9zgRxRdvPYsExnYBsEAd8W5M=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 h1:BsKSRhu0TDB6Snq8SutN9KQHc6vqHEXJTcAFwyGNius=
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8/go.mod h1:/h+UnNGt0IhNNJLkGikcdcJqm66zGD/uJGMRxK/9+Ao=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 h1:Otn9S136ELckZ3KKDyCkxapfufrqDqwmGjcHfAyXRrE=
github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563/go.mod h1:mLqSmt7Dv/CNneF2wfcChfN1rvapyQr01LGKnKex0DQ=
github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs=
github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8=
github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
github.com/tidwall/grect v0.1.1 h1:+kMEkxhoqB7rniVXzMEIA66XwU07STgINqxh+qVIndY=
github.com/tidwall/grect v0.1.1/go.mod h1:CzvbGiFbWUwiJ1JohXLb28McpyBsI00TK9Y6pDWLGRQ=
github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=

View File

@ -1 +0,0 @@
language: go

View File

@ -3,13 +3,10 @@
src="logo.png"
width="240" height="78" border="0" alt="GJSON">
<br>
<a href="https://travis-ci.org/tidwall/gjson"><img src="https://img.shields.io/travis/tidwall/gjson.svg?style=flat-square" alt="Build Status"></a>
<a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
<a href="http://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a>
</p>
<p align="center">get json values quickly</a></p>
GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
@ -17,6 +14,8 @@ It has features such as [one line retrieval](#get-a-value), [dot notation paths]
Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool.
For the Rust version go to [gjson.rs](https://github.com/tidwall/gjson.rs).
Getting Started
===============
@ -476,7 +475,7 @@ JSON document used:
}
```
Each operation was rotated though one of the following search paths:
Each operation was rotated through one of the following search paths:
```
widget.window.name
@ -484,12 +483,4 @@ widget.image.hOffset
widget.text.onMouseUp
```
*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).*
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
## License
GJSON source code is available under the MIT [License](/LICENSE).
*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be found [here](https://github.com/tidwall/gjson-benchmarks).*

View File

@ -77,14 +77,21 @@ Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`.
fav\.movie "Deer Hunter"
```
You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in source code.
You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in you source code.
```go
res := gjson.Get(json, "fav\\.movie") // must escape the slash
res := gjson.Get(json, `fav\.movie`) // no need to escape the slash
// Go
val := gjson.Get(json, "fav\\.movie") // must escape the slash
val := gjson.Get(json, `fav\.movie`) // no need to escape the slash
```
```rust
// Rust
let val = gjson::get(json, "fav\\.movie") // must escape the slash
let val = gjson::get(json, r#"fav\.movie"#) // no need to escape the slash
```
### Arrays
The `#` character allows for digging into JSON Arrays.
@ -248,6 +255,8 @@ gjson.AddModifier("case", func(json, arg string) string {
"children.@case:lower.@reverse" ["jack","alex","sara"]
```
*Note: Custom modifiers are not yet available in the Rust version*
### Multipaths
Starting with v1.3.0, GJSON added the ability to join multiple paths together

View File

@ -3,7 +3,6 @@ package gjson
import (
"encoding/json"
"reflect"
"strconv"
"strings"
"time"
@ -106,7 +105,8 @@ func (t Result) Bool() bool {
case True:
return true
case String:
return t.Str != "" && t.Str != "0" && t.Str != "false"
b, _ := strconv.ParseBool(strings.ToLower(t.Str))
return b
case Number:
return t.Num != 0
}
@ -124,16 +124,17 @@ func (t Result) Int() int64 {
return n
case Number:
// try to directly convert the float64 to int64
n, ok := floatToInt(t.Num)
if !ok {
// now try to parse the raw string
n, ok = parseInt(t.Raw)
if !ok {
// fallback to a standard conversion
return int64(t.Num)
}
i, ok := safeInt(t.Num)
if ok {
return i
}
return n
// now try to parse the raw string
i, ok = parseInt(t.Raw)
if ok {
return i
}
// fallback to a standard conversion
return int64(t.Num)
}
}
@ -149,16 +150,17 @@ func (t Result) Uint() uint64 {
return n
case Number:
// try to directly convert the float64 to uint64
n, ok := floatToUint(t.Num)
if !ok {
// now try to parse the raw string
n, ok = parseUint(t.Raw)
if !ok {
// fallback to a standard conversion
return uint64(t.Num)
}
i, ok := safeInt(t.Num)
if ok && i >= 0 {
return uint64(i)
}
return n
// now try to parse the raw string
u, ok := parseUint(t.Raw)
if ok {
return u
}
// fallback to a standard conversion
return uint64(t.Num)
}
}
@ -495,6 +497,9 @@ func squash(json string) string {
}
}
if depth == 0 {
if i >= len(json) {
return json
}
return json[:i+1]
}
case '{', '[', '(':
@ -726,8 +731,13 @@ func parseArrayPath(path string) (r arrayPathResult) {
}
if path[i] == '.' {
r.part = path[:i]
r.path = path[i+1:]
r.more = true
if !r.arrch && i < len(path)-1 && isDotPiperChar(path[i+1]) {
r.pipe = path[i+1:]
r.piped = true
} else {
r.path = path[i+1:]
r.more = true
}
return
}
if path[i] == '#' {
@ -973,6 +983,11 @@ right:
return s
}
// peek at the next byte and see if it's a '@', '[', or '{'.
func isDotPiperChar(c byte) bool {
return !DisableModifiers && (c == '@' || c == '[' || c == '{')
}
type objectPathResult struct {
part string
path string
@ -991,12 +1006,8 @@ func parseObjectPath(path string) (r objectPathResult) {
return
}
if path[i] == '.' {
// peek at the next byte and see if it's a '@', '[', or '{'.
r.part = path[:i]
if !DisableModifiers &&
i < len(path)-1 &&
(path[i+1] == '@' ||
path[i+1] == '[' || path[i+1] == '{') {
if i < len(path)-1 && isDotPiperChar(path[i+1]) {
r.pipe = path[i+1:]
r.piped = true
} else {
@ -1026,14 +1037,11 @@ func parseObjectPath(path string) (r objectPathResult) {
continue
} else if path[i] == '.' {
r.part = string(epart)
// peek at the next byte and see if it's a '@' modifier
if !DisableModifiers &&
i < len(path)-1 && path[i+1] == '@' {
if i < len(path)-1 && isDotPiperChar(path[i+1]) {
r.pipe = path[i+1:]
r.piped = true
} else {
r.path = path[i+1:]
r.more = true
}
r.more = true
return
@ -1398,7 +1406,6 @@ func parseArray(c *parseContext, i int, path string) (int, bool) {
}
return false
}
for i < len(c.json)+1 {
if !rp.arrch {
pmatch = partidx == h
@ -1600,10 +1607,17 @@ func parseArray(c *parseContext, i int, path string) (int, bool) {
c.calcd = true
return i + 1, true
}
if len(multires) > 0 && !c.value.Exists() {
c.value = Result{
Raw: string(append(multires, ']')),
Type: JSON,
if !c.value.Exists() {
if len(multires) > 0 {
c.value = Result{
Raw: string(append(multires, ']')),
Type: JSON,
}
} else if rp.query.all {
c.value = Result{
Raw: "[]",
Type: JSON,
}
}
}
return i + 1, false
@ -1835,7 +1849,7 @@ type parseContext struct {
// A path is in dot syntax, such as "name.last" or "age".
// When the value is found it's returned immediately.
//
// A path is a series of keys searated by a dot.
// A path is a series of keys separated by a dot.
// A key may contain special wildcard characters '*' and '?'.
// To access an array value use the index as the key.
// To get the number of elements in an array or to access a child path, use
@ -1944,7 +1958,6 @@ func Get(json, path string) Result {
}
}
}
var i int
var c = &parseContext{json: json}
if len(path) >= 2 && path[0] == '.' && path[1] == '.' {
@ -2169,11 +2182,6 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) {
return i, res, false
}
var ( // used for testing
testWatchForFallback bool
testLastWasFallback bool
)
// GetMany searches json for the multiple paths.
// The return value is a Result array where the number of items
// will be equal to the number of input paths.
@ -2374,6 +2382,12 @@ func validnumber(data []byte, i int) (outi int, ok bool) {
// sign
if data[i] == '-' {
i++
if i == len(data) {
return i, false
}
if data[i] < '0' || data[i] > '9' {
return i, false
}
}
// int
if i == len(data) {
@ -2524,25 +2538,14 @@ func parseInt(s string) (n int64, ok bool) {
return n, true
}
const minUint53 = 0
const maxUint53 = 4503599627370495
const minInt53 = -2251799813685248
const maxInt53 = 2251799813685247
func floatToUint(f float64) (n uint64, ok bool) {
n = uint64(f)
if float64(n) == f && n >= minUint53 && n <= maxUint53 {
return n, true
// safeInt validates a given JSON number
// ensures it lies within the minimum and maximum representable JSON numbers
func safeInt(f float64) (n int64, ok bool) {
// https://tc39.es/ecma262/#sec-number.min_safe_integer || https://tc39.es/ecma262/#sec-number.max_safe_integer
if f < -9007199254740991 || f > 9007199254740991 {
return 0, false
}
return 0, false
}
func floatToInt(f float64) (n int64, ok bool) {
n = int64(f)
if float64(n) == f && n >= minInt53 && n <= maxInt53 {
return n, true
}
return 0, false
return int64(f), true
}
// execModifier parses the path to find a matching modifier function.
@ -2600,7 +2603,7 @@ func execModifier(json, path string) (pathOut, res string, ok bool) {
// unwrap removes the '[]' or '{}' characters around json
func unwrap(json string) string {
json = trim(json)
if len(json) >= 2 && json[0] == '[' || json[0] == '{' {
if len(json) >= 2 && (json[0] == '[' || json[0] == '{') {
json = json[1 : len(json)-1]
}
return json
@ -2632,6 +2635,26 @@ func ModifierExists(name string, fn func(json, arg string) string) bool {
return ok
}
// cleanWS remove any non-whitespace from string
func cleanWS(s string) string {
for i := 0; i < len(s); i++ {
switch s[i] {
case ' ', '\t', '\n', '\r':
continue
default:
var s2 []byte
for i := 0; i < len(s); i++ {
switch s[i] {
case ' ', '\t', '\n', '\r':
s2 = append(s2, s[i])
}
}
return string(s2)
}
}
return s
}
// @pretty modifier makes the json look nice.
func modPretty(json, arg string) string {
if len(arg) > 0 {
@ -2641,9 +2664,9 @@ func modPretty(json, arg string) string {
case "sortKeys":
opts.SortKeys = value.Bool()
case "indent":
opts.Indent = value.String()
opts.Indent = cleanWS(value.String())
case "prefix":
opts.Prefix = value.String()
opts.Prefix = cleanWS(value.String())
case "width":
opts.Width = int(value.Int())
}
@ -2729,19 +2752,24 @@ func modFlatten(json, arg string) string {
out = append(out, '[')
var idx int
res.ForEach(func(_, value Result) bool {
if idx > 0 {
out = append(out, ',')
}
var raw string
if value.IsArray() {
if deep {
out = append(out, unwrap(modFlatten(value.Raw, arg))...)
raw = unwrap(modFlatten(value.Raw, arg))
} else {
out = append(out, unwrap(value.Raw)...)
raw = unwrap(value.Raw)
}
} else {
out = append(out, value.Raw...)
raw = value.Raw
}
raw = strings.TrimSpace(raw)
if len(raw) > 0 {
if idx > 0 {
out = append(out, ',')
}
out = append(out, raw...)
idx++
}
idx++
return true
})
out = append(out, ']')
@ -2825,6 +2853,19 @@ func modValid(json, arg string) string {
return json
}
// stringHeader instead of reflect.StringHeader
type stringHeader struct {
data unsafe.Pointer
len int
}
// sliceHeader instead of reflect.SliceHeader
type sliceHeader struct {
data unsafe.Pointer
len int
cap int
}
// getBytes casts the input json bytes to a string and safely returns the
// results as uniquely allocated data. This operation is intended to minimize
// copies and allocations for the large json string->[]byte.
@ -2834,14 +2875,14 @@ func getBytes(json []byte, path string) Result {
// unsafe cast to string
result = Get(*(*string)(unsafe.Pointer(&json)), path)
// safely get the string headers
rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw))
strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str))
rawhi := *(*stringHeader)(unsafe.Pointer(&result.Raw))
strhi := *(*stringHeader)(unsafe.Pointer(&result.Str))
// create byte slice headers
rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len}
strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len}
if strh.Data == 0 {
rawh := sliceHeader{data: rawhi.data, len: rawhi.len, cap: rawhi.len}
strh := sliceHeader{data: strhi.data, len: strhi.len, cap: rawhi.len}
if strh.data == nil {
// str is nil
if rawh.Data == 0 {
if rawh.data == nil {
// raw is nil
result.Raw = ""
} else {
@ -2849,19 +2890,20 @@ func getBytes(json []byte, path string) Result {
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
}
result.Str = ""
} else if rawh.Data == 0 {
} else if rawh.data == nil {
// raw is nil
result.Raw = ""
// str has data, safely copy the slice header to a string
result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
} else if strh.Data >= rawh.Data &&
int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len {
} else if uintptr(strh.data) >= uintptr(rawh.data) &&
uintptr(strh.data)+uintptr(strh.len) <=
uintptr(rawh.data)+uintptr(rawh.len) {
// Str is a substring of Raw.
start := int(strh.Data - rawh.Data)
start := uintptr(strh.data) - uintptr(rawh.data)
// safely copy the raw slice header
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
// substring the raw
result.Str = result.Raw[start : start+strh.Len]
result.Str = result.Raw[start : start+uintptr(strh.len)]
} else {
// safely copy both the raw and str slice headers to strings
result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
@ -2876,9 +2918,9 @@ func getBytes(json []byte, path string) Result {
// used instead.
func fillIndex(json string, c *parseContext) {
if len(c.value.Raw) > 0 && !c.calcd {
jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json))
rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw)))
c.value.Index = int(rhdr.Data - jhdr.Data)
jhdr := *(*stringHeader)(unsafe.Pointer(&json))
rhdr := *(*stringHeader)(unsafe.Pointer(&(c.value.Raw)))
c.value.Index = int(uintptr(rhdr.data) - uintptr(jhdr.data))
if c.value.Index < 0 || c.value.Index >= len(json) {
c.value.Index = 0
}
@ -2886,10 +2928,10 @@ func fillIndex(json string, c *parseContext) {
}
func stringBytes(s string) []byte {
return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{
Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data,
Len: len(s),
Cap: len(s),
return *(*[]byte)(unsafe.Pointer(&sliceHeader{
data: (*stringHeader)(unsafe.Pointer(&s)).data,
len: len(s),
cap: len(s),
}))
}

View File

@ -3,6 +3,6 @@ module github.com/tidwall/gjson
go 1.12
require (
github.com/tidwall/match v1.0.1
github.com/tidwall/pretty v1.0.2
github.com/tidwall/match v1.0.3
github.com/tidwall/pretty v1.1.0
)

View File

@ -1,4 +1,4 @@
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

5
vendor/github.com/tidwall/grect/go.mod generated vendored Normal file
View File

@ -0,0 +1,5 @@
module github.com/tidwall/grect
go 1.15
require github.com/tidwall/gjson v1.7.4

6
vendor/github.com/tidwall/grect/go.sum generated vendored Normal file
View File

@ -0,0 +1,6 @@
github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8=
github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

View File

@ -1 +0,0 @@
language: go

View File

@ -1,20 +1,17 @@
Match
=====
<a href="https://travis-ci.org/tidwall/match"><img src="https://img.shields.io/travis/tidwall/match.svg?style=flat-square" alt="Build Status"></a>
<a href="https://godoc.org/github.com/tidwall/match"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
# Match
[![GoDoc](https://godoc.org/github.com/tidwall/match?status.svg)](https://godoc.org/github.com/tidwall/match)
Match is a very simple pattern matcher where '*' matches on any
number characters and '?' matches on any one character.
Installing
----------
## Installing
```
go get -u github.com/tidwall/match
```
Example
-------
## Example
```go
match.Match("hello", "*llo")
@ -23,10 +20,10 @@ match.Match("hello", "h*o")
```
Contact
-------
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
License
-------
## License
Redcon source code is available under the MIT [License](/LICENSE).

3
vendor/github.com/tidwall/match/go.mod generated vendored Normal file
View File

@ -0,0 +1,3 @@
module github.com/tidwall/match
go 1.15

View File

@ -1,4 +1,4 @@
// Match provides a simple pattern matcher with unicode support.
// Package match provides a simple pattern matcher with unicode support.
package match
import "unicode/utf8"
@ -6,7 +6,7 @@ import "unicode/utf8"
// Match returns true if str matches pattern. This is a very
// simple wildcard match where '*' matches on any number characters
// and '?' matches on any one character.
//
// pattern:
// { term }
// term:
@ -16,12 +16,16 @@ import "unicode/utf8"
// '\\' c matches character c
//
func Match(str, pattern string) bool {
return deepMatch(str, pattern)
}
func deepMatch(str, pattern string) bool {
if pattern == "*" {
return true
}
return deepMatch(str, pattern)
}
func deepMatch(str, pattern string) bool {
for len(pattern) > 1 && pattern[0] == '*' && pattern[1] == '*' {
pattern = pattern[1:]
}
for len(pattern) > 0 {
if pattern[0] > 0x7f {
return deepMatchRune(str, pattern)
@ -52,6 +56,13 @@ func deepMatch(str, pattern string) bool {
}
func deepMatchRune(str, pattern string) bool {
if pattern == "*" {
return true
}
for len(pattern) > 1 && pattern[0] == '*' && pattern[1] == '*' {
pattern = pattern[1:]
}
var sr, pr rune
var srsz, prsz int

View File

@ -1 +0,0 @@
language: go

View File

@ -1,8 +1,6 @@
# Pretty
[![Build Status](https://img.shields.io/travis/tidwall/pretty.svg?style=flat-square)](https://travis-ci.org/tidwall/prettty)
[![Coverage Status](https://img.shields.io/badge/coverage-100%25-brightgreen.svg?style=flat-square)](http://gocover.io/github.com/tidwall/pretty)
[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty)
[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty)
Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads.
@ -81,6 +79,45 @@ Will format the json to:
{"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}```
```
## Spec
Spec cleans comments and trailing commas from input JSON, converting it to
valid JSON per the official spec: https://tools.ietf.org/html/rfc8259
The resulting JSON will always be the same length as the input and it will
include all of the same line breaks at matching offsets. This is to ensure
the result can be later processed by a external parser and that that
parser will report messages or errors with the correct offsets.
The following example uses a JSON document that has comments and trailing
commas and converts it prior to unmarshalling to using the standard Go
JSON library.
```go
data := `
{
/* Dev Machine */
"dbInfo": {
"host": "localhost",
"port": 5432, // use full email address
"username": "josh",
"password": "pass123", // use a hashed password
}
/* Only SMTP Allowed */
"emailInfo": {
"email": "josh@example.com",
"password": "pass123",
"smtp": "smpt.example.com",
}
}
`
err := json.Unmarshal(pretty.Spec(data), &config)
```
## Customized output

View File

@ -118,21 +118,27 @@ type pair struct {
vstart, vend int
}
type byKey struct {
type byKeyVal struct {
sorted bool
json []byte
pairs []pair
}
func (arr *byKey) Len() int {
func (arr *byKeyVal) Len() int {
return len(arr.pairs)
}
func (arr *byKey) Less(i, j int) bool {
func (arr *byKeyVal) Less(i, j int) bool {
key1 := arr.json[arr.pairs[i].kstart+1 : arr.pairs[i].kend-1]
key2 := arr.json[arr.pairs[j].kstart+1 : arr.pairs[j].kend-1]
return string(key1) < string(key2)
if string(key1) < string(key2) {
return true
}
if string(key1) > string(key2) {
return false
}
return arr.pairs[i].vstart < arr.pairs[j].vstart
}
func (arr *byKey) Swap(i, j int) {
func (arr *byKeyVal) Swap(i, j int) {
arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i]
arr.sorted = true
}
@ -174,7 +180,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool,
}
if n > 0 {
nl = len(buf)
buf = append(buf, '\n')
if buf[nl-1] == ' ' {
buf[nl-1] = '\n'
} else {
buf = append(buf, '\n')
}
}
if buf[len(buf)-1] != open {
buf = appendTabs(buf, prefix, indent, tabs)
@ -193,7 +203,11 @@ func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool,
var p pair
if pretty {
nl = len(buf)
buf = append(buf, '\n')
if buf[nl-1] == ' ' {
buf[nl-1] = '\n'
} else {
buf = append(buf, '\n')
}
if open == '{' && sortkeys {
p.kstart = i
p.vstart = len(buf)
@ -235,8 +249,8 @@ func sortPairs(json, buf []byte, pairs []pair) []byte {
}
vstart := pairs[0].vstart
vend := pairs[len(pairs)-1].vend
arr := byKey{false, json, pairs}
sort.Sort(&arr)
arr := byKeyVal{false, json, pairs}
sort.Stable(&arr)
if !arr.sorted {
return buf
}
@ -305,6 +319,7 @@ func appendTabs(buf []byte, prefix, indent string, tabs int) []byte {
type Style struct {
Key, String, Number [2]string
True, False, Null [2]string
Escape [2]string
Append func(dst []byte, c byte) []byte
}
@ -328,6 +343,7 @@ func init() {
True: [2]string{"\x1B[96m", "\x1B[0m"},
False: [2]string{"\x1B[96m", "\x1B[0m"},
Null: [2]string{"\x1B[91m", "\x1B[0m"},
Escape: [2]string{"\x1B[35m", "\x1B[0m"},
Append: func(dst []byte, c byte) []byte {
if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
dst = append(dst, "\\u00"...)
@ -367,8 +383,39 @@ func Color(src []byte, style *Style) []byte {
dst = append(dst, style.String[0]...)
}
dst = apnd(dst, '"')
esc := false
uesc := 0
for i = i + 1; i < len(src); i++ {
dst = apnd(dst, src[i])
if src[i] == '\\' {
if key {
dst = append(dst, style.Key[1]...)
} else {
dst = append(dst, style.String[1]...)
}
dst = append(dst, style.Escape[0]...)
dst = apnd(dst, src[i])
esc = true
if i+1 < len(src) && src[i+1] == 'u' {
uesc = 5
} else {
uesc = 1
}
} else if esc {
dst = apnd(dst, src[i])
if uesc == 1 {
esc = false
dst = append(dst, style.Escape[1]...)
if key {
dst = append(dst, style.Key[0]...)
} else {
dst = append(dst, style.String[0]...)
}
} else {
uesc--
}
} else {
dst = apnd(dst, src[i])
}
if src[i] == '"' {
j := i - 1
for ; ; j-- {
@ -381,7 +428,9 @@ func Color(src []byte, style *Style) []byte {
}
}
}
if key {
if esc {
dst = append(dst, style.Escape[1]...)
} else if key {
dst = append(dst, style.Key[1]...)
} else {
dst = append(dst, style.String[1]...)
@ -434,3 +483,90 @@ func Color(src []byte, style *Style) []byte {
}
return dst
}
// Spec strips out comments and trailing commas and convert the input to a
// valid JSON per the official spec: https://tools.ietf.org/html/rfc8259
//
// The resulting JSON will always be the same length as the input and it will
// include all of the same line breaks at matching offsets. This is to ensure
// the result can be later processed by a external parser and that that
// parser will report messages or errors with the correct offsets.
func Spec(src []byte) []byte {
return spec(src, nil)
}
// SpecInPlace is the same as Spec, but this method reuses the input json
// buffer to avoid allocations. Do not use the original bytes slice upon return.
func SpecInPlace(src []byte) []byte {
return spec(src, src)
}
func spec(src, dst []byte) []byte {
dst = dst[:0]
for i := 0; i < len(src); i++ {
if src[i] == '/' {
if i < len(src)-1 {
if src[i+1] == '/' {
dst = append(dst, ' ', ' ')
i += 2
for ; i < len(src); i++ {
if src[i] == '\n' {
dst = append(dst, '\n')
break
} else if src[i] == '\t' || src[i] == '\r' {
dst = append(dst, src[i])
} else {
dst = append(dst, ' ')
}
}
continue
}
if src[i+1] == '*' {
dst = append(dst, ' ', ' ')
i += 2
for ; i < len(src)-1; i++ {
if src[i] == '*' && src[i+1] == '/' {
dst = append(dst, ' ', ' ')
i++
break
} else if src[i] == '\n' || src[i] == '\t' ||
src[i] == '\r' {
dst = append(dst, src[i])
} else {
dst = append(dst, ' ')
}
}
continue
}
}
}
dst = append(dst, src[i])
if src[i] == '"' {
for i = i + 1; i < len(src); i++ {
dst = append(dst, src[i])
if src[i] == '"' {
j := i - 1
for ; ; j-- {
if src[j] != '\\' {
break
}
}
if (j-i)%2 != 0 {
break
}
}
}
} else if src[i] == '}' || src[i] == ']' {
for j := len(dst) - 2; j >= 0; j-- {
if dst[j] <= ' ' {
continue
}
if dst[j] == ',' {
dst[j] = ' '
}
break
}
}
}
return dst
}

View File

@ -1,7 +1,6 @@
RTree implementation for Go
===========================
[![Build Status](https://travis-ci.org/tidwall/rtree.svg?branch=master)](https://travis-ci.org/tidwall/rtree)
[![GoDoc](https://godoc.org/github.com/tidwall/rtree?status.svg)](https://godoc.org/github.com/tidwall/rtree)
This package provides an in-memory R-Tree implementation for Go, useful as a spatial data structure.

5
vendor/github.com/tidwall/rtred/go.mod generated vendored Normal file
View File

@ -0,0 +1,5 @@
module github.com/tidwall/rtred
go 1.15
require github.com/tidwall/tinyqueue v0.1.1

2
vendor/github.com/tidwall/rtred/go.sum generated vendored Normal file
View File

@ -0,0 +1,2 @@
github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=

View File

@ -1,10 +1,10 @@
package rtree
package rtred
import (
"math"
"sync"
"github.com/tidwall/rtree/base"
"github.com/tidwall/rtred/base"
)
type Iterator func(item Item) bool

View File

@ -1 +0,0 @@
language: go

3
vendor/github.com/tidwall/tinyqueue/go.mod generated vendored Normal file
View File

@ -0,0 +1,3 @@
module github.com/tidwall/tinyqueue
go 1.15

20
vendor/modules.txt vendored
View File

@ -39,23 +39,25 @@ github.com/oragono/confusables
github.com/oragono/go-ident
# github.com/stretchr/testify v1.4.0
## explicit
# github.com/tidwall/btree v0.2.2
# github.com/tidwall/btree v0.4.2
github.com/tidwall/btree
# github.com/tidwall/buntdb v1.1.4
# github.com/tidwall/buntdb v1.2.3
## explicit
github.com/tidwall/buntdb
# github.com/tidwall/gjson v1.6.1
# github.com/tidwall/gjson v1.7.4
github.com/tidwall/gjson
# github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb
# github.com/tidwall/grect v0.1.1
github.com/tidwall/grect
# github.com/tidwall/match v1.0.1
# github.com/tidwall/match v1.0.3
github.com/tidwall/match
# github.com/tidwall/pretty v1.0.2
# github.com/tidwall/pretty v1.1.0
github.com/tidwall/pretty
# github.com/tidwall/rtred v0.1.2
github.com/tidwall/rtred
github.com/tidwall/rtred/base
# github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8
github.com/tidwall/rtree
github.com/tidwall/rtree/base
# github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563
## explicit
# github.com/tidwall/tinyqueue v0.1.1
github.com/tidwall/tinyqueue
# github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208
## explicit