3
0
mirror of https://github.com/ergochat/ergo.git synced 2024-11-15 00:19:29 +01:00

upgrade buntdb

This commit is contained in:
Shivaram Lingamneni 2022-06-16 13:36:02 -04:00
parent 2138847984
commit 86f124e938
11 changed files with 1760 additions and 865 deletions

8
go.mod
View File

@ -17,7 +17,7 @@ require (
github.com/onsi/ginkgo v1.12.0 // indirect github.com/onsi/ginkgo v1.12.0 // indirect
github.com/onsi/gomega v1.9.0 // indirect github.com/onsi/gomega v1.9.0 // indirect
github.com/stretchr/testify v1.4.0 // indirect github.com/stretchr/testify v1.4.0 // indirect
github.com/tidwall/buntdb v1.2.7 github.com/tidwall/buntdb v1.2.9
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208
github.com/xdg-go/scram v1.0.2 github.com/xdg-go/scram v1.0.2
golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8 golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8
@ -28,9 +28,9 @@ require (
require github.com/gofrs/flock v0.8.1 require github.com/gofrs/flock v0.8.1
require ( require (
github.com/tidwall/btree v0.6.1 // indirect github.com/tidwall/btree v1.1.0 // indirect
github.com/tidwall/gjson v1.10.2 // indirect github.com/tidwall/gjson v1.12.1 // indirect
github.com/tidwall/grect v0.1.3 // indirect github.com/tidwall/grect v0.1.4 // indirect
github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/rtred v0.1.2 // indirect

8
go.sum
View File

@ -47,12 +47,20 @@ github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
github.com/tidwall/btree v0.6.1 h1:75VVgBeviiDO+3g4U+7+BaNBNhNINxB0ULPT3fs9pMY= github.com/tidwall/btree v0.6.1 h1:75VVgBeviiDO+3g4U+7+BaNBNhNINxB0ULPT3fs9pMY=
github.com/tidwall/btree v0.6.1/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= github.com/tidwall/btree v0.6.1/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4=
github.com/tidwall/btree v1.1.0 h1:5P+9WU8ui5uhmcg3SoPyTwoI0mVyZ1nps7YQzTZFkYM=
github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4=
github.com/tidwall/buntdb v1.2.7 h1:SIyObKAymzLyGhDeIhVk2Yc1/EwfCC75Uyu77CHlVoA= github.com/tidwall/buntdb v1.2.7 h1:SIyObKAymzLyGhDeIhVk2Yc1/EwfCC75Uyu77CHlVoA=
github.com/tidwall/buntdb v1.2.7/go.mod h1:b6KvZM27x/8JLI5hgRhRu60pa3q0Tz9c50TyD46OHUM= github.com/tidwall/buntdb v1.2.7/go.mod h1:b6KvZM27x/8JLI5hgRhRu60pa3q0Tz9c50TyD46OHUM=
github.com/tidwall/buntdb v1.2.9 h1:XVz684P7X6HCTrdr385yDZWB1zt/n20ZNG3M1iGyFm4=
github.com/tidwall/buntdb v1.2.9/go.mod h1:IwyGSvvDg6hnKSIhtdZ0AqhCZGH8ukdtCAzaP8fI1X4=
github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo= github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo=
github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/grect v0.1.3 h1:z9YwQAMUxVSBde3b7Sl8Da37rffgNfZ6Fq6h9t6KdXE= github.com/tidwall/grect v0.1.3 h1:z9YwQAMUxVSBde3b7Sl8Da37rffgNfZ6Fq6h9t6KdXE=
github.com/tidwall/grect v0.1.3/go.mod h1:8GMjwh3gPZVpLBI/jDz9uslCe0dpxRpWDdtN0lWAS/E= github.com/tidwall/grect v0.1.3/go.mod h1:8GMjwh3gPZVpLBI/jDz9uslCe0dpxRpWDdtN0lWAS/E=
github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=

View File

@ -26,7 +26,7 @@ Take the first example image. The item 9 is at path “1/0”. The item 16 is at
A Path Hint is a predefined path that is provided to B-tree operations. Its just a hint that says, “Hey B-tree, instead of starting your binary search with the middle index, start with what I provide you. My path may be wrong, and if so please provide me with the correct path so I get it right the next time.” A Path Hint is a predefined path that is provided to B-tree operations. Its just a hint that says, “Hey B-tree, instead of starting your binary search with the middle index, start with what I provide you. My path may be wrong, and if so please provide me with the correct path so I get it right the next time.”
Ive found using path hints can lead to a little performance increase of 150% - 300%. This is because in real-world cases the items that Im working with are usually nearby each other in the tree. Ive found using path hints can lead to a little performance increase of 150% - 300%. This is because in real-world cases the items that Im working with are usually nearby each other in the tree.
Take for example inserting a group of timeseries points. They may often be received as chucks of near-contiguous items. Take for example inserting a group of timeseries points. They may often be received as chucks of near-contiguous items.
Or, I'm sequentially inserting an ordered group of rows somewhere in the middle of a table. Or, I'm sequentially inserting an ordered group of rows somewhere in the middle of a table.
@ -37,6 +37,8 @@ While I may see a 3x boost in when the path hint is right on, I'll only see arou
## Using a Path Hint ## Using a Path Hint
All of the functions that take in a path hint argument mutate the path hint argument.
For single-threaded programs, its possible to use one shared path hint per B-tree for the life of the program. For single-threaded programs, its possible to use one shared path hint per B-tree for the life of the program.
For multi-threaded programs, I find it best to use one path hint per B-tree , per thread. For multi-threaded programs, I find it best to use one path hint per B-tree , per thread.
For server-client programs, one path hint per B-tree, per client should suffice. For server-client programs, one path hint per B-tree, per client should suffice.

View File

@ -4,6 +4,8 @@
An [efficient](#performance) [B-tree](https://en.wikipedia.org/wiki/B-tree) implementation in Go. An [efficient](#performance) [B-tree](https://en.wikipedia.org/wiki/B-tree) implementation in Go.
*Check out the [generics branch](https://github.com/tidwall/btree/tree/generics) if you want to try out btree with generic support for Go 1.18+*
## Features ## Features
- `Copy()` method with copy-on-write support. - `Copy()` method with copy-on-write support.
@ -116,10 +118,10 @@ func main() {
### Basic ### Basic
``` ```
Len() # return the number of items in the btree
Set(item) # insert or replace an existing item
Get(item) # get an existing item Get(item) # get an existing item
Set(item) # insert or replace an existing item
Delete(item) # delete an item Delete(item) # delete an item
Len() # return the number of items in the btree
``` ```
### Iteration ### Iteration
@ -127,6 +129,7 @@ Delete(item) # delete an item
``` ```
Ascend(pivot, iter) # scan items in ascending order starting at pivot. Ascend(pivot, iter) # scan items in ascending order starting at pivot.
Descend(pivot, iter) # scan items in descending order starting at pivot. Descend(pivot, iter) # scan items in descending order starting at pivot.
Iter() # returns a read-only iterator for for-loops.
``` ```
### Queues ### Queues
@ -151,11 +154,18 @@ GetHint(item, *hint) # get an existing item
DeleteHint(item, *hint) # delete an item DeleteHint(item, *hint) # delete an item
``` ```
### Array-like operations
```
GetAt(index) # returns the value at index
DeleteAt(index) # deletes the item at index
```
## Performance ## Performance
This implementation was designed with performance in mind. This implementation was designed with performance in mind.
The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel Core i9) using Go 1.16.5. The items are simple 8-byte ints. The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel Core i9) using Go 1.17.3. The items are simple 8-byte ints.
- `google`: The [google/btree](https://github.com/google/btree) package - `google`: The [google/btree](https://github.com/google/btree) package
- `tidwall`: The [tidwall/btree](https://github.com/tidwall/btree) package - `tidwall`: The [tidwall/btree](https://github.com/tidwall/btree) package
@ -163,29 +173,29 @@ The following benchmarks were run on my 2019 Macbook Pro (2.4 GHz 8-Core Intel C
``` ```
** sequential set ** ** sequential set **
google: set-seq 1,000,000 ops in 163ms, 6,140,597/sec, 162 ns/op, 30.9 MB, 32 bytes/op google: set-seq 1,000,000 ops in 178ms, 5,618,049/sec, 177 ns/op, 39.0 MB, 40 bytes/op
tidwall: set-seq 1,000,000 ops in 141ms, 7,075,240/sec, 141 ns/op, 36.6 MB, 38 bytes/op tidwall: set-seq 1,000,000 ops in 156ms, 6,389,837/sec, 156 ns/op, 23.5 MB, 24 bytes/op
tidwall: set-seq-hint 1,000,000 ops in 79ms, 12,673,902/sec, 78 ns/op, 36.6 MB, 38 bytes/op tidwall: set-seq-hint 1,000,000 ops in 78ms, 12,895,355/sec, 77 ns/op, 23.5 MB, 24 bytes/op
tidwall: load-seq 1,000,000 ops in 40ms, 24,887,293/sec, 40 ns/op, 36.6 MB, 38 bytes/op tidwall: load-seq 1,000,000 ops in 53ms, 18,937,400/sec, 52 ns/op, 23.5 MB, 24 bytes/op
go-arr: append 1,000,000 ops in 51ms, 19,617,269/sec, 50 ns/op go-arr: append 1,000,000 ops in 78ms, 12,843,432/sec, 77 ns/op
** random set ** ** random set **
google: set-rand 1,000,000 ops in 666ms, 1,501,583/sec, 665 ns/op, 21.5 MB, 22 bytes/op google: set-rand 1,000,000 ops in 555ms, 1,803,133/sec, 554 ns/op, 29.7 MB, 31 bytes/op
tidwall: set-rand 1,000,000 ops in 569ms, 1,756,845/sec, 569 ns/op, 26.7 MB, 27 bytes/op tidwall: set-rand 1,000,000 ops in 545ms, 1,835,818/sec, 544 ns/op, 29.6 MB, 31 bytes/op
tidwall: set-rand-hint 1,000,000 ops in 670ms, 1,491,637/sec, 670 ns/op, 26.4 MB, 27 bytes/op tidwall: set-rand-hint 1,000,000 ops in 670ms, 1,493,473/sec, 669 ns/op, 29.6 MB, 31 bytes/op
tidwall: set-again 1,000,000 ops in 488ms, 2,050,667/sec, 487 ns/op, 27.1 MB, 28 bytes/op tidwall: set-again 1,000,000 ops in 681ms, 1,469,038/sec, 680 ns/op
tidwall: set-after-copy 1,000,000 ops in 494ms, 2,022,980/sec, 494 ns/op, 27.9 MB, 29 bytes/op tidwall: set-after-copy 1,000,000 ops in 670ms, 1,493,230/sec, 669 ns/op
tidwall: load-rand 1,000,000 ops in 594ms, 1,682,937/sec, 594 ns/op, 26.1 MB, 27 bytes/op tidwall: load-rand 1,000,000 ops in 569ms, 1,756,187/sec, 569 ns/op, 29.6 MB, 31 bytes/op
** sequential get ** ** sequential get **
google: get-seq 1,000,000 ops in 141ms, 7,078,690/sec, 141 ns/op google: get-seq 1,000,000 ops in 165ms, 6,048,307/sec, 165 ns/op
tidwall: get-seq 1,000,000 ops in 124ms, 8,075,925/sec, 123 ns/op tidwall: get-seq 1,000,000 ops in 144ms, 6,940,120/sec, 144 ns/op
tidwall: get-seq-hint 1,000,000 ops in 40ms, 25,142,979/sec, 39 ns/op tidwall: get-seq-hint 1,000,000 ops in 78ms, 12,815,243/sec, 78 ns/op
** random get ** ** random get **
google: get-rand 1,000,000 ops in 152ms, 6,593,518/sec, 151 ns/op google: get-rand 1,000,000 ops in 701ms, 1,427,507/sec, 700 ns/op
tidwall: get-rand 1,000,000 ops in 128ms, 7,783,293/sec, 128 ns/op tidwall: get-rand 1,000,000 ops in 679ms, 1,473,531/sec, 678 ns/op
tidwall: get-rand-hint 1,000,000 ops in 135ms, 7,403,823/sec, 135 ns/op tidwall: get-rand-hint 1,000,000 ops in 824ms, 1,213,805/sec, 823 ns/op
``` ```
*You can find the benchmark utility at [tidwall/btree-benchmark](https://github.com/tidwall/btree-benchmark)* *You can find the benchmark utility at [tidwall/btree-benchmark](https://github.com/tidwall/btree-benchmark)*

File diff suppressed because it is too large Load Diff

1275
vendor/github.com/tidwall/btree/internal/btree.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@ -137,6 +137,7 @@ All keys/value pairs are ordered in the database by the key. To iterate over the
err := db.View(func(tx *buntdb.Tx) error { err := db.View(func(tx *buntdb.Tx) error {
err := tx.Ascend("", func(key, value string) bool { err := tx.Ascend("", func(key, value string) bool {
fmt.Printf("key: %s, value: %s\n", key, value) fmt.Printf("key: %s, value: %s\n", key, value)
return true // continue iteration
}) })
return err return err
}) })

View File

@ -4,7 +4,9 @@
width="240" height="78" border="0" alt="GJSON"> width="240" height="78" border="0" alt="GJSON">
<br> <br>
<a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a> <a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
<a href="http://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a> <a href="https://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a>
<a href="SYNTAX.md"><img src="https://img.shields.io/badge/{}-syntax-33aa33.svg?style=flat-square" alt="GJSON Syntax"></a>
</p> </p>
<p align="center">get json values quickly</a></p> <p align="center">get json values quickly</a></p>

View File

@ -13,11 +13,11 @@ This document is designed to explain the structure of a GJSON Path through examp
- [Dot vs Pipe](#dot-vs-pipe) - [Dot vs Pipe](#dot-vs-pipe)
- [Modifiers](#modifiers) - [Modifiers](#modifiers)
- [Multipaths](#multipaths) - [Multipaths](#multipaths)
- [Literals](#literals)
The definitive implemenation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson). The definitive implemenation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson).
Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online. Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online.
## Path structure ## Path structure
A GJSON Path is intended to be easily expressed as a series of components seperated by a `.` character. A GJSON Path is intended to be easily expressed as a series of components seperated by a `.` character.
@ -296,7 +296,7 @@ Starting with v1.3.0, GJSON added the ability to join multiple paths together
to form new documents. Wrapping comma-separated paths between `[...]` or to form new documents. Wrapping comma-separated paths between `[...]` or
`{...}` will result in a new array or object, respectively. `{...}` will result in a new array or object, respectively.
For example, using the given multipath For example, using the given multipath:
``` ```
{name.first,age,"the_murphys":friends.#(last="Murphy")#.first} {name.first,age,"the_murphys":friends.#(last="Murphy")#.first}
@ -312,8 +312,28 @@ determined, then "_" is used.
This results in This results in
``` ```json
{"first":"Tom","age":37,"the_murphys":["Dale","Jane"]} {"first":"Tom","age":37,"the_murphys":["Dale","Jane"]}
``` ```
### Literals
Starting with v1.12.0, GJSON added support of json literals, which provides a way for constructing static blocks of json. This is can be particularly useful when constructing a new json document using [multipaths](#multipaths).
A json literal begins with the '!' declaration character.
For example, using the given multipath:
```
{name.first,age,"company":!"Happysoft","employed":!true}
```
Here we selected the first name and age. Then add two new fields, "company" and "employed".
This results in
```json
{"first":"Tom","age":37,"company":"Happysoft","employed":true}
```
*See issue [#249](https://github.com/tidwall/gjson/issues/249) for additional context on JSON Literals.*

View File

@ -229,17 +229,19 @@ func (t Result) ForEach(iterator func(key, value Result) bool) {
return return
} }
json := t.Raw json := t.Raw
var keys bool var obj bool
var i int var i int
var key, value Result var key, value Result
for ; i < len(json); i++ { for ; i < len(json); i++ {
if json[i] == '{' { if json[i] == '{' {
i++ i++
key.Type = String key.Type = String
keys = true obj = true
break break
} else if json[i] == '[' { } else if json[i] == '[' {
i++ i++
key.Type = Number
key.Num = -1
break break
} }
if json[i] > ' ' { if json[i] > ' ' {
@ -249,8 +251,9 @@ func (t Result) ForEach(iterator func(key, value Result) bool) {
var str string var str string
var vesc bool var vesc bool
var ok bool var ok bool
var idx int
for ; i < len(json); i++ { for ; i < len(json); i++ {
if keys { if obj {
if json[i] != '"' { if json[i] != '"' {
continue continue
} }
@ -265,7 +268,9 @@ func (t Result) ForEach(iterator func(key, value Result) bool) {
key.Str = str[1 : len(str)-1] key.Str = str[1 : len(str)-1]
} }
key.Raw = str key.Raw = str
key.Index = s key.Index = s + t.Index
} else {
key.Num += 1
} }
for ; i < len(json); i++ { for ; i < len(json); i++ {
if json[i] <= ' ' || json[i] == ',' || json[i] == ':' { if json[i] <= ' ' || json[i] == ',' || json[i] == ':' {
@ -278,10 +283,17 @@ func (t Result) ForEach(iterator func(key, value Result) bool) {
if !ok { if !ok {
return return
} }
value.Index = s if t.Indexes != nil {
if idx < len(t.Indexes) {
value.Index = t.Indexes[idx]
}
} else {
value.Index = s + t.Index
}
if !iterator(key, value) { if !iterator(key, value) {
return return
} }
idx++
} }
} }
@ -298,7 +310,15 @@ func (t Result) Map() map[string]Result {
// Get searches result for the specified path. // Get searches result for the specified path.
// The result should be a JSON array or object. // The result should be a JSON array or object.
func (t Result) Get(path string) Result { func (t Result) Get(path string) Result {
return Get(t.Raw, path) r := Get(t.Raw, path)
if r.Indexes != nil {
for i := 0; i < len(r.Indexes); i++ {
r.Indexes[i] += t.Index
}
} else {
r.Index += t.Index
}
return r
} }
type arrayOrMapResult struct { type arrayOrMapResult struct {
@ -389,6 +409,8 @@ func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) {
value.Raw, value.Str = tostr(json[i:]) value.Raw, value.Str = tostr(json[i:])
value.Num = 0 value.Num = 0
} }
value.Index = i + t.Index
i += len(value.Raw) - 1 i += len(value.Raw) - 1
if r.vc == '{' { if r.vc == '{' {
@ -415,6 +437,17 @@ func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) {
} }
} }
end: end:
if t.Indexes != nil {
if len(t.Indexes) != len(r.a) {
for i := 0; i < len(r.a); i++ {
r.a[i].Index = 0
}
} else {
for i := 0; i < len(r.a); i++ {
r.a[i].Index = t.Indexes[i]
}
}
}
return return
} }
@ -426,7 +459,8 @@ end:
// use the Valid function first. // use the Valid function first.
func Parse(json string) Result { func Parse(json string) Result {
var value Result var value Result
for i := 0; i < len(json); i++ { i := 0
for ; i < len(json); i++ {
if json[i] == '{' || json[i] == '[' { if json[i] == '{' || json[i] == '[' {
value.Type = JSON value.Type = JSON
value.Raw = json[i:] // just take the entire raw value.Raw = json[i:] // just take the entire raw
@ -436,16 +470,20 @@ func Parse(json string) Result {
continue continue
} }
switch json[i] { switch json[i] {
default: case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { 'i', 'I', 'N':
value.Type = Number
value.Raw, value.Num = tonum(json[i:])
case 'n':
if i+1 < len(json) && json[i+1] != 'u' {
// nan
value.Type = Number value.Type = Number
value.Raw, value.Num = tonum(json[i:]) value.Raw, value.Num = tonum(json[i:])
} else { } else {
return Result{} // null
value.Type = Null
value.Raw = tolit(json[i:])
} }
case 'n':
value.Type = Null
value.Raw = tolit(json[i:])
case 't': case 't':
value.Type = True value.Type = True
value.Raw = tolit(json[i:]) value.Raw = tolit(json[i:])
@ -455,9 +493,14 @@ func Parse(json string) Result {
case '"': case '"':
value.Type = String value.Type = String
value.Raw, value.Str = tostr(json[i:]) value.Raw, value.Str = tostr(json[i:])
default:
return Result{}
} }
break break
} }
if value.Exists() {
value.Index = i
}
return value return value
} }
@ -531,20 +574,12 @@ func tonum(json string) (raw string, num float64) {
return return
} }
// could be a '+' or '-'. let's assume so. // could be a '+' or '-'. let's assume so.
continue } else if json[i] == ']' || json[i] == '}' {
// break on ']' or '}'
raw = json[:i]
num, _ = strconv.ParseFloat(raw, 64)
return
} }
if json[i] < ']' {
// probably a valid number
continue
}
if json[i] == 'e' || json[i] == 'E' {
// allow for exponential numbers
continue
}
// likely a ']' or '}'
raw = json[:i]
num, _ = strconv.ParseFloat(raw, 64)
return
} }
raw = json raw = json
num, _ = strconv.ParseFloat(raw, 64) num, _ = strconv.ParseFloat(raw, 64)
@ -1513,7 +1548,6 @@ func parseArray(c *parseContext, i int, path string) (int, bool) {
} }
if idx < len(c.json) && c.json[idx] != ']' { if idx < len(c.json) && c.json[idx] != ']' {
_, res, ok := parseAny(c.json, idx, true) _, res, ok := parseAny(c.json, idx, true)
parentIndex := res.Index
if ok { if ok {
res := res.Get(rp.alogkey) res := res.Get(rp.alogkey)
if res.Exists() { if res.Exists() {
@ -1525,8 +1559,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) {
raw = res.String() raw = res.String()
} }
jsons = append(jsons, []byte(raw)...) jsons = append(jsons, []byte(raw)...)
indexes = append(indexes, indexes = append(indexes, res.Index)
res.Index+parentIndex)
k++ k++
} }
} }
@ -1699,7 +1732,7 @@ type subSelector struct {
// first character in path is either '[' or '{', and has already been checked // first character in path is either '[' or '{', and has already been checked
// prior to calling this function. // prior to calling this function.
func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) { func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) {
modifer := 0 modifier := 0
depth := 1 depth := 1
colon := 0 colon := 0
start := 1 start := 1
@ -1714,6 +1747,7 @@ func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) {
} }
sels = append(sels, sel) sels = append(sels, sel)
colon = 0 colon = 0
modifier = 0
start = i + 1 start = i + 1
} }
for ; i < len(path); i++ { for ; i < len(path); i++ {
@ -1721,11 +1755,11 @@ func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) {
case '\\': case '\\':
i++ i++
case '@': case '@':
if modifer == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') { if modifier == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') {
modifer = i modifier = i
} }
case ':': case ':':
if modifer == 0 && colon == 0 && depth == 1 { if modifier == 0 && colon == 0 && depth == 1 {
colon = i colon = i
} }
case ',': case ',':
@ -1778,7 +1812,7 @@ func isSimpleName(component string) bool {
return false return false
} }
switch component[i] { switch component[i] {
case '[', ']', '{', '}', '(', ')', '#', '|': case '[', ']', '{', '}', '(', ')', '#', '|', '!':
return false return false
} }
} }
@ -1842,23 +1876,25 @@ type parseContext struct {
// use the Valid function first. // use the Valid function first.
func Get(json, path string) Result { func Get(json, path string) Result {
if len(path) > 1 { if len(path) > 1 {
if !DisableModifiers { if (path[0] == '@' && !DisableModifiers) || path[0] == '!' {
if path[0] == '@' { // possible modifier
// possible modifier var ok bool
var ok bool var npath string
var npath string var rjson string
var rjson string if path[0] == '@' && !DisableModifiers {
npath, rjson, ok = execModifier(json, path) npath, rjson, ok = execModifier(json, path)
if ok { } else if path[0] == '!' {
path = npath npath, rjson, ok = execStatic(json, path)
if len(path) > 0 && (path[0] == '|' || path[0] == '.') { }
res := Get(rjson, path[1:]) if ok {
res.Index = 0 path = npath
res.Indexes = nil if len(path) > 0 && (path[0] == '|' || path[0] == '.') {
return res res := Get(rjson, path[1:])
} res.Index = 0
return Parse(rjson) res.Indexes = nil
return res
} }
return Parse(rjson)
} }
} }
if path[0] == '[' || path[0] == '{' { if path[0] == '[' || path[0] == '{' {
@ -2527,8 +2563,40 @@ func safeInt(f float64) (n int64, ok bool) {
return int64(f), true return int64(f), true
} }
// execStatic parses the path to find a static value.
// The input expects that the path already starts with a '!'
func execStatic(json, path string) (pathOut, res string, ok bool) {
name := path[1:]
if len(name) > 0 {
switch name[0] {
case '{', '[', '"', '+', '-', '0', '1', '2', '3', '4', '5', '6', '7',
'8', '9':
_, res = parseSquash(name, 0)
pathOut = name[len(res):]
return pathOut, res, true
}
}
for i := 1; i < len(path); i++ {
if path[i] == '|' {
pathOut = path[i:]
name = path[1:i]
break
}
if path[i] == '.' {
pathOut = path[i:]
name = path[1:i]
break
}
}
switch strings.ToLower(name) {
case "true", "false", "null", "nan", "inf":
return pathOut, name, true
}
return pathOut, res, false
}
// execModifier parses the path to find a matching modifier function. // execModifier parses the path to find a matching modifier function.
// then input expects that the path already starts with a '@' // The input expects that the path already starts with a '@'
func execModifier(json, path string) (pathOut, res string, ok bool) { func execModifier(json, path string) (pathOut, res string, ok bool) {
name := path[1:] name := path[1:]
var hasArgs bool var hasArgs bool
@ -2971,3 +3039,176 @@ func stringBytes(s string) []byte {
func bytesString(b []byte) string { func bytesString(b []byte) string {
return *(*string)(unsafe.Pointer(&b)) return *(*string)(unsafe.Pointer(&b))
} }
func revSquash(json string) string {
// reverse squash
// expects that the tail character is a ']' or '}' or ')' or '"'
// squash the value, ignoring all nested arrays and objects.
i := len(json) - 1
var depth int
if json[i] != '"' {
depth++
}
if json[i] == '}' || json[i] == ']' || json[i] == ')' {
i--
}
for ; i >= 0; i-- {
switch json[i] {
case '"':
i--
for ; i >= 0; i-- {
if json[i] == '"' {
esc := 0
for i > 0 && json[i-1] == '\\' {
i--
esc++
}
if esc%2 == 1 {
continue
}
i += esc
break
}
}
if depth == 0 {
if i < 0 {
i = 0
}
return json[i:]
}
case '}', ']', ')':
depth++
case '{', '[', '(':
depth--
if depth == 0 {
return json[i:]
}
}
}
return json
}
func (t Result) Paths(json string) []string {
if t.Indexes == nil {
return nil
}
paths := make([]string, 0, len(t.Indexes))
t.ForEach(func(_, value Result) bool {
paths = append(paths, value.Path(json))
return true
})
if len(paths) != len(t.Indexes) {
return nil
}
return paths
}
// Path returns the original GJSON path for Result.
// The json param must be the original JSON used when calling Get.
func (t Result) Path(json string) string {
var path []byte
var comps []string // raw components
i := t.Index - 1
if t.Index+len(t.Raw) > len(json) {
// JSON cannot safely contain Result.
goto fail
}
if !strings.HasPrefix(json[t.Index:], t.Raw) {
// Result is not at the JSON index as exepcted.
goto fail
}
for ; i >= 0; i-- {
if json[i] <= ' ' {
continue
}
if json[i] == ':' {
// inside of object, get the key
for ; i >= 0; i-- {
if json[i] != '"' {
continue
}
break
}
raw := revSquash(json[:i+1])
i = i - len(raw)
comps = append(comps, raw)
// key gotten, now squash the rest
raw = revSquash(json[:i+1])
i = i - len(raw)
i++ // increment the index for next loop step
} else if json[i] == '{' {
// Encountered an open object. The original result was probably an
// object key.
goto fail
} else if json[i] == ',' || json[i] == '[' {
// inside of an array, count the position
var arrIdx int
if json[i] == ',' {
arrIdx++
i--
}
for ; i >= 0; i-- {
if json[i] == ':' {
// Encountered an unexpected colon. The original result was
// probably an object key.
goto fail
} else if json[i] == ',' {
arrIdx++
} else if json[i] == '[' {
comps = append(comps, strconv.Itoa(arrIdx))
break
} else if json[i] == ']' || json[i] == '}' || json[i] == '"' {
raw := revSquash(json[:i+1])
i = i - len(raw) + 1
}
}
}
}
if len(comps) == 0 {
if DisableModifiers {
goto fail
}
return "@this"
}
for i := len(comps) - 1; i >= 0; i-- {
rcomp := Parse(comps[i])
if !rcomp.Exists() {
goto fail
}
comp := escapeComp(rcomp.String())
path = append(path, '.')
path = append(path, comp...)
}
if len(path) > 0 {
path = path[1:]
}
return string(path)
fail:
return ""
}
// isSafePathKeyChar returns true if the input character is safe for not
// needing escaping.
func isSafePathKeyChar(c byte) bool {
return c <= ' ' || c > '~' || c == '_' || c == '-' || c == ':' ||
(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9')
}
// escapeComp escaped a path compontent, making it safe for generating a
// path for later use.
func escapeComp(comp string) string {
for i := 0; i < len(comp); i++ {
if !isSafePathKeyChar(comp[i]) {
ncomp := []byte(comp[:i])
for ; i < len(comp); i++ {
if !isSafePathKeyChar(comp[i]) {
ncomp = append(ncomp, '\\')
}
ncomp = append(ncomp, comp[i])
}
return string(ncomp)
}
}
return comp
}

9
vendor/modules.txt vendored
View File

@ -45,16 +45,17 @@ github.com/okzk/sdnotify
## explicit ## explicit
# github.com/stretchr/testify v1.4.0 # github.com/stretchr/testify v1.4.0
## explicit ## explicit
# github.com/tidwall/btree v0.6.1 # github.com/tidwall/btree v1.1.0
## explicit; go 1.16 ## explicit; go 1.16
github.com/tidwall/btree github.com/tidwall/btree
# github.com/tidwall/buntdb v1.2.7 github.com/tidwall/btree/internal
# github.com/tidwall/buntdb v1.2.9
## explicit; go 1.16 ## explicit; go 1.16
github.com/tidwall/buntdb github.com/tidwall/buntdb
# github.com/tidwall/gjson v1.10.2 # github.com/tidwall/gjson v1.12.1
## explicit; go 1.12 ## explicit; go 1.12
github.com/tidwall/gjson github.com/tidwall/gjson
# github.com/tidwall/grect v0.1.3 # github.com/tidwall/grect v0.1.4
## explicit; go 1.15 ## explicit; go 1.15
github.com/tidwall/grect github.com/tidwall/grect
# github.com/tidwall/match v1.1.1 # github.com/tidwall/match v1.1.1