diff --git a/go.mod b/go.mod index 3bd3851c..08ece3c4 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/onsi/ginkgo v1.12.0 // indirect github.com/onsi/gomega v1.9.0 // indirect github.com/stretchr/testify v1.4.0 // indirect - github.com/tidwall/buntdb v1.2.6 + github.com/tidwall/buntdb v1.2.7 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 github.com/xdg-go/scram v1.0.2 golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc @@ -26,11 +26,11 @@ require ( ) require ( - github.com/tidwall/btree v0.6.0 // indirect - github.com/tidwall/gjson v1.8.0 // indirect - github.com/tidwall/grect v0.1.2 // indirect - github.com/tidwall/match v1.0.3 // indirect - github.com/tidwall/pretty v1.1.0 // indirect + github.com/tidwall/btree v0.6.1 // indirect + github.com/tidwall/gjson v1.10.2 // indirect + github.com/tidwall/grect v0.1.3 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect diff --git a/go.sum b/go.sum index 6d77b44b..44b4168f 100644 --- a/go.sum +++ b/go.sum @@ -39,20 +39,33 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v0.6.0 h1:JLYAFGV+1gjyFi3iQbO/fupBin+Ooh7dxqVV0twJ1Bo= github.com/tidwall/btree v0.6.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= +github.com/tidwall/btree v0.6.1 h1:75VVgBeviiDO+3g4U+7+BaNBNhNINxB0ULPT3fs9pMY= +github.com/tidwall/btree v0.6.1/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= github.com/tidwall/buntdb v1.2.6 h1:eS0QSmzHfCKjxxYGh8eH6wnK5VLsJ7UjyyIr29JmnEg= github.com/tidwall/buntdb v1.2.6/go.mod h1:zpXqlA5D2772I4cTqV3ifr2AZihDgi8FV7xAQu6edfc= +github.com/tidwall/buntdb v1.2.7 h1:SIyObKAymzLyGhDeIhVk2Yc1/EwfCC75Uyu77CHlVoA= +github.com/tidwall/buntdb v1.2.7/go.mod h1:b6KvZM27x/8JLI5hgRhRu60pa3q0Tz9c50TyD46OHUM= github.com/tidwall/gjson v1.8.0 h1:Qt+orfosKn0rbNTZqHYDqBrmm3UDA4KRkv70fDzG+PQ= github.com/tidwall/gjson v1.8.0/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= +github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo= +github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.2 h1:wKVeQVZhjaFCKTTlpkDe3Ex4ko3cMGW3MRKawRe8uQ4= github.com/tidwall/grect v0.1.2/go.mod h1:v+n4ewstPGduVJebcp5Eh2WXBJBumNzyhK8GZt4gHNw= +github.com/tidwall/grect v0.1.3 h1:z9YwQAMUxVSBde3b7Sl8Da37rffgNfZ6Fq6h9t6KdXE= +github.com/tidwall/grect v0.1.3/go.mod h1:8GMjwh3gPZVpLBI/jDz9uslCe0dpxRpWDdtN0lWAS/E= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8= github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= diff --git a/vendor/github.com/tidwall/btree/btree.go b/vendor/github.com/tidwall/btree/btree.go index 100cf74d..21afc5b3 100644 --- a/vendor/github.com/tidwall/btree/btree.go +++ b/vendor/github.com/tidwall/btree/btree.go @@ -296,7 +296,38 @@ func (n *node) scan(iter func(item interface{}) bool) bool { // Get a value for key func (tr *BTree) Get(key interface{}) interface{} { - return tr.GetHint(key, nil) + // This operation is basically the same as calling: + // return tr.GetHint(key, nil) + // But here we inline the bsearch to avoid the hint logic and extra + // function call. + if tr.rlock() { + defer tr.runlock() + } + if tr.root == nil || key == nil { + return nil + } + depth := 0 + n := tr.root + for { + low := int16(0) + high := n.numItems - 1 + for low <= high { + mid := low + ((high+1)-low)/2 + if !tr.less(key, n.items[mid]) { + low = mid + 1 + } else { + high = mid - 1 + } + } + if low > 0 && !tr.less(n.items[low-1], key) { + return n.items[low-1] + } + if n.leaf { + return nil + } + n = n.children[low] + depth++ + } } // GetHint gets a value for key using a path hint @@ -310,14 +341,14 @@ func (tr *BTree) GetHint(key interface{}, hint *PathHint) interface{} { depth := 0 n := tr.root for { - i, found := n.find(key, tr.less, hint, depth) + index, found := n.find(key, tr.less, hint, depth) if found { - return n.items[i] + return n.items[index] } if n.leaf { return nil } - n = n.children[i] + n = n.children[index] depth++ } } diff --git a/vendor/github.com/tidwall/buntdb/buntdb.go b/vendor/github.com/tidwall/buntdb/buntdb.go index 1b1224b5..8c3408da 100644 --- a/vendor/github.com/tidwall/buntdb/buntdb.go +++ b/vendor/github.com/tidwall/buntdb/buntdb.go @@ -1263,12 +1263,15 @@ type dbItem struct { keyless bool // keyless item for scanning } +// estIntSize returns the string representions size. +// Has the same result as len(strconv.Itoa(x)). func estIntSize(x int) int { - if x == 0 { - return 1 + n := 1 + if x < 0 { + n++ + x *= -1 } - var n int - for x > 0 { + for x >= 10 { n++ x /= 10 } @@ -1283,7 +1286,10 @@ func estBulkStringSize(s string) int { return 1 + estIntSize(len(s)) + 2 + len(s) + 2 } -func (dbi *dbItem) estAOFSetSize() (n int) { +// estAOFSetSize returns an estimated number of bytes that this item will use +// when stored in the aof file. +func (dbi *dbItem) estAOFSetSize() int { + var n int if dbi.opts != nil && dbi.opts.ex { n += estArraySize(5) n += estBulkStringSize("set") diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index 059570f9..f4898478 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -123,11 +123,12 @@ nil, for JSON null To directly access the value: ```go -result.Type // can be String, Number, True, False, Null, or JSON -result.Str // holds the string -result.Num // holds the float64 number -result.Raw // holds the raw json -result.Index // index of raw value in original json, zero means index unknown +result.Type // can be String, Number, True, False, Null, or JSON +result.Str // holds the string +result.Num // holds the float64 number +result.Raw // holds the raw json +result.Index // index of raw value in original json, zero means index unknown +result.Indexes // indexes of all the elements that match on a path containing the '#' query character. ``` There are a variety of handy functions that work on a result: @@ -199,6 +200,8 @@ There are currently the following built-in modifiers: - `@valid`: Ensure the json document is valid. - `@flatten`: Flattens an array. - `@join`: Joins multiple objects into a single object. +- `@keys`: Returns an array of keys for an object. +- `@values`: Returns an array of values for an object. ### Modifier arguments @@ -433,14 +436,15 @@ Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/js and [json-iterator](https://github.com/json-iterator/go) ``` -BenchmarkGJSONGet-8 3000000 372 ns/op 0 B/op 0 allocs/op -BenchmarkGJSONUnmarshalMap-8 900000 4154 ns/op 1920 B/op 26 allocs/op -BenchmarkJSONUnmarshalMap-8 600000 9019 ns/op 3048 B/op 69 allocs/op -BenchmarkJSONDecoder-8 300000 14120 ns/op 4224 B/op 184 allocs/op -BenchmarkFFJSONLexer-8 1500000 3111 ns/op 896 B/op 8 allocs/op -BenchmarkEasyJSONLexer-8 3000000 887 ns/op 613 B/op 6 allocs/op -BenchmarkJSONParserGet-8 3000000 499 ns/op 21 B/op 0 allocs/op -BenchmarkJSONIterator-8 3000000 812 ns/op 544 B/op 9 allocs/op +BenchmarkGJSONGet-16 11644512 311 ns/op 0 B/op 0 allocs/op +BenchmarkGJSONUnmarshalMap-16 1122678 3094 ns/op 1920 B/op 26 allocs/op +BenchmarkJSONUnmarshalMap-16 516681 6810 ns/op 2944 B/op 69 allocs/op +BenchmarkJSONUnmarshalStruct-16 697053 5400 ns/op 928 B/op 13 allocs/op +BenchmarkJSONDecoder-16 330450 10217 ns/op 3845 B/op 160 allocs/op +BenchmarkFFJSONLexer-16 1424979 2585 ns/op 880 B/op 8 allocs/op +BenchmarkEasyJSONLexer-16 3000000 729 ns/op 501 B/op 5 allocs/op +BenchmarkJSONParserGet-16 3000000 366 ns/op 21 B/op 0 allocs/op +BenchmarkJSONIterator-16 3000000 869 ns/op 693 B/op 14 allocs/op ``` JSON document used: @@ -481,4 +485,4 @@ widget.image.hOffset widget.text.onMouseUp ``` -*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be found [here](https://github.com/tidwall/gjson-benchmarks).* +*These benchmarks were run on a MacBook Pro 16" 2.4 GHz Intel Core i9 using Go 1.17 and can be found [here](https://github.com/tidwall/gjson-benchmarks).* diff --git a/vendor/github.com/tidwall/gjson/SYNTAX.md b/vendor/github.com/tidwall/gjson/SYNTAX.md index 86235b9b..9bc18c88 100644 --- a/vendor/github.com/tidwall/gjson/SYNTAX.md +++ b/vendor/github.com/tidwall/gjson/SYNTAX.md @@ -135,6 +135,37 @@ changed in v1.3.0 as to avoid confusion with the new [multipath](#multipaths) syntax. For backwards compatibility, `#[...]` will continue to work until the next major release.* +The `~` (tilde) operator will convert a value to a boolean before comparison. + +For example, using the following JSON: + +```json +{ + "vals": [ + { "a": 1, "b": true }, + { "a": 2, "b": true }, + { "a": 3, "b": false }, + { "a": 4, "b": "0" }, + { "a": 5, "b": 0 }, + { "a": 6, "b": "1" }, + { "a": 7, "b": 1 }, + { "a": 8, "b": "true" }, + { "a": 9, "b": false }, + { "a": 10, "b": null }, + { "a": 11 } + ] +} +``` + +You can now query for all true(ish) or false(ish) values: + +``` +vals.#(b==~true)#.a >> [1,2,6,7,8] +vals.#(b==~false)#.a >> [3,4,5,9,10,11] +``` + +The last value which was non-existent is treated as `false` + ### Dot vs Pipe The `.` is standard separator, but it's also possible to use a `|`. @@ -205,6 +236,8 @@ There are currently the following built-in modifiers: - `@valid`: Ensure the json document is valid. - `@flatten`: Flattens an array. - `@join`: Joins multiple objects into a single object. +- `@keys`: Returns an array of keys for an object. +- `@values`: Returns an array of values for an object. #### Modifier arguments @@ -260,8 +293,8 @@ gjson.AddModifier("case", func(json, arg string) string { ### Multipaths Starting with v1.3.0, GJSON added the ability to join multiple paths together -to form new documents. Wrapping comma-separated paths between `{...}` or -`[...]` will result in a new array or object, respectively. +to form new documents. Wrapping comma-separated paths between `[...]` or +`{...}` will result in a new array or object, respectively. For example, using the given multipath diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index 9687c0bb..279649ee 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -64,6 +64,9 @@ type Result struct { Num float64 // Index of raw value in original json, zero means index unknown Index int + // Indexes of all the elements that match on a path containing the '#' + // query character. + Indexes []int } // String returns a string representation of the value. @@ -186,14 +189,15 @@ func (t Result) Time() time.Time { } // Array returns back an array of values. -// If the result represents a non-existent value, then an empty array will be -// returned. If the result is not a JSON array, the return value will be an +// If the result represents a null value or is non-existent, then an empty +// array will be returned. +// If the result is not a JSON array, the return value will be an // array containing one result. func (t Result) Array() []Result { if t.Type == Null { return []Result{} } - if t.Type != JSON { + if !t.IsArray() { return []Result{t} } r := t.arrayOrMap('[', false) @@ -281,7 +285,8 @@ func (t Result) ForEach(iterator func(key, value Result) bool) { } } -// Map returns back an map of values. The result should be a JSON array. +// Map returns back a map of values. The result should be a JSON object. +// If the result is not a JSON object, the return value will be an empty map. func (t Result) Map() map[string]Result { if t.Type != JSON { return map[string]Result{} @@ -584,7 +589,7 @@ func tostr(json string) (raw string, str string) { continue } } - break + return json[:i+1], unescape(json[1:i]) } } var ret string @@ -756,7 +761,7 @@ func parseArrayPath(path string) (r arrayPathResult) { // bad query, end now break } - if len(value) > 2 && value[0] == '"' && + if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { value = value[1 : len(value)-1] if vesc { @@ -1085,9 +1090,9 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { } if rp.wild { if kesc { - pmatch = match.Match(unescape(key), rp.part) + pmatch = matchLimit(unescape(key), rp.part) } else { - pmatch = match.Match(key, rp.part) + pmatch = matchLimit(key, rp.part) } } else { if kesc { @@ -1098,6 +1103,7 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { } hit = pmatch && !rp.more for ; i < len(c.json); i++ { + var num bool switch c.json[i] { default: continue @@ -1145,15 +1151,13 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { return i, true } } - case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - i, val = parseNumber(c.json, i) - if hit { - c.value.Raw = val - c.value.Type = Number - c.value.Num, _ = strconv.ParseFloat(val, 64) - return i, true + case 'n': + if i+1 < len(c.json) && c.json[i+1] != 'u' { + num = true + break } - case 't', 'f', 'n': + fallthrough + case 't', 'f': vc := c.json[i] i, val = parseLiteral(c.json, i) if hit { @@ -1166,12 +1170,33 @@ func parseObject(c *parseContext, i int, path string) (int, bool) { } return i, true } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true + } + if num { + i, val = parseNumber(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } } break } } return i, false } + +// matchLimit will limit the complexity of the match operation to avoid ReDos +// attacks from arbritary inputs. +// See the github.com/tidwall/match.MatchLimit function for more information. +func matchLimit(str, pattern string) bool { + matched, _ := match.MatchLimit(str, pattern, 10000) + return matched +} + func queryMatches(rp *arrayPathResult, value Result) bool { rpv := rp.query.value if len(rpv) > 0 && rpv[0] == '~' { @@ -1209,9 +1234,9 @@ func queryMatches(rp *arrayPathResult, value Result) bool { case ">=": return value.Str >= rpv case "%": - return match.Match(value.Str, rpv) + return matchLimit(value.Str, rpv) case "!%": - return !match.Match(value.Str, rpv) + return !matchLimit(value.Str, rpv) } case Number: rpvn, _ := strconv.ParseFloat(rpv, 64) @@ -1261,6 +1286,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { var alog []int var partidx int var multires []byte + var queryIndexes []int rp := parseArrayPath(path) if !rp.arrch { n, ok := parseUint(rp.part) @@ -1281,6 +1307,10 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { multires = append(multires, '[') } } + var tmp parseContext + tmp.value = qval + fillIndex(c.json, &tmp) + parentIndex := tmp.value.Index var res Result if qval.Type == JSON { res = qval.Get(rp.query.path) @@ -1312,6 +1342,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { multires = append(multires, ',') } multires = append(multires, raw...) + queryIndexes = append(queryIndexes, res.Index+parentIndex) } } else { c.value = res @@ -1338,6 +1369,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } else { ch = c.json[i] } + var num bool switch ch { default: continue @@ -1420,26 +1452,13 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { return i, true } } - case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - i, val = parseNumber(c.json, i) - if rp.query.on { - var qval Result - qval.Raw = val - qval.Type = Number - qval.Num, _ = strconv.ParseFloat(val, 64) - if procQuery(qval) { - return i, true - } - } else if hit { - if rp.alogok { - break - } - c.value.Raw = val - c.value.Type = Number - c.value.Num, _ = strconv.ParseFloat(val, 64) - return i, true + case 'n': + if i+1 < len(c.json) && c.json[i+1] != 'u' { + num = true + break } - case 't', 'f', 'n': + fallthrough + case 't', 'f': vc := c.json[i] i, val = parseLiteral(c.json, i) if rp.query.on { @@ -1467,6 +1486,9 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } return i, true } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true case ']': if rp.arrch && rp.part == "#" { if rp.alogok { @@ -1476,6 +1498,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { c.pipe = right c.piped = true } + var indexes = make([]int, 0, 64) var jsons = make([]byte, 0, 64) jsons = append(jsons, '[') for j, k := 0, 0; j < len(alog); j++ { @@ -1490,6 +1513,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } if idx < len(c.json) && c.json[idx] != ']' { _, res, ok := parseAny(c.json, idx, true) + parentIndex := res.Index if ok { res := res.Get(rp.alogkey) if res.Exists() { @@ -1501,6 +1525,8 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { raw = res.String() } jsons = append(jsons, []byte(raw)...) + indexes = append(indexes, + res.Index+parentIndex) k++ } } @@ -1509,6 +1535,7 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { jsons = append(jsons, ']') c.value.Type = JSON c.value.Raw = string(jsons) + c.value.Indexes = indexes return i + 1, true } if rp.alogok { @@ -1524,8 +1551,9 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { if !c.value.Exists() { if len(multires) > 0 { c.value = Result{ - Raw: string(append(multires, ']')), - Type: JSON, + Raw: string(append(multires, ']')), + Type: JSON, + Indexes: queryIndexes, } } else if rp.query.all { c.value = Result{ @@ -1536,6 +1564,26 @@ func parseArray(c *parseContext, i int, path string) (int, bool) { } return i + 1, false } + if num { + i, val = parseNumber(c.json, i) + if rp.query.on { + var qval Result + qval.Raw = val + qval.Type = Number + qval.Num, _ = strconv.ParseFloat(val, 64) + if procQuery(qval) { + return i, true + } + } else if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } + } break } } @@ -1806,6 +1854,7 @@ func Get(json, path string) Result { if len(path) > 0 && (path[0] == '|' || path[0] == '.') { res := Get(rjson, path[1:]) res.Index = 0 + res.Indexes = nil return res } return Parse(rjson) @@ -2046,11 +2095,15 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) { res.Raw = val res.Type = JSON } - return i, res, true + var tmp parseContext + tmp.value = res + fillIndex(json, &tmp) + return i, tmp.value, true } if json[i] <= ' ' { continue } + var num bool switch json[i] { case '"': i++ @@ -2070,15 +2123,13 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) { } } return i, res, true - case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - i, val = parseNumber(json, i) - if hit { - res.Raw = val - res.Type = Number - res.Num, _ = strconv.ParseFloat(val, 64) + case 'n': + if i+1 < len(json) && json[i+1] != 'u' { + num = true + break } - return i, res, true - case 't', 'f', 'n': + fallthrough + case 't', 'f': vc := json[i] i, val = parseLiteral(json, i) if hit { @@ -2091,7 +2142,20 @@ func parseAny(json string, i int, hit bool) (int, Result, bool) { } return i, res, true } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true } + if num { + i, val = parseNumber(json, i) + if hit { + res.Raw = val + res.Type = Number + res.Num, _ = strconv.ParseFloat(val, 64) + } + return i, res, true + } + } return i, res, false } @@ -2455,7 +2519,8 @@ func parseInt(s string) (n int64, ok bool) { // safeInt validates a given JSON number // ensures it lies within the minimum and maximum representable JSON numbers func safeInt(f float64) (n int64, ok bool) { - // https://tc39.es/ecma262/#sec-number.min_safe_integer || https://tc39.es/ecma262/#sec-number.max_safe_integer + // https://tc39.es/ecma262/#sec-number.min_safe_integer + // https://tc39.es/ecma262/#sec-number.max_safe_integer if f < -9007199254740991 || f > 9007199254740991 { return 0, false } @@ -2534,6 +2599,8 @@ var modifiers = map[string]func(json, arg string) string{ "flatten": modFlatten, "join": modJoin, "valid": modValid, + "keys": modKeys, + "values": modValues, } // AddModifier binds a custom modifier command to the GJSON syntax. @@ -2690,6 +2757,58 @@ func modFlatten(json, arg string) string { return bytesString(out) } +// @keys extracts the keys from an object. +// {"first":"Tom","last":"Smith"} -> ["first","last"] +func modKeys(json, arg string) string { + v := Parse(json) + if !v.Exists() { + return "[]" + } + obj := v.IsObject() + var out strings.Builder + out.WriteByte('[') + var i int + v.ForEach(func(key, _ Result) bool { + if i > 0 { + out.WriteByte(',') + } + if obj { + out.WriteString(key.Raw) + } else { + out.WriteString("null") + } + i++ + return true + }) + out.WriteByte(']') + return out.String() +} + +// @values extracts the values from an object. +// {"first":"Tom","last":"Smith"} -> ["Tom","Smith"] +func modValues(json, arg string) string { + v := Parse(json) + if !v.Exists() { + return "[]" + } + if v.IsArray() { + return json + } + var out strings.Builder + out.WriteByte('[') + var i int + v.ForEach(func(_, value Result) bool { + if i > 0 { + out.WriteByte(',') + } + out.WriteString(value.Raw) + i++ + return true + }) + out.WriteByte(']') + return out.String() +} + // @join multiple objects into a single object. // [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"} // The arg can be "true" to specify that duplicate keys should be preserved. diff --git a/vendor/github.com/tidwall/match/match.go b/vendor/github.com/tidwall/match/match.go index 57aabf42..11da28f1 100644 --- a/vendor/github.com/tidwall/match/match.go +++ b/vendor/github.com/tidwall/match/match.go @@ -1,7 +1,9 @@ // Package match provides a simple pattern matcher with unicode support. package match -import "unicode/utf8" +import ( + "unicode/utf8" +) // Match returns true if str matches pattern. This is a very // simple wildcard match where '*' matches on any number characters @@ -16,127 +18,170 @@ import "unicode/utf8" // '\\' c matches character c // func Match(str, pattern string) bool { - return deepMatch(str, pattern) -} - -func deepMatch(str, pattern string) bool { if pattern == "*" { return true } - for len(pattern) > 1 && pattern[0] == '*' && pattern[1] == '*' { - pattern = pattern[1:] - } - for len(pattern) > 0 { - if pattern[0] > 0x7f { - return deepMatchRune(str, pattern) - } - switch pattern[0] { - default: - if len(str) == 0 { - return false - } - if str[0] > 0x7f { - return deepMatchRune(str, pattern) - } - if str[0] != pattern[0] { - return false - } - case '?': - if len(str) == 0 { - return false - } - case '*': - return deepMatch(str, pattern[1:]) || - (len(str) > 0 && deepMatch(str[1:], pattern)) - } - str = str[1:] - pattern = pattern[1:] - } - return len(str) == 0 && len(pattern) == 0 + return match(str, pattern, 0, nil, -1) == rMatch } -func deepMatchRune(str, pattern string) bool { +// MatchLimit is the same as Match but will limit the complexity of the match +// operation. This is to avoid long running matches, specifically to avoid ReDos +// attacks from arbritary inputs. +// +// How it works: +// The underlying match routine is recursive and may call itself when it +// encounters a sandwiched wildcard pattern, such as: `user:*:name`. +// Everytime it calls itself a counter is incremented. +// The operation is stopped when counter > maxcomp*len(str). +func MatchLimit(str, pattern string, maxcomp int) (matched, stopped bool) { if pattern == "*" { - return true + return true, false } - for len(pattern) > 1 && pattern[0] == '*' && pattern[1] == '*' { - pattern = pattern[1:] + counter := 0 + r := match(str, pattern, len(str), &counter, maxcomp) + if r == rStop { + return false, true + } + return r == rMatch, false +} + +type result int + +const ( + rNoMatch result = iota + rMatch + rStop +) + +func match(str, pat string, slen int, counter *int, maxcomp int) result { + // check complexity limit + if maxcomp > -1 { + if *counter > slen*maxcomp { + return rStop + } + *counter++ } - var sr, pr rune - var srsz, prsz int - - // read the first rune ahead of time - if len(str) > 0 { - if str[0] > 0x7f { - sr, srsz = utf8.DecodeRuneInString(str) - } else { - sr, srsz = rune(str[0]), 1 + for len(pat) > 0 { + var wild bool + pc, ps := rune(pat[0]), 1 + if pc > 0x7f { + pc, ps = utf8.DecodeRuneInString(pat) } - } else { - sr, srsz = utf8.RuneError, 0 - } - if len(pattern) > 0 { - if pattern[0] > 0x7f { - pr, prsz = utf8.DecodeRuneInString(pattern) - } else { - pr, prsz = rune(pattern[0]), 1 - } - } else { - pr, prsz = utf8.RuneError, 0 - } - // done reading - for pr != utf8.RuneError { - switch pr { - default: - if srsz == utf8.RuneError { - return false - } - if sr != pr { - return false - } - case '?': - if srsz == utf8.RuneError { - return false - } - case '*': - return deepMatchRune(str, pattern[prsz:]) || - (srsz > 0 && deepMatchRune(str[srsz:], pattern)) - } - str = str[srsz:] - pattern = pattern[prsz:] - // read the next runes + var sc rune + var ss int if len(str) > 0 { - if str[0] > 0x7f { - sr, srsz = utf8.DecodeRuneInString(str) - } else { - sr, srsz = rune(str[0]), 1 + sc, ss = rune(str[0]), 1 + if sc > 0x7f { + sc, ss = utf8.DecodeRuneInString(str) } - } else { - sr, srsz = utf8.RuneError, 0 } - if len(pattern) > 0 { - if pattern[0] > 0x7f { - pr, prsz = utf8.DecodeRuneInString(pattern) - } else { - pr, prsz = rune(pattern[0]), 1 + switch pc { + case '?': + if ss == 0 { + return rNoMatch + } + case '*': + // Ignore repeating stars. + for len(pat) > 1 && pat[1] == '*' { + pat = pat[1:] } - } else { - pr, prsz = utf8.RuneError, 0 - } - // done reading - } - return srsz == 0 && prsz == 0 + // If this star is the last character then it must be a match. + if len(pat) == 1 { + return rMatch + } + + // Match and trim any non-wildcard suffix characters. + var ok bool + str, pat, ok = matchTrimSuffix(str, pat) + if !ok { + return rNoMatch + } + + // Check for single star again. + if len(pat) == 1 { + return rMatch + } + + // Perform recursive wildcard search. + r := match(str, pat[1:], slen, counter, maxcomp) + if r != rNoMatch { + return r + } + if len(str) == 0 { + return rNoMatch + } + wild = true + default: + if ss == 0 { + return rNoMatch + } + if pc == '\\' { + pat = pat[ps:] + pc, ps = utf8.DecodeRuneInString(pat) + if ps == 0 { + return rNoMatch + } + } + if sc != pc { + return rNoMatch + } + } + str = str[ss:] + if !wild { + pat = pat[ps:] + } + } + if len(str) == 0 { + return rMatch + } + return rNoMatch } -var maxRuneBytes = func() []byte { - b := make([]byte, 4) - if utf8.EncodeRune(b, '\U0010FFFF') != 4 { - panic("invalid rune encoding") +// matchTrimSuffix matches and trims any non-wildcard suffix characters. +// Returns the trimed string and pattern. +// +// This is called because the pattern contains extra data after the wildcard +// star. Here we compare any suffix characters in the pattern to the suffix of +// the target string. Basically a reverse match that stops when a wildcard +// character is reached. This is a little trickier than a forward match because +// we need to evaluate an escaped character in reverse. +// +// Any matched characters will be trimmed from both the target +// string and the pattern. +func matchTrimSuffix(str, pat string) (string, string, bool) { + // It's expected that the pattern has at least two bytes and the first byte + // is a wildcard star '*' + match := true + for len(str) > 0 && len(pat) > 1 { + pc, ps := utf8.DecodeLastRuneInString(pat) + var esc bool + for i := 0; ; i++ { + if pat[len(pat)-ps-i-1] != '\\' { + if i&1 == 1 { + esc = true + ps++ + } + break + } + } + if pc == '*' && !esc { + match = true + break + } + sc, ss := utf8.DecodeLastRuneInString(str) + if !((pc == '?' && !esc) || pc == sc) { + match = false + break + } + str = str[:len(str)-ss] + pat = pat[:len(pat)-ps] } - return b -}() + return str, pat, match +} + +var maxRuneBytes = [...]byte{244, 143, 191, 191} // Allowable parses the pattern and determines the minimum and maximum allowable // values that the pattern can represent. @@ -157,7 +202,7 @@ func Allowable(pattern string) (min, max string) { } if pattern[i] == '?' { minb = append(minb, 0) - maxb = append(maxb, maxRuneBytes...) + maxb = append(maxb, maxRuneBytes[:]...) } else { minb = append(minb, pattern[i]) maxb = append(maxb, pattern[i]) diff --git a/vendor/github.com/tidwall/pretty/README.md b/vendor/github.com/tidwall/pretty/README.md index 7a614223..d3be5e54 100644 --- a/vendor/github.com/tidwall/pretty/README.md +++ b/vendor/github.com/tidwall/pretty/README.md @@ -79,46 +79,6 @@ Will format the json to: {"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}``` ``` -## Spec - -Spec cleans comments and trailing commas from input JSON, converting it to -valid JSON per the official spec: https://tools.ietf.org/html/rfc8259 - -The resulting JSON will always be the same length as the input and it will -include all of the same line breaks at matching offsets. This is to ensure -the result can be later processed by a external parser and that that -parser will report messages or errors with the correct offsets. - -The following example uses a JSON document that has comments and trailing -commas and converts it prior to unmarshalling to using the standard Go -JSON library. - -```go - -data := ` -{ - /* Dev Machine */ - "dbInfo": { - "host": "localhost", - "port": 5432, // use full email address - "username": "josh", - "password": "pass123", // use a hashed password - } - /* Only SMTP Allowed */ - "emailInfo": { - "email": "josh@example.com", - "password": "pass123", - "smtp": "smpt.example.com", - } -} -` - -err := json.Unmarshal(pretty.Spec(data), &config) - -``` - - - ## Customized output There's a `PrettyOptions(json, opts)` function which allows for customizing the output with the following options: @@ -143,14 +103,15 @@ type Options struct { Benchmarks of Pretty alongside the builtin `encoding/json` Indent/Compact methods. ``` -BenchmarkPretty-8 1000000 1283 ns/op 720 B/op 2 allocs/op -BenchmarkUgly-8 3000000 426 ns/op 240 B/op 1 allocs/op -BenchmarkUglyInPlace-8 5000000 340 ns/op 0 B/op 0 allocs/op -BenchmarkJSONIndent-8 300000 4628 ns/op 1069 B/op 4 allocs/op -BenchmarkJSONCompact-8 1000000 2469 ns/op 758 B/op 4 allocs/op +BenchmarkPretty-16 1000000 1034 ns/op 720 B/op 2 allocs/op +BenchmarkPrettySortKeys-16 586797 1983 ns/op 2848 B/op 14 allocs/op +BenchmarkUgly-16 4652365 254 ns/op 240 B/op 1 allocs/op +BenchmarkUglyInPlace-16 6481233 183 ns/op 0 B/op 0 allocs/op +BenchmarkJSONIndent-16 450654 2687 ns/op 1221 B/op 0 allocs/op +BenchmarkJSONCompact-16 685111 1699 ns/op 442 B/op 0 allocs/op ``` -*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.* +*These benchmarks were run on a MacBook Pro 2.4 GHz 8-Core Intel Core i9.* ## Contact Josh Baker [@tidwall](http://twitter.com/tidwall) diff --git a/vendor/github.com/tidwall/pretty/pretty.go b/vendor/github.com/tidwall/pretty/pretty.go index 7fa9d639..f3f756aa 100644 --- a/vendor/github.com/tidwall/pretty/pretty.go +++ b/vendor/github.com/tidwall/pretty/pretty.go @@ -1,7 +1,10 @@ package pretty import ( + "bytes" + "encoding/json" "sort" + "strconv" ) // Options is Pretty options @@ -84,6 +87,14 @@ func ugly(dst, src []byte) []byte { return dst } +func isNaNOrInf(src []byte) bool { + return src[0] == 'i' || //Inf + src[0] == 'I' || // inf + src[0] == '+' || // +Inf + src[0] == 'N' || // Nan + (src[0] == 'n' && len(src) > 1 && src[1] != 'u') // nan +} + func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) { for ; i < len(json); i++ { if json[i] <= ' ' { @@ -92,7 +103,8 @@ func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, in if json[i] == '"' { return appendPrettyString(buf, json, i, nl) } - if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { + + if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' || isNaNOrInf(json[i:]) { return appendPrettyNumber(buf, json, i, nl) } if json[i] == '{' { @@ -121,6 +133,7 @@ type pair struct { type byKeyVal struct { sorted bool json []byte + buf []byte pairs []pair } @@ -128,21 +141,110 @@ func (arr *byKeyVal) Len() int { return len(arr.pairs) } func (arr *byKeyVal) Less(i, j int) bool { - key1 := arr.json[arr.pairs[i].kstart+1 : arr.pairs[i].kend-1] - key2 := arr.json[arr.pairs[j].kstart+1 : arr.pairs[j].kend-1] - if string(key1) < string(key2) { + if arr.isLess(i, j, byKey) { return true } - if string(key1) > string(key2) { + if arr.isLess(j, i, byKey) { return false } - return arr.pairs[i].vstart < arr.pairs[j].vstart + return arr.isLess(i, j, byVal) } func (arr *byKeyVal) Swap(i, j int) { arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i] arr.sorted = true } +type byKind int + +const ( + byKey byKind = 0 + byVal byKind = 1 +) + +type jtype int + +const ( + jnull jtype = iota + jfalse + jnumber + jstring + jtrue + jjson +) + +func getjtype(v []byte) jtype { + if len(v) == 0 { + return jnull + } + switch v[0] { + case '"': + return jstring + case 'f': + return jfalse + case 't': + return jtrue + case 'n': + return jnull + case '[', '{': + return jjson + default: + return jnumber + } +} + +func (arr *byKeyVal) isLess(i, j int, kind byKind) bool { + k1 := arr.json[arr.pairs[i].kstart:arr.pairs[i].kend] + k2 := arr.json[arr.pairs[j].kstart:arr.pairs[j].kend] + var v1, v2 []byte + if kind == byKey { + v1 = k1 + v2 = k2 + } else { + v1 = bytes.TrimSpace(arr.buf[arr.pairs[i].vstart:arr.pairs[i].vend]) + v2 = bytes.TrimSpace(arr.buf[arr.pairs[j].vstart:arr.pairs[j].vend]) + if len(v1) >= len(k1)+1 { + v1 = bytes.TrimSpace(v1[len(k1)+1:]) + } + if len(v2) >= len(k2)+1 { + v2 = bytes.TrimSpace(v2[len(k2)+1:]) + } + } + t1 := getjtype(v1) + t2 := getjtype(v2) + if t1 < t2 { + return true + } + if t1 > t2 { + return false + } + if t1 == jstring { + s1 := parsestr(v1) + s2 := parsestr(v2) + return string(s1) < string(s2) + } + if t1 == jnumber { + n1, _ := strconv.ParseFloat(string(v1), 64) + n2, _ := strconv.ParseFloat(string(v2), 64) + return n1 < n2 + } + return string(v1) < string(v2) + +} + +func parsestr(s []byte) []byte { + for i := 1; i < len(s); i++ { + if s[i] == '\\' { + var str string + json.Unmarshal(s, &str) + return []byte(str) + } + if s[i] == '"' { + return s[1:i] + } + } + return nil +} + func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) { var ok bool if width > 0 { @@ -249,7 +351,7 @@ func sortPairs(json, buf []byte, pairs []pair) []byte { } vstart := pairs[0].vstart vend := pairs[len(pairs)-1].vend - arr := byKeyVal{false, json, pairs} + arr := byKeyVal{false, json, buf, pairs} sort.Stable(&arr) if !arr.sorted { return buf @@ -446,7 +548,7 @@ func Color(src []byte, style *Style) []byte { dst = apnd(dst, src[i]) } else { var kind byte - if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' { + if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' || isNaNOrInf(src[i:]) { kind = '0' dst = append(dst, style.Number[0]...) } else if src[i] == 't' { diff --git a/vendor/modules.txt b/vendor/modules.txt index 38f8649c..1fb3dae4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -42,23 +42,23 @@ github.com/okzk/sdnotify ## explicit # github.com/stretchr/testify v1.4.0 ## explicit -# github.com/tidwall/btree v0.6.0 +# github.com/tidwall/btree v0.6.1 ## explicit; go 1.16 github.com/tidwall/btree -# github.com/tidwall/buntdb v1.2.6 +# github.com/tidwall/buntdb v1.2.7 ## explicit; go 1.16 github.com/tidwall/buntdb -# github.com/tidwall/gjson v1.8.0 +# github.com/tidwall/gjson v1.10.2 ## explicit; go 1.12 github.com/tidwall/gjson -# github.com/tidwall/grect v0.1.2 +# github.com/tidwall/grect v0.1.3 ## explicit; go 1.15 github.com/tidwall/grect -# github.com/tidwall/match v1.0.3 +# github.com/tidwall/match v1.1.1 ## explicit; go 1.15 github.com/tidwall/match -# github.com/tidwall/pretty v1.1.0 -## explicit +# github.com/tidwall/pretty v1.2.0 +## explicit; go 1.16 github.com/tidwall/pretty # github.com/tidwall/rtred v0.1.2 ## explicit; go 1.15