upgrade x/text

This commit is contained in:
Shivaram Lingamneni 2021-04-19 06:49:56 -04:00
parent c62edcc909
commit 6f56121662
39 changed files with 334 additions and 125 deletions

2
go.mod
View File

@ -20,7 +20,7 @@ require (
github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 // indirect github.com/tidwall/rtree v0.0.0-20201027154624-32188eeb08a8 // indirect
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208
golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc
golang.org/x/text v0.3.4 golang.org/x/text v0.3.6
gopkg.in/yaml.v2 v2.3.0 gopkg.in/yaml.v2 v2.3.0
) )

2
go.sum
View File

@ -130,6 +130,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build icu
// +build icu // +build icu
package cases package cases

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.10 && !go1.13
// +build go1.10,!go1.13 // +build go1.10,!go1.13
package cases package cases

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.13 && !go1.14
// +build go1.13,!go1.14 // +build go1.13,!go1.14
package cases package cases

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.14 && !go1.16
// +build go1.14,!go1.16 // +build go1.14,!go1.16
package cases package cases

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.16
// +build go1.16 // +build go1.16
package cases package cases

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build !go1.10
// +build !go1.10 // +build !go1.10
package cases package cases

View File

@ -303,9 +303,17 @@ func (t Tag) Extensions() []string {
// are of the allowed values defined for the Unicode locale extension ('u') in // are of the allowed values defined for the Unicode locale extension ('u') in
// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
// TypeForKey will traverse the inheritance chain to get the correct value. // TypeForKey will traverse the inheritance chain to get the correct value.
//
// If there are multiple types associated with a key, only the first will be
// returned. If there is no type associated with a key, it returns the empty
// string.
func (t Tag) TypeForKey(key string) string { func (t Tag) TypeForKey(key string) string {
if start, end, _ := t.findTypeForKey(key); end != start { if _, start, end, _ := t.findTypeForKey(key); end != start {
return t.str[start:end] s := t.str[start:end]
if p := strings.IndexByte(s, '-'); p >= 0 {
s = s[:p]
}
return s
} }
return "" return ""
} }
@ -329,13 +337,13 @@ func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
// Remove the setting if value is "". // Remove the setting if value is "".
if value == "" { if value == "" {
start, end, _ := t.findTypeForKey(key) start, sep, end, _ := t.findTypeForKey(key)
if start != end { if start != sep {
// Remove key tag and leading '-'.
start -= 4
// Remove a possible empty extension. // Remove a possible empty extension.
if (end == len(t.str) || t.str[end+2] == '-') && t.str[start-2] == '-' { switch {
case t.str[start-2] != '-': // has previous elements.
case end == len(t.str), // end of string
end+2 < len(t.str) && t.str[end+2] == '-': // end of extension
start -= 2 start -= 2
} }
if start == int(t.pVariant) && end == len(t.str) { if start == int(t.pVariant) && end == len(t.str) {
@ -381,14 +389,14 @@ func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
t.str = string(buf[:uStart+len(b)]) t.str = string(buf[:uStart+len(b)])
} else { } else {
s := t.str s := t.str
start, end, hasExt := t.findTypeForKey(key) start, sep, end, hasExt := t.findTypeForKey(key)
if start == end { if start == sep {
if hasExt { if hasExt {
b = b[2:] b = b[2:]
} }
t.str = fmt.Sprintf("%s-%s%s", s[:start], b, s[end:]) t.str = fmt.Sprintf("%s-%s%s", s[:sep], b, s[end:])
} else { } else {
t.str = fmt.Sprintf("%s%s%s", s[:start], value, s[end:]) t.str = fmt.Sprintf("%s-%s%s", s[:start+3], value, s[end:])
} }
} }
return t, nil return t, nil
@ -399,10 +407,10 @@ func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
// wasn't found. The hasExt return value reports whether an -u extension was present. // wasn't found. The hasExt return value reports whether an -u extension was present.
// Note: the extensions are typically very small and are likely to contain // Note: the extensions are typically very small and are likely to contain
// only one key-type pair. // only one key-type pair.
func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) { func (t Tag) findTypeForKey(key string) (start, sep, end int, hasExt bool) {
p := int(t.pExt) p := int(t.pExt)
if len(key) != 2 || p == len(t.str) || p == 0 { if len(key) != 2 || p == len(t.str) || p == 0 {
return p, p, false return p, p, p, false
} }
s := t.str s := t.str
@ -410,10 +418,10 @@ func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) {
for p++; s[p] != 'u'; p++ { for p++; s[p] != 'u'; p++ {
if s[p] > 'u' { if s[p] > 'u' {
p-- p--
return p, p, false return p, p, p, false
} }
if p = nextExtension(s, p); p == len(s) { if p = nextExtension(s, p); p == len(s) {
return len(s), len(s), false return len(s), len(s), len(s), false
} }
} }
// Proceed to the hyphen following the extension name. // Proceed to the hyphen following the extension name.
@ -424,40 +432,28 @@ func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) {
// Iterate over keys until we get the end of a section. // Iterate over keys until we get the end of a section.
for { for {
// p points to the hyphen preceding the current token. end = p
if p3 := p + 3; s[p3] == '-' { for p++; p < len(s) && s[p] != '-'; p++ {
// Found a key. }
// Check whether we just processed the key that was requested. n := p - end - 1
if curKey == key { if n <= 2 && curKey == key {
return start, p, true if sep < end {
sep++
} }
// Set to the next key and continue scanning type tokens. return start, sep, end, true
curKey = s[p+1 : p3] }
switch n {
case 0, // invalid string
1: // next extension
return end, end, end, true
case 2:
// next key
curKey = s[end+1 : p]
if curKey > key { if curKey > key {
return p, p, true return end, end, end, true
} }
// Start of the type token sequence. start = end
start = p + 4 sep = p
// A type is at least 3 characters long.
p += 7 // 4 + 3
} else {
// Attribute or type, which is at least 3 characters long.
p += 4
}
// p points past the third character of a type or attribute.
max := p + 5 // maximum length of token plus hyphen.
if len(s) < max {
max = len(s)
}
for ; p < max && s[p] != '-'; p++ {
}
// Bail if we have exhausted all tokens or if the next token starts
// a new extension.
if p == len(s) || s[p+2] == '-' {
if curKey == key {
return start, p, true
}
return p, p, true
} }
} }
} }

View File

@ -133,14 +133,15 @@ func (s *scanner) resizeRange(oldStart, oldEnd, newSize int) {
s.start = oldStart s.start = oldStart
if end := oldStart + newSize; end != oldEnd { if end := oldStart + newSize; end != oldEnd {
diff := end - oldEnd diff := end - oldEnd
if end < cap(s.b) { var b []byte
b := make([]byte, len(s.b)+diff) if n := len(s.b) + diff; n > cap(s.b) {
b = make([]byte, n)
copy(b, s.b[:oldStart]) copy(b, s.b[:oldStart])
copy(b[end:], s.b[oldEnd:])
s.b = b
} else { } else {
s.b = append(s.b[end:], s.b[oldEnd:]...) b = s.b[:n]
} }
copy(b[end:], s.b[oldEnd:])
s.b = b
s.next = end + (s.next - s.end) s.next = end + (s.next - s.end)
s.end = end s.end = end
} }
@ -482,7 +483,7 @@ func parseExtensions(scan *scanner) int {
func parseExtension(scan *scanner) int { func parseExtension(scan *scanner) int {
start, end := scan.start, scan.end start, end := scan.start, scan.end
switch scan.token[0] { switch scan.token[0] {
case 'u': case 'u': // https://www.ietf.org/rfc/rfc6067.txt
attrStart := end attrStart := end
scan.scan() scan.scan()
for last := []byte{}; len(scan.token) > 2; scan.scan() { for last := []byte{}; len(scan.token) > 2; scan.scan() {
@ -502,27 +503,29 @@ func parseExtension(scan *scanner) int {
last = scan.token last = scan.token
end = scan.end end = scan.end
} }
// Scan key-type sequences. A key is of length 2 and may be followed
// by 0 or more "type" subtags from 3 to the maximum of 8 letters.
var last, key []byte var last, key []byte
for attrEnd := end; len(scan.token) == 2; last = key { for attrEnd := end; len(scan.token) == 2; last = key {
key = scan.token key = scan.token
keyEnd := scan.end end = scan.end
end = scan.acceptMinSize(3) for scan.scan(); end < scan.end && len(scan.token) > 2; scan.scan() {
end = scan.end
}
// TODO: check key value validity // TODO: check key value validity
if keyEnd == end || bytes.Compare(key, last) != 1 { if bytes.Compare(key, last) != 1 || scan.err != nil {
// We have an invalid key or the keys are not sorted. // We have an invalid key or the keys are not sorted.
// Start scanning keys from scratch and reorder. // Start scanning keys from scratch and reorder.
p := attrEnd + 1 p := attrEnd + 1
scan.next = p scan.next = p
keys := [][]byte{} keys := [][]byte{}
for scan.scan(); len(scan.token) == 2; { for scan.scan(); len(scan.token) == 2; {
keyStart, keyEnd := scan.start, scan.end keyStart := scan.start
end = scan.acceptMinSize(3) end = scan.end
if keyEnd != end { for scan.scan(); end < scan.end && len(scan.token) > 2; scan.scan() {
keys = append(keys, scan.b[keyStart:end]) end = scan.end
} else {
scan.setError(ErrSyntax)
end = keyStart
} }
keys = append(keys, scan.b[keyStart:end])
} }
sort.Stable(bytesSort{keys, 2}) sort.Stable(bytesSort{keys, 2})
if n := len(keys); n > 0 { if n := len(keys); n > 0 {
@ -546,7 +549,7 @@ func parseExtension(scan *scanner) int {
break break
} }
} }
case 't': case 't': // https://www.ietf.org/rfc/rfc6497.txt
scan.scan() scan.scan()
if n := len(scan.token); n >= 2 && n <= 3 && isAlpha(scan.token[1]) { if n := len(scan.token); n >= 2 && n <= 3 && isAlpha(scan.token[1]) {
_, end = parseTag(scan) _, end = parseTag(scan)

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !go1.2
// +build !go1.2 // +build !go1.2
package language package language

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build go1.2
// +build go1.2 // +build go1.2
package language package language

View File

@ -412,6 +412,10 @@ func (t Tag) Extensions() []Extension {
// are of the allowed values defined for the Unicode locale extension ('u') in // are of the allowed values defined for the Unicode locale extension ('u') in
// https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers. // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
// TypeForKey will traverse the inheritance chain to get the correct value. // TypeForKey will traverse the inheritance chain to get the correct value.
//
// If there are multiple types associated with a key, only the first will be
// returned. If there is no type associated with a key, it returns the empty
// string.
func (t Tag) TypeForKey(key string) string { func (t Tag) TypeForKey(key string) string {
if !compact.Tag(t).MayHaveExtensions() { if !compact.Tag(t).MayHaveExtensions() {
if key != "rg" && key != "va" { if key != "rg" && key != "va" {

View File

@ -47,7 +47,7 @@ const (
_Zzzz = 251 _Zzzz = 251
) )
var regionToGroups = []uint8{ // 357 elements var regionToGroups = []uint8{ // 358 elements
// Entry 0 - 3F // Entry 0 - 3F
0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x04,
0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x00,
@ -98,8 +98,8 @@ var regionToGroups = []uint8{ // 357 elements
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
} // Size: 381 bytes } // Size: 382 bytes
var paradigmLocales = [][3]uint16{ // 3 elements var paradigmLocales = [][3]uint16{ // 3 elements
0: [3]uint16{0x139, 0x0, 0x7b}, 0: [3]uint16{0x139, 0x0, 0x7b},
@ -295,4 +295,4 @@ var matchRegion = []regionIntelligibility{ // 15 elements
14: {lang: 0x529, script: 0x3c, group: 0x80, distance: 0x5}, 14: {lang: 0x529, script: 0x3c, group: 0x80, distance: 0x5},
} // Size: 114 bytes } // Size: 114 bytes
// Total table size 1471 bytes (1KiB); checksum: 4CB1CD46 // Total table size 1472 bytes (1KiB); checksum: F86C669

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build go1.10
// +build go1.10 // +build go1.10
package bidirule package bidirule

View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !go1.10
// +build !go1.10 // +build !go1.10
package bidirule package bidirule

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.10 && !go1.13
// +build go1.10,!go1.13 // +build go1.10,!go1.13
package precis package precis

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.13 && !go1.14
// +build go1.13,!go1.14 // +build go1.13,!go1.14
package precis package precis

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.14 && !go1.16
// +build go1.14,!go1.16 // +build go1.14,!go1.16
package precis package precis

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.16
// +build go1.16 // +build go1.16
package precis package precis

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build !go1.10
// +build !go1.10 // +build !go1.10
package precis package precis

View File

@ -12,15 +12,14 @@
// and without notice. // and without notice.
package bidi // import "golang.org/x/text/unicode/bidi" package bidi // import "golang.org/x/text/unicode/bidi"
// TODO: // TODO
// The following functionality would not be hard to implement, but hinges on
// the definition of a Segmenter interface. For now this is up to the user.
// - Iterate over paragraphs
// - Segmenter to iterate over runs directly from a given text.
// Also:
// - Transformer for reordering? // - Transformer for reordering?
// - Transformer (validator, really) for Bidi Rule. // - Transformer (validator, really) for Bidi Rule.
import (
"bytes"
)
// This API tries to avoid dealing with embedding levels for now. Under the hood // This API tries to avoid dealing with embedding levels for now. Under the hood
// these will be computed, but the question is to which extent the user should // these will be computed, but the question is to which extent the user should
// know they exist. We should at some point allow the user to specify an // know they exist. We should at some point allow the user to specify an
@ -49,7 +48,9 @@ const (
Neutral Neutral
) )
type options struct{} type options struct {
defaultDirection Direction
}
// An Option is an option for Bidi processing. // An Option is an option for Bidi processing.
type Option func(*options) type Option func(*options)
@ -66,12 +67,62 @@ type Option func(*options)
// DefaultDirection sets the default direction for a Paragraph. The direction is // DefaultDirection sets the default direction for a Paragraph. The direction is
// overridden if the text contains directional characters. // overridden if the text contains directional characters.
func DefaultDirection(d Direction) Option { func DefaultDirection(d Direction) Option {
panic("unimplemented") return func(opts *options) {
opts.defaultDirection = d
}
} }
// A Paragraph holds a single Paragraph for Bidi processing. // A Paragraph holds a single Paragraph for Bidi processing.
type Paragraph struct { type Paragraph struct {
// buffers p []byte
o Ordering
opts []Option
types []Class
pairTypes []bracketType
pairValues []rune
runes []rune
options options
}
// Initialize the p.pairTypes, p.pairValues and p.types from the input previously
// set by p.SetBytes() or p.SetString(). Also limit the input up to (and including) a paragraph
// separator (bidi class B).
//
// The function p.Order() needs these values to be set, so this preparation could be postponed.
// But since the SetBytes and SetStrings functions return the length of the input up to the paragraph
// separator, the whole input needs to be processed anyway and should not be done twice.
//
// The function has the same return values as SetBytes() / SetString()
func (p *Paragraph) prepareInput() (n int, err error) {
p.runes = bytes.Runes(p.p)
bytecount := 0
// clear slices from previous SetString or SetBytes
p.pairTypes = nil
p.pairValues = nil
p.types = nil
for _, r := range p.runes {
props, i := LookupRune(r)
bytecount += i
cls := props.Class()
if cls == B {
return bytecount, nil
}
p.types = append(p.types, cls)
if props.IsOpeningBracket() {
p.pairTypes = append(p.pairTypes, bpOpen)
p.pairValues = append(p.pairValues, r)
} else if props.IsBracket() {
// this must be a closing bracket,
// since IsOpeningBracket is not true
p.pairTypes = append(p.pairTypes, bpClose)
p.pairValues = append(p.pairValues, r)
} else {
p.pairTypes = append(p.pairTypes, bpNone)
p.pairValues = append(p.pairValues, 0)
}
}
return bytecount, nil
} }
// SetBytes configures p for the given paragraph text. It replaces text // SetBytes configures p for the given paragraph text. It replaces text
@ -80,70 +131,150 @@ type Paragraph struct {
// consumed from b including this separator. Error may be non-nil if options are // consumed from b including this separator. Error may be non-nil if options are
// given. // given.
func (p *Paragraph) SetBytes(b []byte, opts ...Option) (n int, err error) { func (p *Paragraph) SetBytes(b []byte, opts ...Option) (n int, err error) {
panic("unimplemented") p.p = b
p.opts = opts
return p.prepareInput()
} }
// SetString configures p for the given paragraph text. It replaces text // SetString configures s for the given paragraph text. It replaces text
// previously set by SetBytes or SetString. If b contains a paragraph separator // previously set by SetBytes or SetString. If s contains a paragraph separator
// it will only process the first paragraph and report the number of bytes // it will only process the first paragraph and report the number of bytes
// consumed from b including this separator. Error may be non-nil if options are // consumed from s including this separator. Error may be non-nil if options are
// given. // given.
func (p *Paragraph) SetString(s string, opts ...Option) (n int, err error) { func (p *Paragraph) SetString(s string, opts ...Option) (n int, err error) {
panic("unimplemented") p.p = []byte(s)
p.opts = opts
return p.prepareInput()
} }
// IsLeftToRight reports whether the principle direction of rendering for this // IsLeftToRight reports whether the principle direction of rendering for this
// paragraphs is left-to-right. If this returns false, the principle direction // paragraphs is left-to-right. If this returns false, the principle direction
// of rendering is right-to-left. // of rendering is right-to-left.
func (p *Paragraph) IsLeftToRight() bool { func (p *Paragraph) IsLeftToRight() bool {
panic("unimplemented") return p.Direction() == LeftToRight
} }
// Direction returns the direction of the text of this paragraph. // Direction returns the direction of the text of this paragraph.
// //
// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral. // The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
func (p *Paragraph) Direction() Direction { func (p *Paragraph) Direction() Direction {
panic("unimplemented") return p.o.Direction()
} }
// TODO: what happens if the position is > len(input)? This should return an error.
// RunAt reports the Run at the given position of the input text. // RunAt reports the Run at the given position of the input text.
// //
// This method can be used for computing line breaks on paragraphs. // This method can be used for computing line breaks on paragraphs.
func (p *Paragraph) RunAt(pos int) Run { func (p *Paragraph) RunAt(pos int) Run {
panic("unimplemented") c := 0
runNumber := 0
for i, r := range p.o.runes {
c += len(r)
if pos < c {
runNumber = i
}
}
return p.o.Run(runNumber)
}
func calculateOrdering(levels []level, runes []rune) Ordering {
var curDir Direction
prevDir := Neutral
prevI := 0
o := Ordering{}
// lvl = 0,2,4,...: left to right
// lvl = 1,3,5,...: right to left
for i, lvl := range levels {
if lvl%2 == 0 {
curDir = LeftToRight
} else {
curDir = RightToLeft
}
if curDir != prevDir {
if i > 0 {
o.runes = append(o.runes, runes[prevI:i])
o.directions = append(o.directions, prevDir)
o.startpos = append(o.startpos, prevI)
}
prevI = i
prevDir = curDir
}
}
o.runes = append(o.runes, runes[prevI:])
o.directions = append(o.directions, prevDir)
o.startpos = append(o.startpos, prevI)
return o
} }
// Order computes the visual ordering of all the runs in a Paragraph. // Order computes the visual ordering of all the runs in a Paragraph.
func (p *Paragraph) Order() (Ordering, error) { func (p *Paragraph) Order() (Ordering, error) {
panic("unimplemented") if len(p.types) == 0 {
return Ordering{}, nil
}
for _, fn := range p.opts {
fn(&p.options)
}
lvl := level(-1)
if p.options.defaultDirection == RightToLeft {
lvl = 1
}
para, err := newParagraph(p.types, p.pairTypes, p.pairValues, lvl)
if err != nil {
return Ordering{}, err
}
levels := para.getLevels([]int{len(p.types)})
p.o = calculateOrdering(levels, p.runes)
return p.o, nil
} }
// Line computes the visual ordering of runs for a single line starting and // Line computes the visual ordering of runs for a single line starting and
// ending at the given positions in the original text. // ending at the given positions in the original text.
func (p *Paragraph) Line(start, end int) (Ordering, error) { func (p *Paragraph) Line(start, end int) (Ordering, error) {
panic("unimplemented") lineTypes := p.types[start:end]
para, err := newParagraph(lineTypes, p.pairTypes[start:end], p.pairValues[start:end], -1)
if err != nil {
return Ordering{}, err
}
levels := para.getLevels([]int{len(lineTypes)})
o := calculateOrdering(levels, p.runes[start:end])
return o, nil
} }
// An Ordering holds the computed visual order of runs of a Paragraph. Calling // An Ordering holds the computed visual order of runs of a Paragraph. Calling
// SetBytes or SetString on the originating Paragraph invalidates an Ordering. // SetBytes or SetString on the originating Paragraph invalidates an Ordering.
// The methods of an Ordering should only be called by one goroutine at a time. // The methods of an Ordering should only be called by one goroutine at a time.
type Ordering struct{} type Ordering struct {
runes [][]rune
directions []Direction
startpos []int
}
// Direction reports the directionality of the runs. // Direction reports the directionality of the runs.
// //
// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral. // The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
func (o *Ordering) Direction() Direction { func (o *Ordering) Direction() Direction {
panic("unimplemented") return o.directions[0]
} }
// NumRuns returns the number of runs. // NumRuns returns the number of runs.
func (o *Ordering) NumRuns() int { func (o *Ordering) NumRuns() int {
panic("unimplemented") return len(o.runes)
} }
// Run returns the ith run within the ordering. // Run returns the ith run within the ordering.
func (o *Ordering) Run(i int) Run { func (o *Ordering) Run(i int) Run {
panic("unimplemented") r := Run{
runes: o.runes[i],
direction: o.directions[i],
startpos: o.startpos[i],
}
return r
} }
// TODO: perhaps with options. // TODO: perhaps with options.
@ -155,16 +286,19 @@ func (o *Ordering) Run(i int) Run {
// A Run is a continuous sequence of characters of a single direction. // A Run is a continuous sequence of characters of a single direction.
type Run struct { type Run struct {
runes []rune
direction Direction
startpos int
} }
// String returns the text of the run in its original order. // String returns the text of the run in its original order.
func (r *Run) String() string { func (r *Run) String() string {
panic("unimplemented") return string(r.runes)
} }
// Bytes returns the text of the run in its original order. // Bytes returns the text of the run in its original order.
func (r *Run) Bytes() []byte { func (r *Run) Bytes() []byte {
panic("unimplemented") return []byte(r.String())
} }
// TODO: methods for // TODO: methods for
@ -174,25 +308,52 @@ func (r *Run) Bytes() []byte {
// Direction reports the direction of the run. // Direction reports the direction of the run.
func (r *Run) Direction() Direction { func (r *Run) Direction() Direction {
panic("unimplemented") return r.direction
} }
// Position of the Run within the text passed to SetBytes or SetString of the // Pos returns the position of the Run within the text passed to SetBytes or SetString of the
// originating Paragraph value. // originating Paragraph value.
func (r *Run) Pos() (start, end int) { func (r *Run) Pos() (start, end int) {
panic("unimplemented") return r.startpos, r.startpos + len(r.runes) - 1
} }
// AppendReverse reverses the order of characters of in, appends them to out, // AppendReverse reverses the order of characters of in, appends them to out,
// and returns the result. Modifiers will still follow the runes they modify. // and returns the result. Modifiers will still follow the runes they modify.
// Brackets are replaced with their counterparts. // Brackets are replaced with their counterparts.
func AppendReverse(out, in []byte) []byte { func AppendReverse(out, in []byte) []byte {
panic("unimplemented") ret := make([]byte, len(in)+len(out))
copy(ret, out)
inRunes := bytes.Runes(in)
for i, r := range inRunes {
prop, _ := LookupRune(r)
if prop.IsBracket() {
inRunes[i] = prop.reverseBracket(r)
}
}
for i, j := 0, len(inRunes)-1; i < j; i, j = i+1, j-1 {
inRunes[i], inRunes[j] = inRunes[j], inRunes[i]
}
copy(ret[len(out):], string(inRunes))
return ret
} }
// ReverseString reverses the order of characters in s and returns a new string. // ReverseString reverses the order of characters in s and returns a new string.
// Modifiers will still follow the runes they modify. Brackets are replaced with // Modifiers will still follow the runes they modify. Brackets are replaced with
// their counterparts. // their counterparts.
func ReverseString(s string) string { func ReverseString(s string) string {
panic("unimplemented") input := []rune(s)
li := len(input)
ret := make([]rune, li)
for i, r := range input {
prop, _ := LookupRune(r)
if prop.IsBracket() {
ret[li-i-1] = prop.reverseBracket(r)
} else {
ret[li-i-1] = r
}
}
return string(ret)
} }

View File

@ -4,7 +4,10 @@
package bidi package bidi
import "log" import (
"fmt"
"log"
)
// This implementation is a port based on the reference implementation found at: // This implementation is a port based on the reference implementation found at:
// https://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/ // https://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/
@ -97,13 +100,20 @@ type paragraph struct {
// rune (suggested is the rune of the open bracket for opening and matching // rune (suggested is the rune of the open bracket for opening and matching
// close brackets, after normalization). The embedding levels are optional, but // close brackets, after normalization). The embedding levels are optional, but
// may be supplied to encode embedding levels of styled text. // may be supplied to encode embedding levels of styled text.
// func newParagraph(types []Class, pairTypes []bracketType, pairValues []rune, levels level) (*paragraph, error) {
// TODO: return an error. var err error
func newParagraph(types []Class, pairTypes []bracketType, pairValues []rune, levels level) *paragraph { if err = validateTypes(types); err != nil {
validateTypes(types) return nil, err
validatePbTypes(pairTypes) }
validatePbValues(pairValues, pairTypes) if err = validatePbTypes(pairTypes); err != nil {
validateParagraphEmbeddingLevel(levels) return nil, err
}
if err = validatePbValues(pairValues, pairTypes); err != nil {
return nil, err
}
if err = validateParagraphEmbeddingLevel(levels); err != nil {
return nil, err
}
p := &paragraph{ p := &paragraph{
initialTypes: append([]Class(nil), types...), initialTypes: append([]Class(nil), types...),
@ -115,7 +125,7 @@ func newParagraph(types []Class, pairTypes []bracketType, pairValues []rune, lev
resultTypes: append([]Class(nil), types...), resultTypes: append([]Class(nil), types...),
} }
p.run() p.run()
return p return p, nil
} }
func (p *paragraph) Len() int { return len(p.initialTypes) } func (p *paragraph) Len() int { return len(p.initialTypes) }
@ -1001,58 +1011,61 @@ func typeForLevel(level level) Class {
return R return R
} }
// TODO: change validation to not panic func validateTypes(types []Class) error {
func validateTypes(types []Class) {
if len(types) == 0 { if len(types) == 0 {
log.Panic("types is null") return fmt.Errorf("types is null")
} }
for i, t := range types[:len(types)-1] { for i, t := range types[:len(types)-1] {
if t == B { if t == B {
log.Panicf("B type before end of paragraph at index: %d", i) return fmt.Errorf("B type before end of paragraph at index: %d", i)
} }
} }
return nil
} }
func validateParagraphEmbeddingLevel(embeddingLevel level) { func validateParagraphEmbeddingLevel(embeddingLevel level) error {
if embeddingLevel != implicitLevel && if embeddingLevel != implicitLevel &&
embeddingLevel != 0 && embeddingLevel != 0 &&
embeddingLevel != 1 { embeddingLevel != 1 {
log.Panicf("illegal paragraph embedding level: %d", embeddingLevel) return fmt.Errorf("illegal paragraph embedding level: %d", embeddingLevel)
} }
return nil
} }
func validateLineBreaks(linebreaks []int, textLength int) { func validateLineBreaks(linebreaks []int, textLength int) error {
prev := 0 prev := 0
for i, next := range linebreaks { for i, next := range linebreaks {
if next <= prev { if next <= prev {
log.Panicf("bad linebreak: %d at index: %d", next, i) return fmt.Errorf("bad linebreak: %d at index: %d", next, i)
} }
prev = next prev = next
} }
if prev != textLength { if prev != textLength {
log.Panicf("last linebreak was %d, want %d", prev, textLength) return fmt.Errorf("last linebreak was %d, want %d", prev, textLength)
} }
return nil
} }
func validatePbTypes(pairTypes []bracketType) { func validatePbTypes(pairTypes []bracketType) error {
if len(pairTypes) == 0 { if len(pairTypes) == 0 {
log.Panic("pairTypes is null") return fmt.Errorf("pairTypes is null")
} }
for i, pt := range pairTypes { for i, pt := range pairTypes {
switch pt { switch pt {
case bpNone, bpOpen, bpClose: case bpNone, bpOpen, bpClose:
default: default:
log.Panicf("illegal pairType value at %d: %v", i, pairTypes[i]) return fmt.Errorf("illegal pairType value at %d: %v", i, pairTypes[i])
} }
} }
return nil
} }
func validatePbValues(pairValues []rune, pairTypes []bracketType) { func validatePbValues(pairValues []rune, pairTypes []bracketType) error {
if pairValues == nil { if pairValues == nil {
log.Panic("pairValues is null") return fmt.Errorf("pairValues is null")
} }
if len(pairTypes) != len(pairValues) { if len(pairTypes) != len(pairValues) {
log.Panic("pairTypes is different length from pairValues") return fmt.Errorf("pairTypes is different length from pairValues")
} }
return nil
} }

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.10 && !go1.13
// +build go1.10,!go1.13 // +build go1.10,!go1.13
package bidi package bidi

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.13 && !go1.14
// +build go1.13,!go1.14 // +build go1.13,!go1.14
package bidi package bidi

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.14 && !go1.16
// +build go1.14,!go1.16 // +build go1.14,!go1.16
package bidi package bidi

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.16
// +build go1.16 // +build go1.16
package bidi package bidi

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build !go1.10
// +build !go1.10 // +build !go1.10
package bidi package bidi

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.10 && !go1.13
// +build go1.10,!go1.13 // +build go1.10,!go1.13
package norm package norm

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.13 && !go1.14
// +build go1.13,!go1.14 // +build go1.13,!go1.14
package norm package norm

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.14 && !go1.16
// +build go1.14,!go1.16 // +build go1.14,!go1.16
package norm package norm

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.16
// +build go1.16 // +build go1.16
package norm package norm

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build !go1.10
// +build !go1.10 // +build !go1.10
package norm package norm

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.10 && !go1.13
// +build go1.10,!go1.13 // +build go1.10,!go1.13
package width package width

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.13 && !go1.14
// +build go1.13,!go1.14 // +build go1.13,!go1.14
package width package width

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.14 && !go1.16
// +build go1.14,!go1.16 // +build go1.14,!go1.16
package width package width

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build go1.16
// +build go1.16 // +build go1.16
package width package width

View File

@ -1,5 +1,6 @@
// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
//go:build !go1.10
// +build !go1.10 // +build !go1.10
package width package width

2
vendor/modules.txt vendored
View File

@ -77,7 +77,7 @@ golang.org/x/sys/unix
golang.org/x/sys/windows golang.org/x/sys/windows
# golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 # golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
golang.org/x/term golang.org/x/term
# golang.org/x/text v0.3.4 # golang.org/x/text v0.3.6
## explicit ## explicit
golang.org/x/text/cases golang.org/x/text/cases
golang.org/x/text/internal golang.org/x/text/internal