From d0e11f49adf9c836bfbc93b3dd277237dc57c6ba Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Sat, 13 Feb 2021 20:58:19 -0500 Subject: [PATCH] use the new goshuirc ircreader --- go.mod | 2 +- go.sum | 4 + irc/client.go | 4 +- irc/ircconn.go | 75 ++-------- irc/ircconn_test.go | 135 ------------------ .../goshuirc/irc-go/ircreader/ircreader.go | 117 +++++++++++++++ vendor/modules.txt | 3 +- 7 files changed, 141 insertions(+), 199 deletions(-) delete mode 100644 irc/ircconn_test.go create mode 100644 vendor/github.com/goshuirc/irc-go/ircreader/ircreader.go diff --git a/go.mod b/go.mod index bf86814a..50af4f90 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/go-sql-driver/mysql v1.5.0 github.com/go-test/deep v1.0.6 // indirect github.com/gorilla/websocket v1.4.2 - github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5 + github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a github.com/onsi/ginkgo v1.12.0 // indirect github.com/onsi/gomega v1.9.0 // indirect github.com/oragono/confusables v0.0.0-20201108231250-4ab98ab61fb1 diff --git a/go.sum b/go.sum index 72c16600..ac5acfa1 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,10 @@ github.com/goshuirc/irc-go v0.0.0-20201228002532-4e36cb3f41f1 h1:Kyyey3K8nhx60lt github.com/goshuirc/irc-go v0.0.0-20201228002532-4e36cb3f41f1/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug= github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5 h1:TXGvyYHJEBluqwI8d0V5/QmSnNxEYIMbfPE36B8CNK8= github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug= +github.com/goshuirc/irc-go v0.0.0-20210214005848-fcaabd19f360 h1:ChbmWPZwyfgsZd6zxw7B/4hWJE7ezmb69PezUM9+YA4= +github.com/goshuirc/irc-go v0.0.0-20210214005848-fcaabd19f360/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug= +github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a h1:PR1tw21nn93AwKmjEPA7IVHiT+ld9qgO1H32APCMvL0= +github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/irc/client.go b/irc/client.go index b39d2985..05c5220b 100644 --- a/irc/client.go +++ b/irc/client.go @@ -18,7 +18,9 @@ import ( "github.com/goshuirc/irc-go/ircfmt" "github.com/goshuirc/irc-go/ircmsg" + "github.com/goshuirc/irc-go/ircreader" ident "github.com/oragono/go-ident" + "github.com/oragono/oragono/irc/caps" "github.com/oragono/oragono/irc/connection_limits" "github.com/oragono/oragono/irc/flatip" @@ -689,7 +691,7 @@ func (client *Client) run(session *Session) { } else if err != nil { var quitMessage string switch err { - case errReadQ, errWSBinaryMessage: + case ircreader.ErrReadQ, errWSBinaryMessage: quitMessage = err.Error() default: quitMessage = "connection closed" diff --git a/irc/ircconn.go b/irc/ircconn.go index 916c8655..a1dc6387 100644 --- a/irc/ircconn.go +++ b/irc/ircconn.go @@ -6,12 +6,12 @@ package irc import ( "bytes" "errors" - "io" "net" "unicode/utf8" "github.com/gorilla/websocket" "github.com/goshuirc/irc-go/ircmsg" + "github.com/goshuirc/irc-go/ircreader" "github.com/oragono/oragono/irc/utils" ) @@ -23,7 +23,6 @@ const ( var ( crlf = []byte{'\r', '\n'} - errReadQ = errors.New("ReadQ Exceeded") errWSBinaryMessage = errors.New("WebSocket binary messages are unsupported") ) @@ -48,17 +47,14 @@ type IRCConn interface { type IRCStreamConn struct { conn *utils.WrappedConn - buf []byte - start int // start of valid (i.e., read but not yet consumed) data in the buffer - end int // end of valid data in the buffer - searchFrom int // start of valid data in the buffer not yet searched for \n - eof bool + reader ircreader.IRCReader } func NewIRCStreamConn(conn *utils.WrappedConn) *IRCStreamConn { - return &IRCStreamConn{ - conn: conn, - } + var c IRCStreamConn + c.conn = conn + c.reader.Initialize(conn.Conn, initialBufferSize, maxReadQBytes) + return &c } func (cc *IRCStreamConn) UnderlyingConn() *utils.WrappedConn { @@ -78,56 +74,13 @@ func (cc *IRCStreamConn) WriteLines(buffers [][]byte) (err error) { } func (cc *IRCStreamConn) ReadLine() ([]byte, error) { - for { - // try to find a terminated line in the buffered data already read - nlidx := bytes.IndexByte(cc.buf[cc.searchFrom:cc.end], '\n') - if nlidx != -1 { - // got a complete line - line := cc.buf[cc.start : cc.searchFrom+nlidx] - cc.start = cc.searchFrom + nlidx + 1 - cc.searchFrom = cc.start - if globalUtf8EnforcementSetting && !utf8.Valid(line) { - return line, errInvalidUtf8 - } else { - return line, nil - } - } - - if cc.start == 0 && len(cc.buf) == maxReadQBytes { - return nil, errReadQ // out of space, can't expand or slide - } - - if cc.eof { - return nil, io.EOF - } - - if len(cc.buf) < maxReadQBytes && (len(cc.buf)-(cc.end-cc.start) < initialBufferSize/2) { - // allocate a new buffer, copy any remaining data - newLen := utils.RoundUpToPowerOfTwo(len(cc.buf) + 1) - if newLen > maxReadQBytes { - newLen = maxReadQBytes - } else if newLen < initialBufferSize { - newLen = initialBufferSize - } - newBuf := make([]byte, newLen) - copy(newBuf, cc.buf[cc.start:cc.end]) - cc.buf = newBuf - } else if cc.start != 0 { - // slide remaining data back to the front of the buffer - copy(cc.buf, cc.buf[cc.start:cc.end]) - } - cc.end = cc.end - cc.start - cc.start = 0 - - cc.searchFrom = cc.end - n, err := cc.conn.Read(cc.buf[cc.end:]) - cc.end += n - if n != 0 && err == io.EOF { - // we may have received new \n-terminated lines, try to parse them - cc.eof = true - } else if err != nil { - return nil, err - } + line, err := cc.reader.ReadLine() + if err != nil { + return nil, err + } else if globalUtf8EnforcementSetting && !utf8.Valid(line) { + return line, errInvalidUtf8 + } else { + return line, nil } } @@ -175,7 +128,7 @@ func (wc IRCWSConn) ReadLine() (line []byte, err error) { return nil, errWSBinaryMessage } } else if err == websocket.ErrReadLimit { - return line, errReadQ + return line, ircreader.ErrReadQ } else { return line, err } diff --git a/irc/ircconn_test.go b/irc/ircconn_test.go deleted file mode 100644 index 13a9930c..00000000 --- a/irc/ircconn_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) 2020 Shivaram Lingamneni -// released under the MIT license - -package irc - -import ( - "io" - "math/rand" - "net" - "reflect" - "testing" - "time" - - "github.com/oragono/oragono/irc/utils" -) - -// mockConn is a fake net.Conn / io.Reader that yields len(counts) lines, -// each consisting of counts[i] 'a' characters and a terminating '\n' -type mockConn struct { - counts []int -} - -func min(i, j int) (m int) { - if i < j { - return i - } else { - return j - } -} - -func (c *mockConn) Read(b []byte) (n int, err error) { - for len(b) > 0 { - if len(c.counts) == 0 { - return n, io.EOF - } - if c.counts[0] == 0 { - b[0] = '\n' - c.counts = c.counts[1:] - b = b[1:] - n += 1 - continue - } - size := min(c.counts[0], len(b)) - for i := 0; i < size; i++ { - b[i] = 'a' - } - c.counts[0] -= size - b = b[size:] - n += size - } - return n, nil -} - -func (c *mockConn) Write(b []byte) (n int, err error) { - return -} - -func (c *mockConn) Close() error { - c.counts = nil - return nil -} - -func (c *mockConn) LocalAddr() net.Addr { - return nil -} - -func (c *mockConn) RemoteAddr() net.Addr { - return nil -} - -func (c *mockConn) SetDeadline(t time.Time) error { - return nil -} - -func (c *mockConn) SetReadDeadline(t time.Time) error { - return nil -} - -func (c *mockConn) SetWriteDeadline(t time.Time) error { - return nil -} - -func newMockConn(counts []int) *utils.WrappedConn { - cpCounts := make([]int, len(counts)) - copy(cpCounts, counts) - c := &mockConn{ - counts: cpCounts, - } - return &utils.WrappedConn{ - Conn: c, - } -} - -// construct a mock reader with some number of \n-terminated lines, -// verify that IRCStreamConn can read and split them as expected -func doLineReaderTest(counts []int, t *testing.T) { - c := newMockConn(counts) - r := NewIRCStreamConn(c) - var readCounts []int - for { - line, err := r.ReadLine() - if err == nil { - readCounts = append(readCounts, len(line)) - } else if err == io.EOF { - break - } else { - panic(err) - } - } - - if !reflect.DeepEqual(counts, readCounts) { - t.Errorf("expected %#v, got %#v", counts, readCounts) - } -} - -const ( - maxMockReaderLen = 100 - maxMockReaderLineLen = 4096 + 511 -) - -func TestLineReader(t *testing.T) { - counts := []int{44, 428, 3, 0, 200, 2000, 0, 4044, 33, 3, 2, 1, 0, 1, 2, 3, 48, 555} - doLineReaderTest(counts, t) - - // fuzz - r := rand.New(rand.NewSource(time.Now().UnixNano())) - for i := 0; i < 1000; i++ { - countsLen := r.Intn(maxMockReaderLen) + 1 - counts := make([]int, countsLen) - for i := 0; i < countsLen; i++ { - counts[i] = r.Intn(maxMockReaderLineLen) - } - doLineReaderTest(counts, t) - } -} diff --git a/vendor/github.com/goshuirc/irc-go/ircreader/ircreader.go b/vendor/github.com/goshuirc/irc-go/ircreader/ircreader.go new file mode 100644 index 00000000..7e019b53 --- /dev/null +++ b/vendor/github.com/goshuirc/irc-go/ircreader/ircreader.go @@ -0,0 +1,117 @@ +// Copyright (c) 2020-2021 Shivaram Lingamneni +// released under the MIT license + +package ircreader + +import ( + "bytes" + "errors" + "io" +) + +/* +IRCReader is an optimized line reader for IRC lines containing tags; +most IRC lines will not approach the maximum line length (8191 bytes +of tag data, plus 512 bytes of message data), so we want a buffered +reader that can start with a smaller buffer and expand if necessary, +while also maintaining a hard upper limit on the size of the buffer. +*/ + +var ( + ErrReadQ = errors.New("readQ exceeded (read too many bytes without terminating newline)") +) + +type IRCReader struct { + conn io.Reader + + initialSize int + maxSize int + + buf []byte + start int // start of valid (i.e., read but not yet consumed) data in the buffer + end int // end of valid data in the buffer + searchFrom int // start of valid data in the buffer not yet searched for \n + eof bool +} + +// Returns a new *IRCReader with sane buffer size limits. +func NewIRCReader(conn io.Reader) *IRCReader { + var reader IRCReader + reader.Initialize(conn, 512, 8192+1024) + return &reader +} + +// "Placement new" for an IRCReader; initializes it with custom buffer size +// limits. +func (cc *IRCReader) Initialize(conn io.Reader, initialSize, maxSize int) { + *cc = IRCReader{} + cc.conn = conn + cc.initialSize = initialSize + cc.maxSize = maxSize +} + +// Blocks until a full IRC line is read, then returns it. Accepts either \n +// or \r\n as the line terminator (but not \r in isolation). Passes through +// errors from the underlying connection. Returns ErrReadQ if the buffer limit +// was exceeded without a terminating \n. +func (cc *IRCReader) ReadLine() ([]byte, error) { + for { + // try to find a terminated line in the buffered data already read + nlidx := bytes.IndexByte(cc.buf[cc.searchFrom:cc.end], '\n') + if nlidx != -1 { + // got a complete line + line := cc.buf[cc.start : cc.searchFrom+nlidx] + cc.start = cc.searchFrom + nlidx + 1 + cc.searchFrom = cc.start + return line, nil + } + + if cc.start == 0 && len(cc.buf) == cc.maxSize { + return nil, ErrReadQ // out of space, can't expand or slide + } + + if cc.eof { + return nil, io.EOF + } + + if len(cc.buf) < cc.maxSize && (len(cc.buf)-(cc.end-cc.start) < cc.initialSize/2) { + // allocate a new buffer, copy any remaining data + newLen := roundUpToPowerOfTwo(len(cc.buf) + 1) + if newLen > cc.maxSize { + newLen = cc.maxSize + } else if newLen < cc.initialSize { + newLen = cc.initialSize + } + newBuf := make([]byte, newLen) + copy(newBuf, cc.buf[cc.start:cc.end]) + cc.buf = newBuf + } else if cc.start != 0 { + // slide remaining data back to the front of the buffer + copy(cc.buf, cc.buf[cc.start:cc.end]) + } + cc.end = cc.end - cc.start + cc.start = 0 + + cc.searchFrom = cc.end + n, err := cc.conn.Read(cc.buf[cc.end:]) + cc.end += n + if n != 0 && err == io.EOF { + // we may have received new \n-terminated lines, try to parse them + cc.eof = true + } else if err != nil { + return nil, err + } + } +} + +// return n such that v <= n and n == 2**i for some i +func roundUpToPowerOfTwo(v int) int { + // http://graphics.stanford.edu/~seander/bithacks.html + v -= 1 + v |= v >> 1 + v |= v >> 2 + v |= v >> 4 + v |= v >> 8 + v |= v >> 16 + return v + 1 +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2664e2cb..0812d26e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -21,10 +21,11 @@ github.com/go-sql-driver/mysql # github.com/gorilla/websocket v1.4.2 ## explicit github.com/gorilla/websocket -# github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5 +# github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a ## explicit github.com/goshuirc/irc-go/ircfmt github.com/goshuirc/irc-go/ircmsg +github.com/goshuirc/irc-go/ircreader # github.com/onsi/ginkgo v1.12.0 ## explicit # github.com/onsi/gomega v1.9.0