Merge pull request #1536 from slingamn/ircreader.1

use the new goshuirc ircreader

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-test/deep v1.0.6 // indirect
 	github.com/gorilla/websocket v1.4.2
-	github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5
+	github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a
 	github.com/onsi/ginkgo v1.12.0 // indirect
 	github.com/onsi/gomega v1.9.0 // indirect
 	github.com/oragono/confusables v0.0.0-20201108231250-4ab98ab61fb1

go.sum

@@ -28,6 +28,10 @@ github.com/goshuirc/irc-go v0.0.0-20201228002532-4e36cb3f41f1 h1:Kyyey3K8nhx60lt
 github.com/goshuirc/irc-go v0.0.0-20201228002532-4e36cb3f41f1/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug=
 github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5 h1:TXGvyYHJEBluqwI8d0V5/QmSnNxEYIMbfPE36B8CNK8=
 github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug=
+github.com/goshuirc/irc-go v0.0.0-20210214005848-fcaabd19f360 h1:ChbmWPZwyfgsZd6zxw7B/4hWJE7ezmb69PezUM9+YA4=
+github.com/goshuirc/irc-go v0.0.0-20210214005848-fcaabd19f360/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug=
+github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a h1:PR1tw21nn93AwKmjEPA7IVHiT+ld9qgO1H32APCMvL0=
+github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a/go.mod h1:q/JhvvKLmif3y9q8MDQM+gRCnjEKnu5ClF298TTXJug=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

irc/client.go

@@ -18,7 +18,9 @@ import (
 
 	"github.com/goshuirc/irc-go/ircfmt"
 	"github.com/goshuirc/irc-go/ircmsg"
+	"github.com/goshuirc/irc-go/ircreader"
 	ident "github.com/oragono/go-ident"
+
 	"github.com/oragono/oragono/irc/caps"
 	"github.com/oragono/oragono/irc/connection_limits"
 	"github.com/oragono/oragono/irc/flatip"
@@ -689,7 +691,7 @@ func (client *Client) run(session *Session) {
 		} else if err != nil {
 			var quitMessage string
 			switch err {
-			case errReadQ, errWSBinaryMessage:
+			case ircreader.ErrReadQ, errWSBinaryMessage:
 				quitMessage = err.Error()
 			default:
 				quitMessage = "connection closed"

irc/ircconn.go

@@ -6,12 +6,12 @@ package irc
 import (
 	"bytes"
 	"errors"
-	"io"
 	"net"
 	"unicode/utf8"
 
 	"github.com/gorilla/websocket"
 	"github.com/goshuirc/irc-go/ircmsg"
+	"github.com/goshuirc/irc-go/ircreader"
 
 	"github.com/oragono/oragono/irc/utils"
 )
@@ -23,7 +23,6 @@ const (
 
 var (
 	crlf               = []byte{'\r', '\n'}
-	errReadQ           = errors.New("ReadQ Exceeded")
 	errWSBinaryMessage = errors.New("WebSocket binary messages are unsupported")
 )
 
@@ -48,17 +47,14 @@ type IRCConn interface {
 type IRCStreamConn struct {
 	conn *utils.WrappedConn
 
-	buf        []byte
+	reader ircreader.IRCReader
-	start      int // start of valid (i.e., read but not yet consumed) data in the buffer
-	end        int // end of valid data in the buffer
-	searchFrom int // start of valid data in the buffer not yet searched for \n
-	eof        bool
 }
 
 func NewIRCStreamConn(conn *utils.WrappedConn) *IRCStreamConn {
-	return &IRCStreamConn{
+	var c IRCStreamConn
-		conn: conn,
+	c.conn = conn
-	}
+	c.reader.Initialize(conn.Conn, initialBufferSize, maxReadQBytes)
+	return &c
 }
 
 func (cc *IRCStreamConn) UnderlyingConn() *utils.WrappedConn {
@@ -78,59 +74,16 @@ func (cc *IRCStreamConn) WriteLines(buffers [][]byte) (err error) {
 }
 
 func (cc *IRCStreamConn) ReadLine() ([]byte, error) {
-	for {
+	line, err := cc.reader.ReadLine()
-		// try to find a terminated line in the buffered data already read
+	if err != nil {
-		nlidx := bytes.IndexByte(cc.buf[cc.searchFrom:cc.end], '\n')
+		return nil, err
-		if nlidx != -1 {
+	} else if globalUtf8EnforcementSetting && !utf8.Valid(line) {
-			// got a complete line
-			line := cc.buf[cc.start : cc.searchFrom+nlidx]
-			cc.start = cc.searchFrom + nlidx + 1
-			cc.searchFrom = cc.start
-			if globalUtf8EnforcementSetting && !utf8.Valid(line) {
 		return line, errInvalidUtf8
 	} else {
 		return line, nil
 	}
 }
 
-		if cc.start == 0 && len(cc.buf) == maxReadQBytes {
-			return nil, errReadQ // out of space, can't expand or slide
-		}
-
-		if cc.eof {
-			return nil, io.EOF
-		}
-
-		if len(cc.buf) < maxReadQBytes && (len(cc.buf)-(cc.end-cc.start) < initialBufferSize/2) {
-			// allocate a new buffer, copy any remaining data
-			newLen := utils.RoundUpToPowerOfTwo(len(cc.buf) + 1)
-			if newLen > maxReadQBytes {
-				newLen = maxReadQBytes
-			} else if newLen < initialBufferSize {
-				newLen = initialBufferSize
-			}
-			newBuf := make([]byte, newLen)
-			copy(newBuf, cc.buf[cc.start:cc.end])
-			cc.buf = newBuf
-		} else if cc.start != 0 {
-			// slide remaining data back to the front of the buffer
-			copy(cc.buf, cc.buf[cc.start:cc.end])
-		}
-		cc.end = cc.end - cc.start
-		cc.start = 0
-
-		cc.searchFrom = cc.end
-		n, err := cc.conn.Read(cc.buf[cc.end:])
-		cc.end += n
-		if n != 0 && err == io.EOF {
-			// we may have received new \n-terminated lines, try to parse them
-			cc.eof = true
-		} else if err != nil {
-			return nil, err
-		}
-	}
-}
-
 func (cc *IRCStreamConn) Close() (err error) {
 	return cc.conn.Close()
 }
@@ -175,7 +128,7 @@ func (wc IRCWSConn) ReadLine() (line []byte, err error) {
 			return nil, errWSBinaryMessage
 		}
 	} else if err == websocket.ErrReadLimit {
-		return line, errReadQ
+		return line, ircreader.ErrReadQ
 	} else {
 		return line, err
 	}

irc/ircconn_test.go

@@ -1,135 +0,0 @@
-// Copyright (c) 2020 Shivaram Lingamneni <slingamn@cs.stanford.edu>
-// released under the MIT license
-
-package irc
-
-import (
-	"io"
-	"math/rand"
-	"net"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/oragono/oragono/irc/utils"
-)
-
-// mockConn is a fake net.Conn / io.Reader that yields len(counts) lines,
-// each consisting of counts[i] 'a' characters and a terminating '\n'
-type mockConn struct {
-	counts []int
-}
-
-func min(i, j int) (m int) {
-	if i < j {
-		return i
-	} else {
-		return j
-	}
-}
-
-func (c *mockConn) Read(b []byte) (n int, err error) {
-	for len(b) > 0 {
-		if len(c.counts) == 0 {
-			return n, io.EOF
-		}
-		if c.counts[0] == 0 {
-			b[0] = '\n'
-			c.counts = c.counts[1:]
-			b = b[1:]
-			n += 1
-			continue
-		}
-		size := min(c.counts[0], len(b))
-		for i := 0; i < size; i++ {
-			b[i] = 'a'
-		}
-		c.counts[0] -= size
-		b = b[size:]
-		n += size
-	}
-	return n, nil
-}
-
-func (c *mockConn) Write(b []byte) (n int, err error) {
-	return
-}
-
-func (c *mockConn) Close() error {
-	c.counts = nil
-	return nil
-}
-
-func (c *mockConn) LocalAddr() net.Addr {
-	return nil
-}
-
-func (c *mockConn) RemoteAddr() net.Addr {
-	return nil
-}
-
-func (c *mockConn) SetDeadline(t time.Time) error {
-	return nil
-}
-
-func (c *mockConn) SetReadDeadline(t time.Time) error {
-	return nil
-}
-
-func (c *mockConn) SetWriteDeadline(t time.Time) error {
-	return nil
-}
-
-func newMockConn(counts []int) *utils.WrappedConn {
-	cpCounts := make([]int, len(counts))
-	copy(cpCounts, counts)
-	c := &mockConn{
-		counts: cpCounts,
-	}
-	return &utils.WrappedConn{
-		Conn: c,
-	}
-}
-
-// construct a mock reader with some number of \n-terminated lines,
-// verify that IRCStreamConn can read and split them as expected
-func doLineReaderTest(counts []int, t *testing.T) {
-	c := newMockConn(counts)
-	r := NewIRCStreamConn(c)
-	var readCounts []int
-	for {
-		line, err := r.ReadLine()
-		if err == nil {
-			readCounts = append(readCounts, len(line))
-		} else if err == io.EOF {
-			break
-		} else {
-			panic(err)
-		}
-	}
-
-	if !reflect.DeepEqual(counts, readCounts) {
-		t.Errorf("expected %#v, got %#v", counts, readCounts)
-	}
-}
-
-const (
-	maxMockReaderLen     = 100
-	maxMockReaderLineLen = 4096 + 511
-)
-
-func TestLineReader(t *testing.T) {
-	counts := []int{44, 428, 3, 0, 200, 2000, 0, 4044, 33, 3, 2, 1, 0, 1, 2, 3, 48, 555}
-	doLineReaderTest(counts, t)
-
-	// fuzz
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	for i := 0; i < 1000; i++ {
-		countsLen := r.Intn(maxMockReaderLen) + 1
-		counts := make([]int, countsLen)
-		for i := 0; i < countsLen; i++ {
-			counts[i] = r.Intn(maxMockReaderLineLen)
-		}
-		doLineReaderTest(counts, t)
-	}
-}

vendor/github.com/goshuirc/irc-go/ircreader/ircreader.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2020-2021 Shivaram Lingamneni
+// released under the MIT license
+
+package ircreader
+
+import (
+	"bytes"
+	"errors"
+	"io"
+)
+
+/*
+IRCReader is an optimized line reader for IRC lines containing tags;
+most IRC lines will not approach the maximum line length (8191 bytes
+of tag data, plus 512 bytes of message data), so we want a buffered
+reader that can start with a smaller buffer and expand if necessary,
+while also maintaining a hard upper limit on the size of the buffer.
+*/
+
+var (
+	ErrReadQ = errors.New("readQ exceeded (read too many bytes without terminating newline)")
+)
+
+type IRCReader struct {
+	conn io.Reader
+
+	initialSize int
+	maxSize     int
+
+	buf        []byte
+	start      int // start of valid (i.e., read but not yet consumed) data in the buffer
+	end        int // end of valid data in the buffer
+	searchFrom int // start of valid data in the buffer not yet searched for \n
+	eof        bool
+}
+
+// Returns a new *IRCReader with sane buffer size limits.
+func NewIRCReader(conn io.Reader) *IRCReader {
+	var reader IRCReader
+	reader.Initialize(conn, 512, 8192+1024)
+	return &reader
+}
+
+// "Placement new" for an IRCReader; initializes it with custom buffer size
+// limits.
+func (cc *IRCReader) Initialize(conn io.Reader, initialSize, maxSize int) {
+	*cc = IRCReader{}
+	cc.conn = conn
+	cc.initialSize = initialSize
+	cc.maxSize = maxSize
+}
+
+// Blocks until a full IRC line is read, then returns it. Accepts either \n
+// or \r\n as the line terminator (but not \r in isolation). Passes through
+// errors from the underlying connection. Returns ErrReadQ if the buffer limit
+// was exceeded without a terminating \n.
+func (cc *IRCReader) ReadLine() ([]byte, error) {
+	for {
+		// try to find a terminated line in the buffered data already read
+		nlidx := bytes.IndexByte(cc.buf[cc.searchFrom:cc.end], '\n')
+		if nlidx != -1 {
+			// got a complete line
+			line := cc.buf[cc.start : cc.searchFrom+nlidx]
+			cc.start = cc.searchFrom + nlidx + 1
+			cc.searchFrom = cc.start
+			return line, nil
+		}
+
+		if cc.start == 0 && len(cc.buf) == cc.maxSize {
+			return nil, ErrReadQ // out of space, can't expand or slide
+		}
+
+		if cc.eof {
+			return nil, io.EOF
+		}
+
+		if len(cc.buf) < cc.maxSize && (len(cc.buf)-(cc.end-cc.start) < cc.initialSize/2) {
+			// allocate a new buffer, copy any remaining data
+			newLen := roundUpToPowerOfTwo(len(cc.buf) + 1)
+			if newLen > cc.maxSize {
+				newLen = cc.maxSize
+			} else if newLen < cc.initialSize {
+				newLen = cc.initialSize
+			}
+			newBuf := make([]byte, newLen)
+			copy(newBuf, cc.buf[cc.start:cc.end])
+			cc.buf = newBuf
+		} else if cc.start != 0 {
+			// slide remaining data back to the front of the buffer
+			copy(cc.buf, cc.buf[cc.start:cc.end])
+		}
+		cc.end = cc.end - cc.start
+		cc.start = 0
+
+		cc.searchFrom = cc.end
+		n, err := cc.conn.Read(cc.buf[cc.end:])
+		cc.end += n
+		if n != 0 && err == io.EOF {
+			// we may have received new \n-terminated lines, try to parse them
+			cc.eof = true
+		} else if err != nil {
+			return nil, err
+		}
+	}
+}
+
+// return n such that v <= n and n == 2**i for some i
+func roundUpToPowerOfTwo(v int) int {
+	// http://graphics.stanford.edu/~seander/bithacks.html
+	v -= 1
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	return v + 1
+}

vendor/modules.txt

@@ -21,10 +21,11 @@ github.com/go-sql-driver/mysql
 # github.com/gorilla/websocket v1.4.2
 ## explicit
 github.com/gorilla/websocket
-# github.com/goshuirc/irc-go v0.0.0-20210108124156-ec778d0252a5
+# github.com/goshuirc/irc-go v0.0.0-20210214015142-9d703e6ac38a
 ## explicit
 github.com/goshuirc/irc-go/ircfmt
 github.com/goshuirc/irc-go/ircmsg
+github.com/goshuirc/irc-go/ircreader
 # github.com/onsi/ginkgo v1.12.0
 ## explicit
 # github.com/onsi/gomega v1.9.0