implement draft/multiline

gencapdefs.py

@@ -177,6 +177,12 @@ CAPDEFS = [
         url="https://oragono.io/nope",
         standard="Oragono vendor",
     ),
+    CapDef(
+        identifier="Multiline",
+        name="draft/multiline",
+        url="https://github.com/ircv3/ircv3-specifications/pull/398",
+        standard="Proposed IRCv3",
+    ),
 ]
 
 def validate_defs():

irc/caps/constants.go

@@ -55,6 +55,10 @@ const (
 	// LabelTagName is the tag name used for the labeled-response spec.
 	// https://ircv3.net/specs/extensions/labeled-response.html
 	LabelTagName = "draft/label"
+	// More draft names associated with draft/multiline:
+	MultilineBatchType = "draft/multiline"
+	MultilineConcatTag = "draft/multiline-concat"
+	MultilineFmsgidTag = "draft/fmsgid"
 )
 
 func init() {

irc/caps/defs.go

@@ -7,7 +7,7 @@ package caps
 
 const (
 	// number of recognized capabilities:
-	numCapabs = 27
+	numCapabs = 28
 	// length of the uint64 array that represents the bitset:
 	bitsetLen = 1
 )
@@ -53,6 +53,10 @@ const (
 	// https://gist.github.com/DanielOaks/8126122f74b26012a3de37db80e4e0c6
 	Languages Capability = iota
 
+	// Multiline is the Proposed IRCv3 capability named "draft/multiline":
+	// https://github.com/ircv3/ircv3-specifications/pull/398
+	Multiline Capability = iota
+
 	// Rename is the proposed IRCv3 capability named "draft/rename":
 	// https://github.com/SaberUK/ircv3-specifications/blob/rename/extensions/rename.md
 	Rename Capability = iota
@@ -135,6 +139,7 @@ var (
 		"draft/event-playback",
 		"draft/labeled-response-0.2",
 		"draft/languages",
+		"draft/multiline",
 		"draft/rename",
 		"draft/resume-0.5",
 		"draft/setname",

irc/channel.go

@@ -1042,7 +1042,7 @@ func (channel *Channel) CanSpeak(client *Client) bool {
 	return true
 }
 
-func msgCommandToHistType(server *Server, command string) (history.ItemType, error) {
+func msgCommandToHistType(command string) (history.ItemType, error) {
 	switch command {
 	case "PRIVMSG":
 		return history.Privmsg, nil
@@ -1051,13 +1051,23 @@ func msgCommandToHistType(server *Server, command string) (history.ItemType, err
 	case "TAGMSG":
 		return history.Tagmsg, nil
 	default:
-		server.logger.Error("internal", "unrecognized messaging command", command)
 		return history.ItemType(0), errInvalidParams
 	}
 }
 
+func histTypeToMsgCommand(t history.ItemType) string {
+	switch t {
+	case history.Notice:
+		return "NOTICE"
+	case history.Tagmsg:
+		return "TAGMSG"
+	default:
+		return "PRIVMSG"
+	}
+}
+
 func (channel *Channel) SendSplitMessage(command string, minPrefixMode modes.Mode, clientOnlyTags map[string]string, client *Client, message utils.SplitMessage, rb *ResponseBuffer) {
-	histType, err := msgCommandToHistType(channel.server, command)
+	histType, err := msgCommandToHistType(command)
 	if err != nil {
 		return
 	}

irc/client.go

@@ -107,6 +107,8 @@ type Session struct {
 	fakelag   Fakelag
 	destroyed uint32
 
+	batchCounter uint32
+
 	quitMessage string
 
 	capabilities caps.Set
@@ -119,6 +121,18 @@ type Session struct {
 	resumeID         string
 	resumeDetails    *ResumeDetails
 	zncPlaybackTimes *zncPlaybackTimes
+
+	batch MultilineBatch
+}
+
+// MultilineBatch tracks the state of a client-to-server multiline batch.
+type MultilineBatch struct {
+	label         string // this is the first param to BATCH (the "reference tag")
+	command       string
+	target        string
+	responseLabel string // this is the value of the labeled-response tag sent with BATCH
+	message       utils.SplitMessage
+	tags          map[string]string
 }
 
 // sets the session quit message, if there isn't one already
@@ -170,6 +184,15 @@ func (session *Session) HasHistoryCaps() bool {
 	return session.capabilities.Has(caps.ZNCPlayback)
 }
 
+// generates a batch ID. the uniqueness requirements for this are fairly weak:
+// any two batch IDs that are active concurrently (either through interleaving
+// or nesting) on an individual session connection need to be unique.
+// this allows ~4 billion such batches which should be fine.
+func (session *Session) generateBatchID() string {
+	id := atomic.AddUint32(&session.batchCounter, 1)
+	return strconv.Itoa(int(id))
+}
+
 // WhoWas is the subset of client details needed to answer a WHOWAS query
 type WhoWas struct {
 	nick           string
@@ -530,6 +553,19 @@ func (client *Client) run(session *Session, proxyLine string) {
 			break
 		}
 
+		// "Clients MUST NOT send messages other than PRIVMSG while a multiline batch is open."
+		// in future we might want to whitelist some commands that are allowed here, like PONG
+		if session.batch.label != "" && msg.Command != "BATCH" {
+			_, batchTag := msg.GetTag("batch")
+			if batchTag != session.batch.label {
+				if msg.Command != "NOTICE" {
+					session.Send(nil, client.server.name, "FAIL", "BATCH", "MULTILINE_INVALID", client.t("Incorrect batch tag sent"))
+				}
+				session.batch = MultilineBatch{}
+				continue
+			}
+		}
+
 		cmd, exists := Commands[msg.Command]
 		if !exists {
 			if len(msg.Command) > 0 {
@@ -1186,14 +1222,20 @@ func (client *Client) destroy(session *Session) {
 // SendSplitMsgFromClient sends an IRC PRIVMSG/NOTICE coming from a specific client.
 // Adds account-tag to the line as well.
 func (session *Session) sendSplitMsgFromClientInternal(blocking bool, nickmask, accountName string, tags map[string]string, command, target string, message utils.SplitMessage) {
-	if session.capabilities.Has(caps.MaxLine) || message.Wrapped == nil {
+	if message.Is512() || session.capabilities.Has(caps.MaxLine) {
 		session.sendFromClientInternal(blocking, message.Time, message.Msgid, nickmask, accountName, tags, command, target, message.Message)
+	} else {
+		if message.IsMultiline() && session.capabilities.Has(caps.Multiline) {
+			for _, msg := range session.composeMultilineBatch(nickmask, accountName, tags, command, target, message) {
+				session.SendRawMessage(msg, blocking)
+			}
 		} else {
 			for _, messagePair := range message.Wrapped {
 				session.sendFromClientInternal(blocking, message.Time, messagePair.Msgid, nickmask, accountName, tags, command, target, messagePair.Message)
 			}
 		}
 	}
+}
 
 // Sends a line with `nickmask` as the prefix, adding `time` and `account` tags if supported
 func (client *Client) sendFromClientInternal(blocking bool, serverTime time.Time, msgid string, nickmask, accountName string, tags map[string]string, command string, params ...string) (err error) {
@@ -1222,6 +1264,30 @@ func (session *Session) sendFromClientInternal(blocking bool, serverTime time.Ti
 	return session.SendRawMessage(msg, blocking)
 }
 
+func (session *Session) composeMultilineBatch(fromNickMask, fromAccount string, tags map[string]string, command, target string, message utils.SplitMessage) (result []ircmsg.IrcMessage) {
+	batchID := session.generateBatchID()
+	batchStart := ircmsg.MakeMessage(tags, fromNickMask, "BATCH", "+"+batchID, caps.MultilineBatchType)
+	batchStart.SetTag("time", message.Time.Format(IRCv3TimestampFormat))
+	batchStart.SetTag("msgid", message.Msgid)
+	if session.capabilities.Has(caps.AccountTag) && fromAccount != "*" {
+		batchStart.SetTag("account", fromAccount)
+	}
+	result = append(result, batchStart)
+
+	for _, msg := range message.Wrapped {
+		message := ircmsg.MakeMessage(nil, fromNickMask, command, target, msg.Message)
+		message.SetTag("batch", batchID)
+		message.SetTag(caps.MultilineFmsgidTag, msg.Msgid)
+		if msg.Concat {
+			message.SetTag(caps.MultilineConcatTag, "")
+		}
+		result = append(result, message)
+	}
+
+	result = append(result, ircmsg.MakeMessage(nil, fromNickMask, "BATCH", "-"+batchID))
+	return
+}
+
 var (
 	// these are all the output commands that MUST have their last param be a trailing.
 	// this is needed because dumb clients like to treat trailing params separately from the

irc/client_test.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2019 Shivaram Lingamneni
+// released under the MIT license
+
+package irc
+
+import (
+	"testing"
+)
+
+func TestGenerateBatchID(t *testing.T) {
+	var session Session
+	s := make(StringSet)
+
+	count := 100000
+	for i := 0; i < count; i++ {
+		s.Add(session.generateBatchID())
+	}
+
+	if len(s) != count {
+		t.Error("duplicate batch ID detected")
+	}
+}
+
+func BenchmarkGenerateBatchID(b *testing.B) {
+	var session Session
+	for i := 0; i < b.N; i++ {
+		session.generateBatchID()
+	}
+}

irc/commands.go

@@ -16,6 +16,7 @@ type Command struct {
 	oper            bool
 	usablePreReg    bool
 	leaveClientIdle bool // if true, leaves the client active time alone
+	allowedInBatch  bool // allowed in client-to-server batches
 	minParams       int
 	capabs          []string
 }
@@ -44,6 +45,11 @@ func (cmd *Command) Run(server *Server, client *Client, session *Session, msg ir
 			rb.Add(nil, server.name, ERR_NEEDMOREPARAMS, client.Nick(), msg.Command, rb.target.t("Not enough parameters"))
 			return false
 		}
+		if session.batch.label != "" && !cmd.allowedInBatch {
+			rb.Add(nil, server.name, "FAIL", "BATCH", "MULTILINE_INVALID", client.t("Command not allowed during a multiline batch"))
+			session.batch = MultilineBatch{}
+			return false
+		}
 
 		return cmd.handler(server, client, msg, rb)
 	}()
@@ -92,6 +98,11 @@ func init() {
 			handler:   awayHandler,
 			minParams: 0,
 		},
+		"BATCH": {
+			handler:        batchHandler,
+			minParams:      1,
+			allowedInBatch: true,
+		},
 		"BRB": {
 			handler:   brbHandler,
 			minParams: 0,
@@ -195,6 +206,7 @@ func init() {
 		"NOTICE": {
 			handler:        messageHandler,
 			minParams:      2,
+			allowedInBatch: true,
 		},
 		"NPC": {
 			handler:   npcHandler,
@@ -232,6 +244,7 @@ func init() {
 		"PRIVMSG": {
 			handler:        messageHandler,
 			minParams:      2,
+			allowedInBatch: true,
 		},
 		"RENAME": {
 			handler:   renameHandler,

irc/config.go

@@ -234,6 +234,10 @@ type Limits struct {
 	TopicLen             int           `yaml:"topiclen"`
 	WhowasEntries        int           `yaml:"whowas-entries"`
 	RegistrationMessages int           `yaml:"registration-messages"`
+	Multiline            struct {
+		MaxBytes int `yaml:"max-bytes"`
+		MaxLines int `yaml:"max-lines"`
+	}
 }
 
 // STSConfig controls the STS configuration/
@@ -683,6 +687,18 @@ func LoadConfig(filename string) (config *Config, err error) {
 		config.Server.capValues[caps.MaxLine] = strconv.Itoa(config.Limits.LineLen.Rest)
 	}
 
+	if config.Limits.Multiline.MaxBytes <= 0 {
+		config.Server.supportedCaps.Disable(caps.Multiline)
+	} else {
+		var multilineCapValue string
+		if config.Limits.Multiline.MaxLines == 0 {
+			multilineCapValue = fmt.Sprintf("max-bytes=%d", config.Limits.Multiline.MaxBytes)
+		} else {
+			multilineCapValue = fmt.Sprintf("max-bytes=%d,max-lines=%d", config.Limits.Multiline.MaxBytes, config.Limits.Multiline.MaxLines)
+		}
+		config.Server.capValues[caps.Multiline] = multilineCapValue
+	}
+
 	if !config.Accounts.Bouncer.Enabled {
 		config.Server.supportedCaps.Disable(caps.Bouncer)
 	}
@@ -869,3 +885,47 @@ func LoadConfig(filename string) (config *Config, err error) {
 
 	return config, nil
 }
+
+// Diff returns changes in supported caps across a rehash.
+func (config *Config) Diff(oldConfig *Config) (addedCaps, removedCaps *caps.Set) {
+	addedCaps = caps.NewSet()
+	removedCaps = caps.NewSet()
+	if oldConfig == nil {
+		return
+	}
+
+	if oldConfig.Server.capValues[caps.Languages] != config.Server.capValues[caps.Languages] {
+		// XXX updated caps get a DEL line and then a NEW line with the new value
+		addedCaps.Add(caps.Languages)
+		removedCaps.Add(caps.Languages)
+	}
+
+	if !oldConfig.Accounts.AuthenticationEnabled && config.Accounts.AuthenticationEnabled {
+		addedCaps.Add(caps.SASL)
+	} else if oldConfig.Accounts.AuthenticationEnabled && !config.Accounts.AuthenticationEnabled {
+		removedCaps.Add(caps.SASL)
+	}
+
+	if !oldConfig.Accounts.Bouncer.Enabled && config.Accounts.Bouncer.Enabled {
+		addedCaps.Add(caps.Bouncer)
+	} else if oldConfig.Accounts.Bouncer.Enabled && !config.Accounts.Bouncer.Enabled {
+		removedCaps.Add(caps.Bouncer)
+	}
+
+	if oldConfig.Limits.Multiline.MaxBytes != 0 && config.Limits.Multiline.MaxBytes == 0 {
+		removedCaps.Add(caps.Multiline)
+	} else if oldConfig.Limits.Multiline.MaxBytes == 0 && config.Limits.Multiline.MaxBytes != 0 {
+		addedCaps.Add(caps.Multiline)
+	} else if oldConfig.Limits.Multiline != config.Limits.Multiline {
+		removedCaps.Add(caps.Multiline)
+		addedCaps.Add(caps.Multiline)
+	}
+
+	if oldConfig.Server.STS.Enabled != config.Server.STS.Enabled || oldConfig.Server.capValues[caps.STS] != config.Server.capValues[caps.STS] {
+		// XXX: STS is always removed by CAP NEW sts=duration=0, not CAP DEL
+		// so the appropriate notify is always a CAP NEW; put it in addedCaps for any change
+		addedCaps.Add(caps.STS)
+	}
+
+	return
+}

irc/handlers.go

@@ -509,6 +509,59 @@ func awayHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Resp
 	return false
 }
 
+// BATCH {+,-}reference-tag type [params...]
+func batchHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool {
+	tag := msg.Params[0]
+	fail := false
+	sendErrors := rb.session.batch.command != "NOTICE"
+	if len(tag) == 0 {
+		fail = true
+	} else if tag[0] == '+' {
+		if rb.session.batch.label != "" || msg.Params[1] != caps.MultilineBatchType {
+			fail = true
+		} else {
+			rb.session.batch.label = tag[1:]
+			rb.session.batch.tags = msg.ClientOnlyTags()
+			if len(msg.Params) == 2 {
+				fail = true
+			} else {
+				rb.session.batch.target = msg.Params[2]
+				// save the response label for later
+				// XXX changing the label inside a handler is a bit dodgy, but it works here
+				// because there's no way we could have triggered a flush up to this point
+				rb.session.batch.responseLabel = rb.Label
+				rb.Label = ""
+			}
+		}
+	} else if tag[0] == '-' {
+		if rb.session.batch.label == "" || rb.session.batch.label != tag[1:] {
+			fail = true
+		} else if rb.session.batch.message.LenLines() == 0 {
+			fail = true
+		} else {
+			batch := rb.session.batch
+			rb.session.batch = MultilineBatch{}
+			batch.message.Time = time.Now().UTC()
+			histType, err := msgCommandToHistType(batch.command)
+			if err != nil {
+				histType = history.Privmsg
+			}
+			// see previous caution about modifying ResponseBuffer.Label
+			rb.Label = batch.responseLabel
+			dispatchMessageToTarget(client, batch.tags, histType, batch.target, batch.message, rb)
+		}
+	}
+
+	if fail {
+		rb.session.batch = MultilineBatch{}
+		if sendErrors {
+			rb.Add(nil, server.name, "FAIL", "BATCH", "MULTILINE_INVALID", client.t("Invalid multiline batch"))
+		}
+	}
+
+	return false
+}
+
 // BRB [message]
 func brbHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool {
 	success, duration := client.brbTimer.Enable()
@@ -665,11 +718,6 @@ func chathistoryHandler(server *Server, client *Client, msg ircmsg.IrcMessage, r
 	defer func() {
 		// successful responses are sent as a chathistory or history batch
 		if success && 0 < len(items) {
-			batchType := "chathistory"
-			if rb.session.capabilities.Has(caps.EventPlayback) {
-				batchType = "history"
-			}
-			rb.ForceBatchStart(batchType, true)
 			if channel == nil {
 				client.replayPrivmsgHistory(rb, items, true)
 			} else {
@@ -2019,15 +2067,44 @@ func nickHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Resp
 	return false
 }
 
+// helper to store a batched PRIVMSG in the session object
+func absorbBatchedMessage(server *Server, client *Client, msg ircmsg.IrcMessage, batchTag string, histType history.ItemType, rb *ResponseBuffer) {
+	// sanity checks. batch tag correctness was already checked and is redundant here
+	// as a defensive measure. TAGMSG is checked without an error message: "don't eat paste"
+	if batchTag != rb.session.batch.label || histType == history.Tagmsg || len(msg.Params) == 1 || msg.Params[1] == "" {
+		return
+	}
+	rb.session.batch.command = msg.Command
+	isConcat, _ := msg.GetTag(caps.MultilineConcatTag)
+	rb.session.batch.message.Append(msg.Params[1], isConcat)
+	config := server.Config()
+	if config.Limits.Multiline.MaxBytes < rb.session.batch.message.LenBytes() {
+		if histType != history.Notice {
+			rb.Add(nil, server.name, "FAIL", "BATCH", "MULTILINE_MAX_BYTES", strconv.Itoa(config.Limits.Multiline.MaxBytes))
+		}
+		rb.session.batch = MultilineBatch{}
+	} else if config.Limits.Multiline.MaxLines != 0 && config.Limits.Multiline.MaxLines < rb.session.batch.message.LenLines() {
+		if histType != history.Notice {
+			rb.Add(nil, server.name, "FAIL", "BATCH", "MULTILINE_MAX_LINES", strconv.Itoa(config.Limits.Multiline.MaxLines))
+		}
+		rb.session.batch = MultilineBatch{}
+	}
+}
+
 // NOTICE <target>{,<target>} <message>
 // PRIVMSG <target>{,<target>} <message>
 // TAGMSG <target>{,<target>}
 func messageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool {
-	histType, err := msgCommandToHistType(server, msg.Command)
+	histType, err := msgCommandToHistType(msg.Command)
 	if err != nil {
 		return false
 	}
 
+	if isBatched, batchTag := msg.GetTag("batch"); isBatched {
+		absorbBatchedMessage(server, client, msg, batchTag, histType, rb)
+		return false
+	}
+
 	cnick := client.Nick()
 	clientOnlyTags := msg.ClientOnlyTags()
 	if histType == history.Tagmsg && len(clientOnlyTags) == 0 {
@@ -2040,57 +2117,68 @@ func messageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *R
 	if len(msg.Params) > 1 {
 		message = msg.Params[1]
 	}
+	if histType != history.Tagmsg && message == "" {
+		rb.Add(nil, server.name, ERR_NOTEXTTOSEND, cnick, client.t("No text to send"))
+		return false
+	}
 
+	if client.isTor && utils.IsRestrictedCTCPMessage(message) {
 		// note that error replies are never sent for NOTICE
-
-	if client.isTor && isRestrictedCTCPMessage(message) {
 		if histType != history.Notice {
-			rb.Add(nil, server.name, "NOTICE", client.t("CTCP messages are disabled over Tor"))
+			rb.Notice(client.t("CTCP messages are disabled over Tor"))
 		}
 		return false
 	}
 
 	for i, targetString := range targets {
-		// each target gets distinct msgids
-		splitMsg := utils.MakeSplitMessage(message, !rb.session.capabilities.Has(caps.MaxLine))
-
 		// max of four targets per privmsg
-		if i > maxTargets-1 {
+		if i == maxTargets {
 			break
 		}
-		prefixes, targetString := modes.SplitChannelMembershipPrefixes(targetString)
+		// each target gets distinct msgids
+		splitMsg := utils.MakeSplitMessage(message, !rb.session.capabilities.Has(caps.MaxLine))
+		dispatchMessageToTarget(client, clientOnlyTags, histType, targetString, splitMsg, rb)
+	}
+	return false
+}
+
+func dispatchMessageToTarget(client *Client, tags map[string]string, histType history.ItemType, target string, message utils.SplitMessage, rb *ResponseBuffer) {
+	server := client.server
+	command := histTypeToMsgCommand(histType)
+
+	prefixes, target := modes.SplitChannelMembershipPrefixes(target)
 	lowestPrefix := modes.GetLowestChannelModePrefix(prefixes)
 
-		if len(targetString) == 0 {
+	if len(target) == 0 {
-			continue
+		return
-		} else if targetString[0] == '#' {
+	} else if target[0] == '#' {
-			channel := server.channels.Get(targetString)
+		channel := server.channels.Get(target)
 		if channel == nil {
 			if histType != history.Notice {
-					rb.Add(nil, server.name, ERR_NOSUCHCHANNEL, cnick, utils.SafeErrorParam(targetString), client.t("No such channel"))
+				rb.Add(nil, server.name, ERR_NOSUCHCHANNEL, client.Nick(), utils.SafeErrorParam(target), client.t("No such channel"))
 			}
-				continue
+			return
 		}
-			channel.SendSplitMessage(msg.Command, lowestPrefix, clientOnlyTags, client, splitMsg, rb)
+		channel.SendSplitMessage(command, lowestPrefix, tags, client, message, rb)
 	} else {
 		// NOTICE and TAGMSG to services are ignored
 		if histType == history.Privmsg {
-				lowercaseTarget := strings.ToLower(targetString)
+			lowercaseTarget := strings.ToLower(target)
 			if service, isService := OragonoServices[lowercaseTarget]; isService {
-					servicePrivmsgHandler(service, server, client, message, rb)
+				servicePrivmsgHandler(service, server, client, message.Message, rb)
-					continue
+				return
 			} else if _, isZNC := zncHandlers[lowercaseTarget]; isZNC {
-					zncPrivmsgHandler(client, lowercaseTarget, message, rb)
+				zncPrivmsgHandler(client, lowercaseTarget, message.Message, rb)
-					continue
+				return
 			}
 		}
 
-			user := server.clients.Get(targetString)
+		user := server.clients.Get(target)
 		if user == nil {
 			if histType != history.Notice {
-					rb.Add(nil, server.name, ERR_NOSUCHNICK, cnick, targetString, "No such nick")
+				rb.Add(nil, server.name, ERR_NOSUCHNICK, client.Nick(), target, "No such nick")
 			}
-				continue
+			return
 		}
 		tnick := user.Nick()
 
@@ -2099,25 +2187,25 @@ func messageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *R
 		// restrict messages appropriately when +R is set
 		// intentionally make the sending user think the message went through fine
 		allowedPlusR := !user.HasMode(modes.RegisteredOnly) || client.LoggedIntoAccount()
-			allowedTor := !user.isTor || !isRestrictedCTCPMessage(message)
+		allowedTor := !user.isTor || !message.IsRestrictedCTCPMessage()
 		if allowedPlusR && allowedTor {
 			for _, session := range user.Sessions() {
 				if histType == history.Tagmsg {
 					// don't send TAGMSG at all if they don't have the tags cap
 					if session.capabilities.Has(caps.MessageTags) {
-							session.sendFromClientInternal(false, splitMsg.Time, splitMsg.Msgid, nickMaskString, accountName, clientOnlyTags, msg.Command, tnick)
+						session.sendFromClientInternal(false, message.Time, message.Msgid, nickMaskString, accountName, tags, command, tnick)
 					}
 				} else {
-						session.sendSplitMsgFromClientInternal(false, nickMaskString, accountName, clientOnlyTags, msg.Command, tnick, splitMsg)
+					session.sendSplitMsgFromClientInternal(false, nickMaskString, accountName, tags, command, tnick, message)
 				}
 			}
 		}
 		// an echo-message may need to be included in the response:
 		if rb.session.capabilities.Has(caps.EchoMessage) {
 			if histType == history.Tagmsg && rb.session.capabilities.Has(caps.MessageTags) {
-					rb.AddFromClient(splitMsg.Time, splitMsg.Msgid, nickMaskString, accountName, clientOnlyTags, msg.Command, tnick)
+				rb.AddFromClient(message.Time, message.Msgid, nickMaskString, accountName, tags, command, tnick)
 			} else {
-					rb.AddSplitMessageFromClient(nickMaskString, accountName, clientOnlyTags, msg.Command, tnick, splitMsg)
+				rb.AddSplitMessageFromClient(nickMaskString, accountName, tags, command, tnick, message)
 			}
 		}
 		// an echo-message may need to go out to other client sessions:
@@ -2126,19 +2214,19 @@ func messageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *R
 				continue
 			}
 			if histType == history.Tagmsg && rb.session.capabilities.Has(caps.MessageTags) {
-					session.sendFromClientInternal(false, splitMsg.Time, splitMsg.Msgid, nickMaskString, accountName, clientOnlyTags, msg.Command, tnick)
+				session.sendFromClientInternal(false, message.Time, message.Msgid, nickMaskString, accountName, tags, command, tnick)
 			} else if histType != history.Tagmsg {
-					session.sendSplitMsgFromClientInternal(false, nickMaskString, accountName, clientOnlyTags, msg.Command, tnick, splitMsg)
+				session.sendSplitMsgFromClientInternal(false, nickMaskString, accountName, tags, command, tnick, message)
 			}
 		}
 		if histType != history.Notice && user.Away() {
 			//TODO(dan): possibly implement cooldown of away notifications to users
-				rb.Add(nil, server.name, RPL_AWAY, cnick, tnick, user.AwayMessage())
+			rb.Add(nil, server.name, RPL_AWAY, client.Nick(), tnick, user.AwayMessage())
 		}
 
 		item := history.Item{
 			Type:        histType,
-				Message:     splitMsg,
+			Message:     message,
 			Nick:        nickMaskString,
 			AccountName: accountName,
 		}
@@ -2149,8 +2237,6 @@ func messageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *R
 		client.history.Add(item)
 	}
 }
-	return false
-}
 
 // NPC <target> <sourcenick> <message>
 func npcHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool {
@@ -2308,12 +2394,6 @@ func pongHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Resp
 	return false
 }
 
-func isRestrictedCTCPMessage(message string) bool {
-	// block all CTCP privmsgs to Tor clients except for ACTION
-	// DCC can potentially be used for deanonymization, the others for fingerprinting
-	return strings.HasPrefix(message, "\x01") && !strings.HasPrefix(message, "\x01ACTION")
-}
-
 // QUIT [<reason>]
 func quitHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool {
 	reason := "Quit"

irc/help.go

@@ -120,6 +120,12 @@ http://ircv3.net/specs/extensions/sasl-3.1.html`,
 
 If [message] is sent, marks you away. If [message] is not sent, marks you no
 longer away.`,
+	},
+	"batch": {
+		text: `BATCH {+,-}reference-tag type [params...]
+
+BATCH initiates an IRCv3 client-to-server batch. You should never need to
+issue this command manually.`,
 	},
 	"brb": {
 		text: `BRB [message]

irc/responsebuffer.go

@@ -66,10 +66,16 @@ func (rb *ResponseBuffer) AddMessage(msg ircmsg.IrcMessage) {
 		return
 	}
 
+	rb.session.setTimeTag(&msg, time.Time{})
+	rb.setNestedBatchTag(&msg)
+
+	rb.messages = append(rb.messages, msg)
+}
+
+func (rb *ResponseBuffer) setNestedBatchTag(msg *ircmsg.IrcMessage) {
 	if 0 < len(rb.nestedBatches) {
 		msg.SetTag("batch", rb.nestedBatches[len(rb.nestedBatches)-1])
 	}
-	rb.messages = append(rb.messages, msg)
 }
 
 // Add adds a standard new message to our queue.
@@ -112,22 +118,20 @@ func (rb *ResponseBuffer) AddFromClient(time time.Time, msgid string, fromNickMa
 
 // AddSplitMessageFromClient adds a new split message from a specific client to our queue.
 func (rb *ResponseBuffer) AddSplitMessageFromClient(fromNickMask string, fromAccount string, tags map[string]string, command string, target string, message utils.SplitMessage) {
-	if rb.session.capabilities.Has(caps.MaxLine) || message.Wrapped == nil {
+	if message.Is512() || rb.session.capabilities.Has(caps.MaxLine) {
 		rb.AddFromClient(message.Time, message.Msgid, fromNickMask, fromAccount, tags, command, target, message.Message)
+	} else {
+		if message.IsMultiline() && rb.session.capabilities.Has(caps.Multiline) {
+			batch := rb.session.composeMultilineBatch(fromNickMask, fromAccount, tags, command, target, message)
+			rb.setNestedBatchTag(&batch[0])
+			rb.setNestedBatchTag(&batch[len(batch)-1])
+			rb.messages = append(rb.messages, batch...)
 		} else {
 			for _, messagePair := range message.Wrapped {
 				rb.AddFromClient(message.Time, messagePair.Msgid, fromNickMask, fromAccount, tags, command, target, messagePair.Message)
 			}
 		}
 	}
-
-// ForceBatchStart forcibly starts a batch of batch `batchType`.
-// Normally, Send/Flush will decide automatically whether to start a batch
-// of type draft/labeled-response. This allows changing the batch type
-// and forcing the creation of a possibly empty batch.
-func (rb *ResponseBuffer) ForceBatchStart(batchType string, blocking bool) {
-	rb.batchType = batchType
-	rb.sendBatchStart(blocking)
 }
 
 func (rb *ResponseBuffer) sendBatchStart(blocking bool) {
@@ -136,7 +140,7 @@ func (rb *ResponseBuffer) sendBatchStart(blocking bool) {
 		return
 	}
 
-	rb.batchID = utils.GenerateSecretToken()
+	rb.batchID = rb.session.generateBatchID()
 	message := ircmsg.MakeMessage(nil, rb.target.server.name, "BATCH", "+"+rb.batchID, rb.batchType)
 	if rb.Label != "" {
 		message.SetTag(caps.LabelTagName, rb.Label)
@@ -157,7 +161,7 @@ func (rb *ResponseBuffer) sendBatchEnd(blocking bool) {
 // Starts a nested batch (see the ResponseBuffer struct definition for a description of
 // how this works)
 func (rb *ResponseBuffer) StartNestedBatch(batchType string, params ...string) (batchID string) {
-	batchID = utils.GenerateSecretToken()
+	batchID = rb.session.generateBatchID()
 	msgParams := make([]string, len(params)+2)
 	msgParams[0] = "+" + batchID
 	msgParams[1] = batchType
@@ -213,6 +217,23 @@ func (rb *ResponseBuffer) Flush(blocking bool) error {
 	return rb.flushInternal(false, blocking)
 }
 
+// detects whether the response buffer consists of a single, unflushed nested batch,
+// in which case it can be collapsed down to that batch
+func (rb *ResponseBuffer) isCollapsible() (result bool) {
+	// rb.batchID indicates that we already flushed some lines
+	if rb.batchID != "" || len(rb.messages) < 2 {
+		return false
+	}
+	first, last := rb.messages[0], rb.messages[len(rb.messages)-1]
+	if first.Command != "BATCH" || last.Command != "BATCH" {
+		return false
+	}
+	if len(first.Params) == 0 || len(first.Params[0]) == 0 || len(last.Params) == 0 || len(last.Params[0]) == 0 {
+		return false
+	}
+	return first.Params[0][1:] == last.Params[0][1:]
+}
+
 // flushInternal sends the contents of the buffer, either blocking or nonblocking
 // It sends the `BATCH +` message if the client supports it and it hasn't been sent already.
 // If `final` is true, it also sends `BATCH -` (if necessary).
@@ -221,30 +242,28 @@ func (rb *ResponseBuffer) flushInternal(final bool, blocking bool) error {
 		return nil
 	}
 
-	useLabel := rb.session.capabilities.Has(caps.LabeledResponse) && rb.Label != ""
+	if rb.session.capabilities.Has(caps.LabeledResponse) && rb.Label != "" {
-	// use a batch if we have a label, and we either currently have 2+ messages,
+		if final && rb.isCollapsible() {
-	// or we are doing a Flush() and we have to assume that there will be more messages
+			// collapse to the outermost nested batch
-	// in the future.
+			rb.messages[0].SetTag(caps.LabelTagName, rb.Label)
-	startBatch := useLabel && (1 < len(rb.messages) || !final)
+		} else if !final || 2 <= len(rb.messages) {
-
+			// we either have 2+ messages, or we are doing a Flush() and have to assume
-	if startBatch {
+			// there will be more messages in the future
 			rb.sendBatchStart(blocking)
-	} else if useLabel && len(rb.messages) == 0 && rb.batchID == "" && final {
+		} else if len(rb.messages) == 1 && rb.batchID == "" {
+			// single labeled message
+			rb.messages[0].SetTag(caps.LabelTagName, rb.Label)
+		} else if len(rb.messages) == 0 && rb.batchID == "" {
 			// ACK message
 			message := ircmsg.MakeMessage(nil, rb.session.client.server.name, "ACK")
 			message.SetTag(caps.LabelTagName, rb.Label)
 			rb.session.setTimeTag(&message, time.Time{})
 			rb.session.SendRawMessage(message, blocking)
-	} else if useLabel && len(rb.messages) == 1 && rb.batchID == "" && final {
+		}
-		// single labeled message
-		rb.messages[0].SetTag(caps.LabelTagName, rb.Label)
 	}
 
 	// send each message out
 	for _, message := range rb.messages {
-		// attach server-time if needed
-		rb.session.setTimeTag(&message, time.Time{})
-
 		// attach batch ID, unless this message was part of a nested batch and is
 		// already tagged
 		if rb.batchID != "" && !message.HasTag("batch") {

irc/server.go

@@ -629,39 +629,11 @@ func (server *Server) applyConfig(config *Config, initial bool) (err error) {
 	tlConf := &config.Server.TorListeners
 	server.torLimiter.Configure(tlConf.MaxConnections, tlConf.ThrottleDuration, tlConf.MaxConnectionsPerDuration)
 
-	// setup new and removed caps
-	addedCaps := caps.NewSet()
-	removedCaps := caps.NewSet()
-	updatedCaps := caps.NewSet()
-
 	// Translations
 	server.logger.Debug("server", "Regenerating HELP indexes for new languages")
 	server.helpIndexManager.GenerateIndices(config.languageManager)
 
 	if oldConfig != nil {
-		// cap changes
-		if oldConfig.Server.capValues[caps.Languages] != config.Server.capValues[caps.Languages] {
-			updatedCaps.Add(caps.Languages)
-		}
-
-		if !oldConfig.Accounts.AuthenticationEnabled && config.Accounts.AuthenticationEnabled {
-			addedCaps.Add(caps.SASL)
-		} else if oldConfig.Accounts.AuthenticationEnabled && !config.Accounts.AuthenticationEnabled {
-			removedCaps.Add(caps.SASL)
-		}
-
-		if !oldConfig.Accounts.Bouncer.Enabled && config.Accounts.Bouncer.Enabled {
-			addedCaps.Add(caps.Bouncer)
-		} else if oldConfig.Accounts.Bouncer.Enabled && !config.Accounts.Bouncer.Enabled {
-			removedCaps.Add(caps.Bouncer)
-		}
-
-		if oldConfig.Server.STS.Enabled != config.Server.STS.Enabled || oldConfig.Server.capValues[caps.STS] != config.Server.capValues[caps.STS] {
-			// XXX: STS is always removed by CAP NEW sts=duration=0, not CAP DEL
-			// so the appropriate notify is always a CAP NEW; put it in addedCaps for any change
-			addedCaps.Add(caps.STS)
-		}
-
 		// if certain features were enabled by rehash, we need to load the corresponding data
 		// from the store
 		if !oldConfig.Accounts.NickReservation.Enabled {
@@ -689,16 +661,11 @@ func (server *Server) applyConfig(config *Config, initial bool) (err error) {
 	server.SetConfig(config)
 
 	// burst new and removed caps
+	addedCaps, removedCaps := config.Diff(oldConfig)
 	var capBurstSessions []*Session
 	added := make(map[caps.Version][]string)
 	var removed []string
 
-	// updated caps get DEL'd and then NEW'd
-	// so, we can just add updated ones to both removed and added lists here and they'll be correctly handled
-	server.logger.Debug("server", "Updated Caps", strings.Join(updatedCaps.Strings(caps.Cap301, config.Server.capValues, 0), " "))
-	addedCaps.Union(updatedCaps)
-	removedCaps.Union(updatedCaps)
-
 	if !addedCaps.Empty() || !removedCaps.Empty() {
 		capBurstSessions = server.clients.AllWithCapsNotify()
 

irc/utils/text.go

@@ -3,8 +3,17 @@
 
 package utils
 
-import "bytes"
+import (
-import "time"
+	"bytes"
+	"strings"
+	"time"
+)
+
+func IsRestrictedCTCPMessage(message string) bool {
+	// block all CTCP privmsgs to Tor clients except for ACTION
+	// DCC can potentially be used for deanonymization, the others for fingerprinting
+	return strings.HasPrefix(message, "\x01") && !strings.HasPrefix(message, "\x01ACTION")
+}
 
 // WordWrap wraps the given text into a series of lines that don't exceed lineWidth characters.
 func WordWrap(text string, lineWidth int) []string {
@@ -54,9 +63,17 @@ func WordWrap(text string, lineWidth int) []string {
 type MessagePair struct {
 	Message string
 	Msgid   string
+	Concat  bool // should be relayed with the multiline-concat tag
 }
 
 // SplitMessage represents a message that's been split for sending.
+// Three possibilities:
+// (a) Standard message that can be relayed on a single 512-byte line
+//     (MessagePair contains the message, Wrapped == nil)
+// (b) oragono.io/maxline-2 message that was split on the server side
+//     (MessagePair contains the unsplit message, Wrapped contains the split lines)
+// (c) multiline message that was split on the client side
+//     (MessagePair is zero, Wrapped contains the split lines)
 type SplitMessage struct {
 	MessagePair
 	Wrapped []MessagePair // if this is nil, `Message` didn't need wrapping and can be sent to anyone
@@ -84,6 +101,58 @@ func MakeSplitMessage(original string, origIs512 bool) (result SplitMessage) {
 	return
 }
 
+func (sm *SplitMessage) Append(message string, concat bool) {
+	if sm.Msgid == "" {
+		sm.Msgid = GenerateSecretToken()
+	}
+	sm.Wrapped = append(sm.Wrapped, MessagePair{
+		Message: message,
+		Msgid:   GenerateSecretToken(),
+		Concat:  concat,
+	})
+}
+
+func (sm *SplitMessage) LenLines() int {
+	if sm.Wrapped == nil {
+		if (sm.MessagePair == MessagePair{}) {
+			return 0
+		} else {
+			return 1
+		}
+	}
+	return len(sm.Wrapped)
+}
+
+func (sm *SplitMessage) LenBytes() (result int) {
+	if sm.Wrapped == nil {
+		return len(sm.Message)
+	}
+	for i := 0; i < len(sm.Wrapped); i++ {
+		result += len(sm.Wrapped[i].Message)
+	}
+	return
+}
+
+func (sm *SplitMessage) IsRestrictedCTCPMessage() bool {
+	if IsRestrictedCTCPMessage(sm.Message) {
+		return true
+	}
+	for i := 0; i < len(sm.Wrapped); i++ {
+		if IsRestrictedCTCPMessage(sm.Wrapped[i].Message) {
+			return true
+		}
+	}
+	return false
+}
+
+func (sm *SplitMessage) IsMultiline() bool {
+	return sm.Message == "" && len(sm.Wrapped) != 0
+}
+
+func (sm *SplitMessage) Is512() bool {
+	return sm.Message != "" && sm.Wrapped == nil
+}
+
 // TokenLineBuilder is a helper for building IRC lines composed of delimited tokens,
 // with a maximum line length.
 type TokenLineBuilder struct {

oragono.yaml

@@ -579,6 +579,11 @@ limits:
     # DoS / resource exhaustion attacks):
     registration-messages: 1024
 
+    # message length limits for the new multiline cap
+    multiline:
+        max-bytes: 4096 # 0 means disabled
+        max-lines: 24   # 0 means no limit
+
 # fakelag: prevents clients from spamming commands too rapidly
 fakelag:
     # whether to enforce fakelag