From 33dac4c0babf36c41816d3e0f7cd7637ad7dba1b Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Tue, 18 Feb 2020 19:38:42 -0500 Subject: [PATCH] initial persistent history implementation --- Makefile | 1 + gencapdefs.py | 6 + go.mod | 12 +- irc/accounts.go | 90 ++++-- irc/caps/defs.go | 7 +- irc/channel.go | 154 ++++++++--- irc/channelreg.go | 15 + irc/chanserv.go | 141 ++++++++-- irc/client.go | 318 +++++++++++++++++---- irc/client_lookup_set.go | 70 +++-- irc/commands.go | 13 +- irc/config.go | 188 ++++++++++++- irc/database.go | 20 +- irc/errors.go | 2 + irc/gateways.go | 2 +- irc/getters.go | 89 ++++-- irc/handlers.go | 389 +++++++++++--------------- irc/help.go | 5 +- irc/history/history.go | 168 ++++++----- irc/history/history_test.go | 71 +++-- irc/history/queries.go | 71 +++++ irc/idletimer.go | 44 ++- irc/mysql/history.go | 535 ++++++++++++++++++++++++++++++++++++ irc/mysql/serialization.go | 23 ++ irc/nickname.go | 18 +- irc/nickserv.go | 77 +++++- irc/responsebuffer.go | 44 ++- irc/server.go | 105 ++++++- irc/stats.go | 20 +- irc/utils/args.go | 23 +- irc/utils/net.go | 8 - irc/utils/text.go | 13 +- irc/znc.go | 18 +- oragono.yaml | 64 ++++- 34 files changed, 2229 insertions(+), 595 deletions(-) create mode 100644 irc/history/queries.go create mode 100644 irc/mysql/history.go create mode 100644 irc/mysql/serialization.go diff --git a/Makefile b/Makefile index d06dd07f..6888b05c 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ test: cd irc/history && go test . && go vet . cd irc/isupport && go test . && go vet . cd irc/modes && go test . && go vet . + cd irc/mysql && go test . && go vet . cd irc/passwd && go test . && go vet . cd irc/utils && go test . && go vet . ./.check-gofmt.sh diff --git a/gencapdefs.py b/gencapdefs.py index 3f14625d..bb1ffdc3 100644 --- a/gencapdefs.py +++ b/gencapdefs.py @@ -171,6 +171,12 @@ CAPDEFS = [ url="https://github.com/ircv3/ircv3-specifications/pull/398", standard="proposed IRCv3", ), + CapDef( + identifier="Chathistory", + name="draft/chathistory", + url="https://github.com/ircv3/ircv3-specifications/pull/393", + standard="proposed IRCv3", + ), ] def validate_defs(): diff --git a/go.mod b/go.mod index bc4d9b04..6ad6558c 100644 --- a/go.mod +++ b/go.mod @@ -6,23 +6,19 @@ require ( code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 github.com/go-ldap/ldap/v3 v3.1.6 + github.com/go-sql-driver/mysql v1.5.0 github.com/goshuirc/e-nfa v0.0.0-20160917075329-7071788e3940 // indirect github.com/goshuirc/irc-go v0.0.0-20190713001546-05ecc95249a0 github.com/mattn/go-colorable v0.1.4 github.com/mattn/go-isatty v0.0.10 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b + github.com/onsi/ginkgo v1.12.0 // indirect + github.com/onsi/gomega v1.9.0 // indirect github.com/oragono/confusables v0.0.0-20190624102032-fe1cf31a24b0 github.com/oragono/go-ident v0.0.0-20170110123031-337fed0fd21a - github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 // indirect + github.com/stretchr/testify v1.4.0 // indirect github.com/tidwall/buntdb v1.1.2 - github.com/tidwall/gjson v1.3.4 // indirect - github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb // indirect - github.com/tidwall/match v1.0.1 // indirect - github.com/tidwall/pretty v1.0.0 // indirect - github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e // indirect - github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 - golang.org/x/sys v0.0.0-20191115151921-52ab43148777 // indirect golang.org/x/text v0.3.2 gopkg.in/yaml.v2 v2.2.5 ) diff --git a/irc/accounts.go b/irc/accounts.go index 0bf822c7..d70abd7d 100644 --- a/irc/accounts.go +++ b/irc/accounts.go @@ -33,7 +33,8 @@ const ( keyAccountSettings = "account.settings %s" keyAccountVHost = "account.vhost %s" keyCertToAccount = "account.creds.certfp %s" - keyAccountChannels = "account.channels %s" + keyAccountChannels = "account.channels %s" // channels registered to the account + keyAccountJoinedChannels = "account.joinedto %s" // channels a persistent client has joined keyVHostQueueAcctToId = "vhostQueue %s" vhostRequestIdx = "vhostQueue" @@ -71,6 +72,40 @@ func (am *AccountManager) Initialize(server *Server) { config := server.Config() am.buildNickToAccountIndex(config) am.initVHostRequestQueue(config) + am.createAlwaysOnClients(config) +} + +func (am *AccountManager) createAlwaysOnClients(config *Config) { + if config.Accounts.Bouncer.AlwaysOn == PersistentDisabled { + return + } + + verifiedPrefix := fmt.Sprintf(keyAccountVerified, "") + + am.serialCacheUpdateMutex.Lock() + defer am.serialCacheUpdateMutex.Unlock() + + var accounts []string + + am.server.store.View(func(tx *buntdb.Tx) error { + err := tx.AscendGreaterOrEqual("", verifiedPrefix, func(key, value string) bool { + if !strings.HasPrefix(key, verifiedPrefix) { + return false + } + account := strings.TrimPrefix(key, verifiedPrefix) + accounts = append(accounts, account) + return true + }) + return err + }) + + for _, accountName := range accounts { + account, err := am.LoadAccount(accountName) + if err == nil && account.Verified && + persistenceEnabled(config.Accounts.Bouncer.AlwaysOn, account.Settings.AlwaysOn) { + am.server.AddAlwaysOnClient(account, am.loadChannels(accountName)) + } + } } func (am *AccountManager) buildNickToAccountIndex(config *Config) { @@ -477,6 +512,28 @@ func (am *AccountManager) setPassword(account string, password string, hasPrivs return err } +func (am *AccountManager) saveChannels(account string, channels []string) { + channelsStr := strings.Join(channels, ",") + key := fmt.Sprintf(keyAccountJoinedChannels, account) + am.server.store.Update(func(tx *buntdb.Tx) error { + tx.Set(key, channelsStr, nil) + return nil + }) +} + +func (am *AccountManager) loadChannels(account string) (channels []string) { + key := fmt.Sprintf(keyAccountJoinedChannels, account) + var channelsStr string + am.server.store.View(func(tx *buntdb.Tx) error { + channelsStr, _ = tx.Get(key) + return nil + }) + if channelsStr != "" { + return strings.Split(channelsStr, ",") + } + return +} + func (am *AccountManager) addRemoveCertfp(account, certfp string, add bool, hasPrivs bool) (err error) { certfp, err = utils.NormalizeCertfp(certfp) if err != nil { @@ -685,7 +742,7 @@ func (am *AccountManager) Verify(client *Client, account string, code string) er } am.server.logger.Info("accounts", "client", nick, "registered account", casefoldedAccount) raw.Verified = true - clientAccount, err := am.deserializeRawAccount(raw) + clientAccount, err := am.deserializeRawAccount(raw, casefoldedAccount) if err != nil { return err } @@ -892,13 +949,13 @@ func (am *AccountManager) LoadAccount(accountName string) (result ClientAccount, return } - result, err = am.deserializeRawAccount(raw) - result.NameCasefolded = casefoldedAccount + result, err = am.deserializeRawAccount(raw, casefoldedAccount) return } -func (am *AccountManager) deserializeRawAccount(raw rawClientAccount) (result ClientAccount, err error) { +func (am *AccountManager) deserializeRawAccount(raw rawClientAccount, cfName string) (result ClientAccount, err error) { result.Name = raw.Name + result.NameCasefolded = cfName regTimeInt, _ := strconv.ParseInt(raw.RegisteredAt, 10, 64) result.RegisteredAt = time.Unix(regTimeInt, 0).UTC() e := json.Unmarshal([]byte(raw.Credentials), &result.Credentials) @@ -976,6 +1033,7 @@ func (am *AccountManager) Unregister(account string) error { vhostKey := fmt.Sprintf(keyAccountVHost, casefoldedAccount) vhostQueueKey := fmt.Sprintf(keyVHostQueueAcctToId, casefoldedAccount) channelsKey := fmt.Sprintf(keyAccountChannels, casefoldedAccount) + joinedChannelsKey := fmt.Sprintf(keyAccountJoinedChannels, casefoldedAccount) var clients []*Client @@ -1011,6 +1069,7 @@ func (am *AccountManager) Unregister(account string) error { tx.Delete(vhostKey) channelsStr, _ = tx.Get(channelsKey) tx.Delete(channelsKey) + tx.Delete(joinedChannelsKey) _, err := tx.Delete(vhostQueueKey) am.decrementVHostQueueCount(casefoldedAccount, err) @@ -1455,10 +1514,7 @@ func (am *AccountManager) applyVhostToClients(account string, result VHostInfo) } func (am *AccountManager) Login(client *Client, account ClientAccount) { - changed := client.SetAccountName(account.Name) - if !changed { - return - } + client.Login(account) client.nickTimer.Touch(nil) @@ -1468,9 +1524,6 @@ func (am *AccountManager) Login(client *Client, account ClientAccount) { am.Lock() defer am.Unlock() am.accountToClients[casefoldedAccount] = append(am.accountToClients[casefoldedAccount], client) - for _, client := range am.accountToClients[casefoldedAccount] { - client.SetAccountSettings(account.Settings) - } } func (am *AccountManager) Logout(client *Client) { @@ -1623,10 +1676,13 @@ func replayJoinsSettingFromString(str string) (result ReplayJoinsSetting, err er } type AccountSettings struct { - AutoreplayLines *int - NickEnforcement NickEnforcementMethod - AllowBouncer BouncerAllowedSetting - ReplayJoins ReplayJoinsSetting + AutoreplayLines *int + NickEnforcement NickEnforcementMethod + AllowBouncer BouncerAllowedSetting + ReplayJoins ReplayJoinsSetting + AlwaysOn PersistentStatus + AutoreplayMissed bool + DMHistory HistoryStatus } // ClientAccount represents a user account. @@ -1661,7 +1717,7 @@ func (am *AccountManager) logoutOfAccount(client *Client) { return } - client.SetAccountName("") + client.Logout() go client.nickTimer.Touch(nil) // dispatch account-notify diff --git a/irc/caps/defs.go b/irc/caps/defs.go index da2ae597..75898899 100644 --- a/irc/caps/defs.go +++ b/irc/caps/defs.go @@ -7,7 +7,7 @@ package caps const ( // number of recognized capabilities: - numCapabs = 26 + numCapabs = 27 // length of the uint64 array that represents the bitset: bitsetLen = 1 ) @@ -37,6 +37,10 @@ const ( // https://ircv3.net/specs/extensions/chghost-3.2.html ChgHost Capability = iota + // Chathistory is the proposed IRCv3 capability named "draft/chathistory": + // https://github.com/ircv3/ircv3-specifications/pull/393 + Chathistory Capability = iota + // EventPlayback is the proposed IRCv3 capability named "draft/event-playback": // https://github.com/ircv3/ircv3-specifications/pull/362 EventPlayback Capability = iota @@ -127,6 +131,7 @@ var ( "batch", "cap-notify", "chghost", + "draft/chathistory", "draft/event-playback", "draft/languages", "draft/multiline", diff --git a/irc/channel.go b/irc/channel.go index ce3b21ce..c16ff6d6 100644 --- a/irc/channel.go +++ b/irc/channel.go @@ -24,6 +24,10 @@ const ( histServMask = "HistServ!HistServ@localhost" ) +type ChannelSettings struct { + History HistoryStatus +} + // Channel represents a channel that clients can join. type Channel struct { flags modes.ModeSet @@ -49,6 +53,7 @@ type Channel struct { joinPartMutex sync.Mutex // tier 3 ensureLoaded utils.Once // manages loading stored registration info from the database dirtyBits uint + settings ChannelSettings } // NewChannel creates a new channel from a `Server` and a `name` @@ -66,9 +71,10 @@ func NewChannel(s *Server, name, casefoldedName string, registered bool) *Channe channel.initializeLists() channel.writerSemaphore.Initialize(1) - channel.history.Initialize(config.History.ChannelLength, config.History.AutoresizeWindow) + channel.history.Initialize(0, 0) if !registered { + channel.resizeHistory(config) for _, mode := range config.Channels.defaultModes { channel.flags.SetMode(mode, true) } @@ -106,8 +112,19 @@ func (channel *Channel) IsLoaded() bool { return channel.ensureLoaded.Done() } +func (channel *Channel) resizeHistory(config *Config) { + _, ephemeral, _ := channel.historyStatus(config) + if ephemeral { + channel.history.Resize(config.History.ChannelLength, config.History.AutoresizeWindow) + } else { + channel.history.Resize(0, 0) + } +} + // read in channel state that was persisted in the DB func (channel *Channel) applyRegInfo(chanReg RegisteredChannel) { + defer channel.resizeHistory(channel.server.Config()) + channel.stateMutex.Lock() defer channel.stateMutex.Unlock() @@ -120,6 +137,7 @@ func (channel *Channel) applyRegInfo(chanReg RegisteredChannel) { channel.createdTime = chanReg.RegisteredAt channel.key = chanReg.Key channel.userLimit = chanReg.UserLimit + channel.settings = chanReg.Settings for _, mode := range chanReg.Modes { channel.flags.SetMode(mode, true) @@ -164,6 +182,10 @@ func (channel *Channel) ExportRegistration(includeFlags uint) (info RegisteredCh } } + if includeFlags&IncludeSettings != 0 { + info.Settings = channel.settings + } + return } @@ -434,7 +456,7 @@ func (channel *Channel) Names(client *Client, rb *ResponseBuffer) { if modeSet == nil { continue } - if !isJoined && target.flags.HasMode(modes.Invisible) && !isOper { + if !isJoined && target.HasMode(modes.Invisible) && !isOper { continue } prefix := modeSet.Prefixes(isMultiPrefix) @@ -564,6 +586,48 @@ func (channel *Channel) IsEmpty() bool { return len(channel.members) == 0 } +// figure out where history is being stored: persistent, ephemeral, or neither +// target is only needed if we're doing persistent history +func (channel *Channel) historyStatus(config *Config) (persistent, ephemeral bool, target string) { + if !config.History.Persistent.Enabled { + return false, config.History.Enabled, "" + } + + channel.stateMutex.RLock() + target = channel.nameCasefolded + historyStatus := channel.settings.History + registered := channel.registeredFounder != "" + channel.stateMutex.RUnlock() + + historyStatus = historyEnabled(config.History.Persistent.RegisteredChannels, historyStatus) + + // ephemeral history: either the channel owner explicitly set the ephemeral preference, + // or persistent history is disabled for unregistered channels + if registered { + ephemeral = (historyStatus == HistoryEphemeral) + persistent = (historyStatus == HistoryPersistent) + } else { + ephemeral = config.History.Enabled && !config.History.Persistent.UnregisteredChannels + persistent = config.History.Persistent.UnregisteredChannels + } + return +} + +func (channel *Channel) AddHistoryItem(item history.Item) (err error) { + if !item.IsStorable() { + return + } + + persistent, ephemeral, target := channel.historyStatus(channel.server.Config()) + if ephemeral { + channel.history.Add(item) + } + if persistent { + return channel.server.historyDB.AddChannelItem(target, item) + } + return nil +} + // Join joins the given client to this channel (if they can be joined). func (channel *Channel) Join(client *Client, key string, isSajoin bool, rb *ResponseBuffer) { details := client.Details() @@ -643,15 +707,18 @@ func (channel *Channel) Join(client *Client, key string, isSajoin bool, rb *Resp channel.regenerateMembersCache() - message = utils.MakeMessage("") - histItem := history.Item{ - Type: history.Join, - Nick: details.nickMask, - AccountName: details.accountName, - Message: message, + // no history item for fake persistent joins + if rb != nil { + message = utils.MakeMessage("") + histItem := history.Item{ + Type: history.Join, + Nick: details.nickMask, + AccountName: details.accountName, + Message: message, + } + histItem.Params[0] = details.realname + channel.AddHistoryItem(histItem) } - histItem.Params[0] = details.realname - channel.history.Add(histItem) return }() @@ -665,7 +732,7 @@ func (channel *Channel) Join(client *Client, key string, isSajoin bool, rb *Resp for _, member := range channel.Members() { for _, session := range member.Sessions() { - if session == rb.session { + if rb != nil && session == rb.session { continue } else if client == session.client { channel.playJoinForSession(session) @@ -682,13 +749,13 @@ func (channel *Channel) Join(client *Client, key string, isSajoin bool, rb *Resp } } - if rb.session.capabilities.Has(caps.ExtendedJoin) { + if rb != nil && rb.session.capabilities.Has(caps.ExtendedJoin) { rb.AddFromClient(message.Time, message.Msgid, details.nickMask, details.accountName, nil, "JOIN", chname, details.accountName, details.realname) } else { rb.AddFromClient(message.Time, message.Msgid, details.nickMask, details.accountName, nil, "JOIN", chname) } - if rb.session.client == client { + if rb != nil && rb.session.client == client { // don't send topic and names for a SAJOIN of a different client channel.SendTopic(client, rb, false) channel.Names(client, rb) @@ -697,15 +764,29 @@ func (channel *Channel) Join(client *Client, key string, isSajoin bool, rb *Resp // TODO #259 can be implemented as Flush(false) (i.e., nonblocking) while holding joinPartMutex rb.Flush(true) - channel.autoReplayHistory(client, rb, message.Msgid) + if rb != nil { + channel.autoReplayHistory(client, rb, message.Msgid) + } } func (channel *Channel) autoReplayHistory(client *Client, rb *ResponseBuffer, skipMsgid string) { // autoreplay any messages as necessary - config := channel.server.Config() var items []history.Item + + var after, before time.Time if rb.session.zncPlaybackTimes != nil && (rb.session.zncPlaybackTimes.targets == nil || rb.session.zncPlaybackTimes.targets.Has(channel.NameCasefolded())) { - items, _ = channel.history.Between(rb.session.zncPlaybackTimes.after, rb.session.zncPlaybackTimes.before, false, config.History.ChathistoryMax) + after, before = rb.session.zncPlaybackTimes.after, rb.session.zncPlaybackTimes.before + } else if !rb.session.lastSignoff.IsZero() { + // we already checked for history caps in `playReattachMessages` + after = rb.session.lastSignoff + } + + if !after.IsZero() || !before.IsZero() { + _, seq, _ := channel.server.GetHistorySequence(channel, client, "") + if seq != nil { + zncMax := channel.server.Config().History.ZNCMax + items, _, _ = seq.Between(history.Selector{Time: after}, history.Selector{Time: before}, zncMax) + } } else if !rb.session.HasHistoryCaps() { var replayLimit int customReplayLimit := client.AccountSettings().AutoreplayLines @@ -719,7 +800,10 @@ func (channel *Channel) autoReplayHistory(client *Client, rb *ResponseBuffer, sk replayLimit = channel.server.Config().History.AutoreplayOnJoin } if 0 < replayLimit { - items = channel.history.Latest(replayLimit) + _, seq, _ := channel.server.GetHistorySequence(channel, client, "") + if seq != nil { + items, _, _ = seq.Between(history.Selector{}, history.Selector{}, replayLimit) + } } } // remove the client's own JOIN line from the replay @@ -784,7 +868,7 @@ func (channel *Channel) Part(client *Client, message string, rb *ResponseBuffer) } } - channel.history.Add(history.Item{ + channel.AddHistoryItem(history.Item{ Type: history.Part, Nick: details.nickMask, AccountName: details.accountName, @@ -799,10 +883,9 @@ func (channel *Channel) Part(client *Client, message string, rb *ResponseBuffer) // 2. Send JOIN and MODE lines to channel participants (including the new client) // 3. Replay missed message history to the client func (channel *Channel) Resume(session *Session, timestamp time.Time) { - now := time.Now().UTC() channel.resumeAndAnnounce(session) if !timestamp.IsZero() { - channel.replayHistoryForResume(session, timestamp, now) + channel.replayHistoryForResume(session, timestamp, time.Time{}) } } @@ -852,7 +935,13 @@ func (channel *Channel) resumeAndAnnounce(session *Session) { } func (channel *Channel) replayHistoryForResume(session *Session, after time.Time, before time.Time) { - items, complete := channel.history.Between(after, before, false, 0) + var items []history.Item + var complete bool + afterS, beforeS := history.Selector{Time: after}, history.Selector{Time: before} + _, seq, _ := channel.server.GetHistorySequence(channel, session.client, "") + if seq != nil { + items, complete, _ = seq.Between(afterS, beforeS, channel.server.Config().History.ZNCMax) + } rb := NewResponseBuffer(session) channel.replayHistoryItems(rb, items, false) if !complete && !session.resumeDetails.HistoryIncomplete { @@ -908,9 +997,9 @@ func (channel *Channel) replayHistoryItems(rb *ResponseBuffer, items []history.I case history.Join: if eventPlayback { if extendedJoin { - rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "JOIN", chname, item.AccountName, item.Params[0]) + rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "JOIN", chname, item.AccountName, item.Params[0]) } else { - rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "JOIN", chname) + rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "JOIN", chname) } } else { if !playJoinsAsPrivmsg { @@ -926,7 +1015,7 @@ func (channel *Channel) replayHistoryItems(rb *ResponseBuffer, items []history.I } case history.Part: if eventPlayback { - rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "PART", chname, item.Message.Message) + rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "PART", chname, item.Message.Message) } else { if !playJoinsAsPrivmsg { continue // #474 @@ -936,14 +1025,14 @@ func (channel *Channel) replayHistoryItems(rb *ResponseBuffer, items []history.I } case history.Kick: if eventPlayback { - rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "KICK", chname, item.Params[0], item.Message.Message) + rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "KICK", chname, item.Params[0], item.Message.Message) } else { message := fmt.Sprintf(client.t("%[1]s kicked %[2]s (%[3]s)"), nick, item.Params[0], item.Message.Message) rb.AddFromClient(item.Message.Time, utils.MungeSecretToken(item.Message.Msgid), histServMask, "*", nil, "PRIVMSG", chname, message) } case history.Quit: if eventPlayback { - rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "QUIT", item.Message.Message) + rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "QUIT", item.Message.Message) } else { if !playJoinsAsPrivmsg { continue // #474 @@ -953,7 +1042,7 @@ func (channel *Channel) replayHistoryItems(rb *ResponseBuffer, items []history.I } case history.Nick: if eventPlayback { - rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "NICK", item.Params[0]) + rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "NICK", item.Params[0]) } else { message := fmt.Sprintf(client.t("%[1]s changed nick to %[2]s"), nick, item.Params[0]) rb.AddFromClient(item.Message.Time, utils.MungeSecretToken(item.Message.Msgid), histServMask, "*", nil, "PRIVMSG", chname, message) @@ -1124,11 +1213,12 @@ func (channel *Channel) SendSplitMessage(command string, minPrefixMode modes.Mod // STATUSMSG continue } - if isCTCP && member.isTor { - continue // #753 - } for _, session := range member.Sessions() { + if isCTCP && session.isTor { + continue // #753 + } + var tagsToUse map[string]string if session.capabilities.Has(caps.MessageTags) { tagsToUse = clientOnlyTags @@ -1144,7 +1234,7 @@ func (channel *Channel) SendSplitMessage(command string, minPrefixMode modes.Mod } } - channel.history.Add(history.Item{ + channel.AddHistoryItem(history.Item{ Type: histType, Message: message, Nick: nickmask, @@ -1266,7 +1356,7 @@ func (channel *Channel) Kick(client *Client, target *Client, comment string, rb Message: message, } histItem.Params[0] = targetNick - channel.history.Add(histItem) + channel.AddHistoryItem(histItem) channel.Quit(target) } diff --git a/irc/channelreg.go b/irc/channelreg.go index 1258cd97..eb3733c9 100644 --- a/irc/channelreg.go +++ b/irc/channelreg.go @@ -33,6 +33,7 @@ const ( keyChannelModes = "channel.modes %s" keyChannelAccountToUMode = "channel.accounttoumode %s" keyChannelUserLimit = "channel.userlimit %s" + keyChannelSettings = "channel.settings %s" keyChannelPurged = "channel.purged %s" ) @@ -53,6 +54,7 @@ var ( keyChannelModes, keyChannelAccountToUMode, keyChannelUserLimit, + keyChannelSettings, } ) @@ -63,6 +65,7 @@ const ( IncludeTopic IncludeModes IncludeLists + IncludeSettings ) // this is an OR of all possible flags @@ -100,6 +103,8 @@ type RegisteredChannel struct { Excepts map[string]MaskInfo // Invites represents the invite exceptions set on the channel. Invites map[string]MaskInfo + // Settings are the chanserv-modifiable settings + Settings ChannelSettings } type ChannelPurgeRecord struct { @@ -203,6 +208,7 @@ func (reg *ChannelRegistry) LoadChannel(nameCasefolded string) (info RegisteredC exceptlistString, _ := tx.Get(fmt.Sprintf(keyChannelExceptlist, channelKey)) invitelistString, _ := tx.Get(fmt.Sprintf(keyChannelInvitelist, channelKey)) accountToUModeString, _ := tx.Get(fmt.Sprintf(keyChannelAccountToUMode, channelKey)) + settingsString, _ := tx.Get(fmt.Sprintf(keyChannelSettings, channelKey)) modeSlice := make([]modes.Mode, len(modeString)) for i, mode := range modeString { @@ -220,6 +226,9 @@ func (reg *ChannelRegistry) LoadChannel(nameCasefolded string) (info RegisteredC accountToUMode := make(map[string]modes.Mode) _ = json.Unmarshal([]byte(accountToUModeString), &accountToUMode) + var settings ChannelSettings + _ = json.Unmarshal([]byte(settingsString), &settings) + info = RegisteredChannel{ Name: name, RegisteredAt: time.Unix(regTimeInt, 0).UTC(), @@ -234,6 +243,7 @@ func (reg *ChannelRegistry) LoadChannel(nameCasefolded string) (info RegisteredC Invites: invitelist, AccountToUMode: accountToUMode, UserLimit: int(userLimit), + Settings: settings, } return nil }) @@ -357,6 +367,11 @@ func (reg *ChannelRegistry) saveChannel(tx *buntdb.Tx, channelInfo RegisteredCha accountToUModeString, _ := json.Marshal(channelInfo.AccountToUMode) tx.Set(fmt.Sprintf(keyChannelAccountToUMode, channelKey), string(accountToUModeString), nil) } + + if includeFlags&IncludeSettings != 0 { + settingsString, _ := json.Marshal(channelInfo.Settings) + tx.Set(fmt.Sprintf(keyChannelSettings, channelKey), string(settingsString), nil) + } } // PurgeChannel records a channel purge. diff --git a/irc/chanserv.go b/irc/chanserv.go index 6fb748e5..38db725c 100644 --- a/irc/chanserv.go +++ b/irc/chanserv.go @@ -137,6 +137,35 @@ INFO displays info about a registered channel.`, enabled: chanregEnabled, minParams: 1, }, + "get": { + handler: csGetHandler, + help: `Syntax: $bGET #channel $b + +GET queries the current values of the channel settings. For more information +on the settings and their possible values, see HELP SET.`, + helpShort: `$bGET$b queries the current values of a channel's settings`, + enabled: chanregEnabled, + minParams: 2, + }, + "set": { + handler: csSetHandler, + helpShort: `$bSET$b modifies a channel's settings`, + // these are broken out as separate strings so they can be translated separately + helpStrings: []string{ + `Syntax $bSET #channel $b + +SET modifies a channel's settings. The following settings are available:`, + + `$bHISTORY$b +'history' lets you control how channel history is stored. Your options are: +1. 'off' [no history] +2. 'ephemeral' [a limited amount of temporary history, not stored on disk] +3. 'on' [history stored in a permanent database, if available] +4. 'default' [use the server default]`, + }, + enabled: chanregEnabled, + minParams: 3, + }, } ) @@ -320,6 +349,22 @@ func checkChanLimit(client *Client, rb *ResponseBuffer) (ok bool) { return } +func csPrivsCheck(channel RegisteredChannel, client *Client, rb *ResponseBuffer) (success bool) { + founder := channel.Founder + if founder == "" { + csNotice(rb, client.t("That channel is not registered")) + return false + } + if client.HasRoleCapabs("chanreg") { + return true + } + if founder != client.Account() { + csNotice(rb, client.t("Insufficient privileges")) + return false + } + return true +} + func csUnregisterHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) { channelName := params[0] var verificationCode string @@ -327,31 +372,18 @@ func csUnregisterHandler(server *Server, client *Client, command string, params verificationCode = params[1] } - channelKey, err := CasefoldChannel(channelName) - if channelKey == "" || err != nil { - csNotice(rb, client.t("Channel name is not valid")) - return - } - - channel := server.channels.Get(channelKey) + channel := server.channels.Get(channelName) if channel == nil { csNotice(rb, client.t("No such channel")) return } - founder := channel.Founder() - if founder == "" { - csNotice(rb, client.t("That channel is not registered")) - return - } - - hasPrivs := client.HasRoleCapabs("chanreg") || founder == client.Account() - if !hasPrivs { - csNotice(rb, client.t("Insufficient privileges")) - return - } - info := channel.ExportRegistration(0) + channelKey := info.NameCasefolded + if !csPrivsCheck(info, client, rb) { + return + } + expectedCode := unregisterConfirmationCode(info.Name, info.RegisteredAt) if expectedCode != verificationCode { csNotice(rb, ircfmt.Unescape(client.t("$bWarning: unregistering this channel will remove all stored channel attributes.$b"))) @@ -359,7 +391,7 @@ func csUnregisterHandler(server *Server, client *Client, command string, params return } - server.channels.SetUnregistered(channelKey, founder) + server.channels.SetUnregistered(channelKey, info.Founder) csNotice(rb, fmt.Sprintf(client.t("Channel %s is now unregistered"), channelKey)) } @@ -377,9 +409,7 @@ func csClearHandler(server *Server, client *Client, command string, params []str csNotice(rb, client.t("Channel does not exist")) return } - account := client.Account() - if !(client.HasRoleCapabs("chanreg") || (account != "" && account == channel.Founder())) { - csNotice(rb, client.t("Insufficient privileges")) + if !csPrivsCheck(channel.ExportRegistration(0), client, rb) { return } @@ -573,3 +603,68 @@ func csInfoHandler(server *Server, client *Client, command string, params []stri csNotice(rb, fmt.Sprintf(client.t("Founder: %s"), chinfo.Founder)) csNotice(rb, fmt.Sprintf(client.t("Registered at: %s"), chinfo.RegisteredAt.Format(time.RFC1123))) } + +func displayChannelSetting(settingName string, settings ChannelSettings, client *Client, rb *ResponseBuffer) { + config := client.server.Config() + + switch strings.ToLower(settingName) { + case "history": + effectiveValue := historyEnabled(config.History.Persistent.RegisteredChannels, settings.History) + csNotice(rb, fmt.Sprintf(client.t("The stored channel history setting is: %s"), historyStatusToString(settings.History))) + csNotice(rb, fmt.Sprintf(client.t("Given current server settings, the channel history setting is: %s"), historyStatusToString(effectiveValue))) + default: + csNotice(rb, client.t("Invalid params")) + } +} + +func csGetHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) { + chname, setting := params[0], params[1] + channel := server.channels.Get(chname) + if channel == nil { + csNotice(rb, client.t("No such channel")) + return + } + info := channel.ExportRegistration(IncludeSettings) + if !csPrivsCheck(info, client, rb) { + return + } + + displayChannelSetting(setting, info.Settings, client, rb) +} + +func csSetHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) { + chname, setting, value := params[0], params[1], params[2] + channel := server.channels.Get(chname) + if channel == nil { + csNotice(rb, client.t("No such channel")) + return + } + info := channel.ExportRegistration(IncludeSettings) + settings := info.Settings + if !csPrivsCheck(info, client, rb) { + return + } + + var err error + switch strings.ToLower(setting) { + case "history": + settings.History, err = historyStatusFromString(value) + if err != nil { + err = errInvalidParams + break + } + channel.SetSettings(settings) + channel.resizeHistory(server.Config()) + } + + switch err { + case nil: + csNotice(rb, client.t("Successfully changed the channel settings")) + displayChannelSetting(setting, settings, client, rb) + case errInvalidParams: + csNotice(rb, client.t("Invalid parameters")) + default: + server.logger.Error("internal", "CS SET error:", err.Error()) + csNotice(rb, client.t("An error occurred")) + } +} diff --git a/irc/client.go b/irc/client.go index 06fde3dd..1da7c6a2 100644 --- a/irc/client.go +++ b/irc/client.go @@ -29,7 +29,7 @@ import ( const ( // IdentTimeoutSeconds is how many seconds before our ident (username) check times out. IdentTimeoutSeconds = 1.5 - IRCv3TimestampFormat = "2006-01-02T15:04:05.000Z" + IRCv3TimestampFormat = utils.IRCv3TimestampFormat ) // ResumeDetails is a place to stash data at various stages of @@ -45,6 +45,7 @@ type ResumeDetails struct { type Client struct { account string accountName string // display name of the account: uncasefolded, '*' if not logged in + accountRegDate time.Time accountSettings AccountSettings atime time.Time away bool @@ -55,12 +56,12 @@ type Client struct { ctime time.Time destroyed bool exitedSnomaskSent bool - flags modes.ModeSet + modes modes.ModeSet hostname string invitedTo map[string]bool isSTSOnly bool - isTor bool languages []string + lastSignoff time.Time // for always-on clients, the time their last session quit loginThrottle connection_limits.GenericThrottle nick string nickCasefolded string @@ -84,9 +85,12 @@ type Client struct { skeleton string sessions []*Session stateMutex sync.RWMutex // tier 1 + alwaysOn bool username string vhost string history history.Buffer + dirtyBits uint + writerSemaphore utils.Semaphore // tier 1.5 } // Session is an individual client connection to the server (TCP connection @@ -102,6 +106,7 @@ type Session struct { realIP net.IP proxiedIP net.IP rawHostname string + isTor bool idletimer IdleTimer fakelag Fakelag @@ -120,6 +125,7 @@ type Session struct { resumeID string resumeDetails *ResumeDetails zncPlaybackTimes *zncPlaybackTimes + lastSignoff time.Time batch MultilineBatch } @@ -147,6 +153,13 @@ func (sd *Session) SetQuitMessage(message string) (set bool) { } } +func (s *Session) IP() net.IP { + if s.proxiedIP != nil { + return s.proxiedIP + } + return s.realIP +} + // returns whether the session was actively destroyed (for example, by ping // timeout or NS GHOST). // avoids a race condition between asynchronous idle-timing-out of sessions, @@ -164,8 +177,7 @@ func (session *Session) SetDestroyed() { // returns whether the client supports a smart history replay cap, // and therefore autoreplay-on-join and similar should be suppressed func (session *Session) HasHistoryCaps() bool { - // TODO the chathistory cap will go here as well - return session.capabilities.Has(caps.ZNCPlayback) + return session.capabilities.Has(caps.Chathistory) || session.capabilities.Has(caps.ZNCPlayback) } // generates a batch ID. the uniqueness requirements for this are fairly weak: @@ -231,7 +243,6 @@ func (server *Server) RunClient(conn clientConn, proxyLine string) { channels: make(ChannelSet), ctime: now, isSTSOnly: conn.Config.STSOnly, - isTor: conn.Config.Tor, languages: server.Languages().Default(), loginThrottle: connection_limits.GenericThrottle{ Duration: config.Accounts.LoginThrottling.Duration, @@ -253,6 +264,7 @@ func (server *Server) RunClient(conn clientConn, proxyLine string) { ctime: now, atime: now, realIP: realIP, + isTor: conn.Config.Tor, } client.sessions = []*Session{session} @@ -272,7 +284,7 @@ func (server *Server) RunClient(conn clientConn, proxyLine string) { client.rawHostname = session.rawHostname } else { remoteAddr := conn.Conn.RemoteAddr() - if utils.AddrIsLocal(remoteAddr) { + if realIP.IsLoopback() || utils.IPInNets(realIP, config.Server.secureNets) { // treat local connections as secure (may be overridden later by WEBIRC) client.SetMode(modes.TLS, true) } @@ -286,10 +298,65 @@ func (server *Server) RunClient(conn clientConn, proxyLine string) { client.run(session, proxyLine) } +func (server *Server) AddAlwaysOnClient(account ClientAccount, chnames []string) { + now := time.Now().UTC() + config := server.Config() + + client := &Client{ + atime: now, + channels: make(ChannelSet), + ctime: now, + languages: server.Languages().Default(), + server: server, + + // TODO figure out how to set these on reattach? + username: "~user", + rawHostname: server.name, + realIP: utils.IPv4LoopbackAddress, + + alwaysOn: true, + } + + client.SetMode(modes.TLS, true) + client.writerSemaphore.Initialize(1) + client.history.Initialize(0, 0) + client.brbTimer.Initialize(client) + + server.accounts.Login(client, account) + + client.resizeHistory(config) + + _, err := server.clients.SetNick(client, nil, account.Name) + if err != nil { + server.logger.Error("internal", "could not establish always-on client", account.Name, err.Error()) + return + } else { + server.logger.Debug("accounts", "established always-on client", account.Name) + } + + // XXX set this last to avoid confusing SetNick: + client.registered = true + + for _, chname := range chnames { + // XXX we're using isSajoin=true, to make these joins succeed even without channel key + // this is *probably* ok as long as the persisted memberships are accurate + server.channels.Join(client, chname, "", true, nil) + } +} + +func (client *Client) resizeHistory(config *Config) { + _, ephemeral := client.historyStatus(config) + if ephemeral { + client.history.Resize(config.History.ClientLength, config.History.AutoresizeWindow) + } else { + client.history.Resize(0, 0) + } +} + // resolve an IP to an IRC-ready hostname, using reverse DNS, forward-confirming if necessary, // and sending appropriate notices to the client func (client *Client) lookupHostname(session *Session, overwrite bool) { - if client.isTor { + if session.isTor { return } // else: even if cloaking is enabled, look up the real hostname to show to operators @@ -384,14 +451,14 @@ const ( authFailSaslRequired ) -func (client *Client) isAuthorized(config *Config) AuthOutcome { +func (client *Client) isAuthorized(config *Config, isTor bool) AuthOutcome { saslSent := client.account != "" // PASS requirement if (config.Server.passwordBytes != nil) && !client.sentPassCommand && !(config.Accounts.SkipServerPassword && saslSent) { return authFailPass } // Tor connections may be required to authenticate with SASL - if client.isTor && config.Server.TorListeners.RequireSasl && !saslSent { + if isTor && config.Server.TorListeners.RequireSasl && !saslSent { return authFailTorSaslRequired } // finally, enforce require-sasl @@ -572,9 +639,13 @@ func (client *Client) run(session *Session, proxyLine string) { func (client *Client) playReattachMessages(session *Session) { client.server.playRegistrationBurst(session) + hasHistoryCaps := session.HasHistoryCaps() for _, channel := range session.client.Channels() { channel.playJoinForSession(session) - // clients should receive autoreplay-on-join lines, if applicable; + // clients should receive autoreplay-on-join lines, if applicable: + if hasHistoryCaps { + continue + } // if they negotiated znc.in/playback or chathistory, they will receive nothing, // because those caps disable autoreplay-on-join and they haven't sent the relevant // *playback PRIVMSG or CHATHISTORY command yet @@ -582,6 +653,12 @@ func (client *Client) playReattachMessages(session *Session) { channel.autoReplayHistory(client, rb, "") rb.Send(true) } + if !session.lastSignoff.IsZero() && !hasHistoryCaps { + rb := NewResponseBuffer(session) + zncPlayPrivmsgs(client, rb, session.lastSignoff, time.Time{}) + rb.Send(true) + } + session.lastSignoff = time.Time{} } // @@ -634,11 +711,6 @@ func (session *Session) tryResume() (success bool) { return } - if oldClient.isTor != client.isTor { - session.Send(nil, server.name, "FAIL", "RESUME", "INSECURE_SESSION", client.t("Cannot resume connection from Tor to non-Tor or vice versa")) - return - } - err := server.clients.Resume(oldClient, session) if err != nil { session.Send(nil, server.name, "FAIL", "RESUME", "CANNOT_RESUME", client.t("Cannot resume connection")) @@ -657,37 +729,45 @@ func (session *Session) tryResume() (success bool) { func (session *Session) playResume() { client := session.client server := client.server + config := server.Config() friends := make(ClientSet) - oldestLostMessage := time.Now().UTC() + var oldestLostMessage time.Time // work out how much time, if any, is not covered by history buffers + // assume that a persistent buffer covers the whole resume period for _, channel := range client.Channels() { for _, member := range channel.Members() { friends.Add(member) + } + _, ephemeral, _ := channel.historyStatus(config) + if ephemeral { lastDiscarded := channel.history.LastDiscarded() - if lastDiscarded.Before(oldestLostMessage) { + if oldestLostMessage.Before(lastDiscarded) { oldestLostMessage = lastDiscarded } } } - privmsgMatcher := func(item history.Item) bool { - return item.Type == history.Privmsg || item.Type == history.Notice || item.Type == history.Tagmsg + _, cEphemeral := client.historyStatus(config) + if cEphemeral { + lastDiscarded := client.history.LastDiscarded() + if oldestLostMessage.Before(lastDiscarded) { + oldestLostMessage = lastDiscarded + } } - privmsgHistory := client.history.Match(privmsgMatcher, false, 0) - lastDiscarded := client.history.LastDiscarded() - if lastDiscarded.Before(oldestLostMessage) { - oldestLostMessage = lastDiscarded - } - for _, item := range privmsgHistory { - sender := server.clients.Get(stripMaskFromNick(item.Nick)) - if sender != nil { - friends.Add(sender) + _, privmsgSeq, _ := server.GetHistorySequence(nil, client, "*") + if privmsgSeq != nil { + privmsgs, _, _ := privmsgSeq.Between(history.Selector{}, history.Selector{}, config.History.ClientLength) + for _, item := range privmsgs { + sender := server.clients.Get(stripMaskFromNick(item.Nick)) + if sender != nil { + friends.Add(sender) + } } } timestamp := session.resumeDetails.Timestamp - gap := lastDiscarded.Sub(timestamp) + gap := oldestLostMessage.Sub(timestamp) session.resumeDetails.HistoryIncomplete = gap > 0 || timestamp.IsZero() gapSeconds := int(gap.Seconds()) + 1 // round up to avoid confusion @@ -723,10 +803,12 @@ func (session *Session) playResume() { } } - if session.resumeDetails.HistoryIncomplete && !timestamp.IsZero() { - session.Send(nil, client.server.name, "WARN", "RESUME", "HISTORY_LOST", fmt.Sprintf(client.t("Resume may have lost up to %d seconds of history"), gapSeconds)) - } else { - session.Send(nil, client.server.name, "WARN", "RESUME", "HISTORY_LOST", client.t("Resume may have lost some message history")) + if session.resumeDetails.HistoryIncomplete { + if !timestamp.IsZero() { + session.Send(nil, client.server.name, "WARN", "RESUME", "HISTORY_LOST", fmt.Sprintf(client.t("Resume may have lost up to %d seconds of history"), gapSeconds)) + } else { + session.Send(nil, client.server.name, "WARN", "RESUME", "HISTORY_LOST", client.t("Resume may have lost some message history")) + } } session.Send(nil, client.server.name, "RESUME", "SUCCESS", details.nick) @@ -738,23 +820,26 @@ func (session *Session) playResume() { } // replay direct PRIVSMG history - if !timestamp.IsZero() { - now := time.Now().UTC() - items, complete := client.history.Between(timestamp, now, false, 0) - rb := NewResponseBuffer(client.Sessions()[0]) - client.replayPrivmsgHistory(rb, items, complete) + if !timestamp.IsZero() && privmsgSeq != nil { + after := history.Selector{Time: timestamp} + items, complete, _ := privmsgSeq.Between(after, history.Selector{}, config.History.ZNCMax) + rb := NewResponseBuffer(session) + client.replayPrivmsgHistory(rb, items, "", complete) rb.Send(true) } session.resumeDetails = nil } -func (client *Client) replayPrivmsgHistory(rb *ResponseBuffer, items []history.Item, complete bool) { +func (client *Client) replayPrivmsgHistory(rb *ResponseBuffer, items []history.Item, target string, complete bool) { var batchID string details := client.Details() nick := details.nick if 0 < len(items) { - batchID = rb.StartNestedHistoryBatch(nick) + if target == "" { + target = nick + } + batchID = rb.StartNestedHistoryBatch(target) } allowTags := rb.session.capabilities.Has(caps.MessageTags) @@ -778,12 +863,12 @@ func (client *Client) replayPrivmsgHistory(rb *ResponseBuffer, items []history.I if allowTags { tags = item.Tags } - if item.Params[0] == "" { + if item.Params[0] == "" || item.Params[0] == nick { // this message was sent *to* the client from another nick rb.AddSplitMessageFromClient(item.Nick, item.AccountName, tags, command, nick, item.Message) } else { // this message was sent *from* the client to another nick; the target is item.Params[0] - // substitute the client's current nickmask in case they changed nick + // substitute client's current nickmask in case client changed nick rb.AddSplitMessageFromClient(details.nickMask, item.AccountName, tags, command, item.Params[0], item.Message) } } @@ -875,7 +960,7 @@ func (client *Client) HasRoleCapabs(capabs ...string) bool { // ModeString returns the mode string for this client. func (client *Client) ModeString() (str string) { - return "+" + client.flags.String() + return "+" + client.modes.String() } // Friends refers to clients that share a channel with this client. @@ -1053,6 +1138,12 @@ func (client *Client) Quit(message string, session *Session) { // has no more sessions. func (client *Client) destroy(session *Session) { var sessionsToDestroy []*Session + var lastSignoff time.Time + if session != nil { + lastSignoff = session.idletimer.LastTouch() + } else { + lastSignoff = time.Now().UTC() + } client.stateMutex.Lock() details := client.detailsNoMutex() @@ -1060,6 +1151,8 @@ func (client *Client) destroy(session *Session) { brbAt := client.brbTimer.brbAt wasReattach := session != nil && session.client != client sessionRemoved := false + registered := client.registered + alwaysOn := client.alwaysOn var remainingSessions int if session == nil { sessionsToDestroy = client.sessions @@ -1074,12 +1167,15 @@ func (client *Client) destroy(session *Session) { // should we destroy the whole client this time? // BRB is not respected if this is a destroy of the whole client (i.e., session == nil) - brbEligible := session != nil && (brbState == BrbEnabled || brbState == BrbSticky) + brbEligible := session != nil && (brbState == BrbEnabled || alwaysOn) shouldDestroy := !client.destroyed && remainingSessions == 0 && !brbEligible if shouldDestroy { // if it's our job to destroy it, don't let anyone else try client.destroyed = true } + if alwaysOn && remainingSessions == 0 { + client.lastSignoff = lastSignoff + } exitedSnomaskSent := client.exitedSnomaskSent client.stateMutex.Unlock() @@ -1099,7 +1195,7 @@ func (client *Client) destroy(session *Session) { // remove from connection limits var source string - if client.isTor { + if session.isTor { client.server.torLimiter.RemoveClient() source = "tor" } else { @@ -1113,11 +1209,33 @@ func (client *Client) destroy(session *Session) { client.server.logger.Info("localconnect-ip", fmt.Sprintf("disconnecting session of %s from %s", details.nick, source)) } + // decrement stats if we have no more sessions, even if the client will not be destroyed + if shouldDestroy || remainingSessions == 0 { + invisible := client.HasMode(modes.Invisible) + operator := client.HasMode(modes.LocalOperator) || client.HasMode(modes.Operator) + client.server.stats.Remove(registered, invisible, operator) + } + // do not destroy the client if it has either remaining sessions, or is BRB'ed if !shouldDestroy { return } + splitQuitMessage := utils.MakeMessage(quitMessage) + quitItem := history.Item{ + Type: history.Quit, + Nick: details.nickMask, + AccountName: details.accountName, + Message: splitQuitMessage, + } + var channels []*Channel + defer func() { + for _, channel := range channels { + // TODO it's dangerous to write to mysql while holding the destroy semaphore + channel.AddHistoryItem(quitItem) + } + }() + // see #235: deduplicating the list of PART recipients uses (comparatively speaking) // a lot of RAM, so limit concurrency to avoid thrashing client.server.semaphores.ClientDestroy.Acquire() @@ -1127,7 +1245,6 @@ func (client *Client) destroy(session *Session) { client.server.logger.Debug("quit", fmt.Sprintf("%s is no longer on the server", details.nick)) } - registered := client.Registered() if registered { client.server.whoWas.Append(client.WhoWas()) } @@ -1141,18 +1258,12 @@ func (client *Client) destroy(session *Session) { // clean up monitor state client.server.monitorManager.RemoveAll(client) - splitQuitMessage := utils.MakeMessage(quitMessage) // clean up channels // (note that if this is a reattach, client has no channels and therefore no friends) friends := make(ClientSet) - for _, channel := range client.Channels() { + channels = client.Channels() + for _, channel := range channels { channel.Quit(client) - channel.history.Add(history.Item{ - Type: history.Quit, - Nick: details.nickMask, - AccountName: details.accountName, - Message: splitQuitMessage, - }) for _, member := range channel.Members() { friends.Add(member) } @@ -1168,9 +1279,6 @@ func (client *Client) destroy(session *Session) { client.server.accounts.Logout(client) - client.server.stats.Remove(registered, client.HasMode(modes.Invisible), - client.HasMode(modes.Operator) || client.HasMode(modes.LocalOperator)) - // this happens under failure to return from BRB if quitMessage == "" { if brbState == BrbDead && !brbAt.IsZero() { @@ -1196,11 +1304,10 @@ func (client *Client) destroy(session *Session) { // SendSplitMsgFromClient sends an IRC PRIVMSG/NOTICE coming from a specific client. // Adds account-tag to the line as well. func (session *Session) sendSplitMsgFromClientInternal(blocking bool, nickmask, accountName string, tags map[string]string, command, target string, message utils.SplitMessage) { - // TODO no maxline support if message.Is512() { session.sendFromClientInternal(blocking, message.Time, message.Msgid, nickmask, accountName, tags, command, target, message.Message) } else { - if message.IsMultiline() && session.capabilities.Has(caps.Multiline) { + if session.capabilities.Has(caps.Multiline) { for _, msg := range session.composeMultilineBatch(nickmask, accountName, tags, command, target, message) { session.SendRawMessage(msg, blocking) } @@ -1366,13 +1473,23 @@ func (session *Session) Notice(text string) { func (client *Client) addChannel(channel *Channel) { client.stateMutex.Lock() client.channels[channel] = true + alwaysOn := client.alwaysOn client.stateMutex.Unlock() + + if alwaysOn { + client.markDirty(IncludeChannels) + } } func (client *Client) removeChannel(channel *Channel) { client.stateMutex.Lock() delete(client.channels, channel) + alwaysOn := client.alwaysOn client.stateMutex.Unlock() + + if alwaysOn { + client.markDirty(IncludeChannels) + } } // Records that the client has been invited to join an invite-only channel @@ -1413,3 +1530,84 @@ func (client *Client) attemptAutoOper(session *Session) { } } } + +func (client *Client) historyStatus(config *Config) (persistent, ephemeral bool) { + if !config.History.Enabled { + return false, false + } else if !config.History.Persistent.Enabled { + return false, true + } + + client.stateMutex.RLock() + alwaysOn := client.alwaysOn + historyStatus := client.accountSettings.DMHistory + client.stateMutex.RUnlock() + + if !alwaysOn { + return false, true + } + + historyStatus = historyEnabled(config.History.Persistent.DirectMessages, historyStatus) + ephemeral = (historyStatus == HistoryEphemeral) + persistent = (historyStatus == HistoryPersistent) + return +} + +// these are bit flags indicating what part of the client status is "dirty" +// and needs to be read from memory and written to the db +// TODO add a dirty flag for lastSignoff +const ( + IncludeChannels uint = 1 << iota +) + +func (client *Client) markDirty(dirtyBits uint) { + client.stateMutex.Lock() + alwaysOn := client.alwaysOn + client.dirtyBits = client.dirtyBits | dirtyBits + client.stateMutex.Unlock() + + if alwaysOn { + client.wakeWriter() + } +} + +func (client *Client) wakeWriter() { + if client.writerSemaphore.TryAcquire() { + go client.writeLoop() + } +} + +func (client *Client) writeLoop() { + for { + client.performWrite() + client.writerSemaphore.Release() + + client.stateMutex.RLock() + isDirty := client.dirtyBits != 0 + client.stateMutex.RUnlock() + + if !isDirty || !client.writerSemaphore.TryAcquire() { + return + } + } +} + +func (client *Client) performWrite() { + client.stateMutex.Lock() + // TODO actually read dirtyBits in the future + client.dirtyBits = 0 + account := client.account + client.stateMutex.Unlock() + + if account == "" { + client.server.logger.Error("internal", "attempting to persist logged-out client", client.Nick()) + return + } + + channels := client.Channels() + channelNames := make([]string, len(channels)) + for i, channel := range channels { + channelNames[i] = channel.Name() + } + client.server.accounts.saveChannels(account, channelNames) +} diff --git a/irc/client_lookup_set.go b/irc/client_lookup_set.go index 87775fad..7e38b9aa 100644 --- a/irc/client_lookup_set.go +++ b/irc/client_lookup_set.go @@ -105,7 +105,8 @@ func (clients *ClientManager) Resume(oldClient *Client, session *Session) (err e return errNickMissing } - if !oldClient.AddSession(session) { + success, _, _ := oldClient.AddSession(session) + if !success { return errNickMissing } @@ -113,32 +114,50 @@ func (clients *ClientManager) Resume(oldClient *Client, session *Session) (err e } // SetNick sets a client's nickname, validating it against nicknames in use -func (clients *ClientManager) SetNick(client *Client, session *Session, newNick string) error { - if len(newNick) > client.server.Config().Limits.NickLen { - return errNicknameInvalid - } +func (clients *ClientManager) SetNick(client *Client, session *Session, newNick string) (setNick string, err error) { newcfnick, err := CasefoldName(newNick) if err != nil { - return errNicknameInvalid + return "", errNicknameInvalid + } + if len(newcfnick) > client.server.Config().Limits.NickLen { + return "", errNicknameInvalid } newSkeleton, err := Skeleton(newNick) if err != nil { - return errNicknameInvalid + return "", errNicknameInvalid } if restrictedCasefoldedNicks[newcfnick] || restrictedSkeletons[newSkeleton] { - return errNicknameInvalid + return "", errNicknameInvalid } reservedAccount, method := client.server.accounts.EnforcementStatus(newcfnick, newSkeleton) - account := client.Account() config := client.server.Config() + client.stateMutex.RLock() + account := client.account + accountName := client.accountName + settings := client.accountSettings + registered := client.registered + realname := client.realname + client.stateMutex.RUnlock() + + // recompute this (client.alwaysOn is not set for unregistered clients): + alwaysOn := account != "" && persistenceEnabled(config.Accounts.Bouncer.AlwaysOn, settings.AlwaysOn) + + if alwaysOn && registered { + return "", errCantChangeNick + } + var bouncerAllowed bool if config.Accounts.Bouncer.Enabled { - if session != nil && session.capabilities.Has(caps.Bouncer) { + if alwaysOn { + // ignore the pre-reg nick, force a reattach + newNick = accountName + newcfnick = account + bouncerAllowed = true + } else if session != nil && session.capabilities.Has(caps.Bouncer) { bouncerAllowed = true } else { - settings := client.AccountSettings() if config.Accounts.Bouncer.AllowedByDefault && settings.AllowBouncer != BouncerDisallowedByUser { bouncerAllowed = true } else if settings.AllowBouncer == BouncerAllowedByUser { @@ -154,28 +173,41 @@ func (clients *ClientManager) SetNick(client *Client, session *Session, newNick // the client may just be changing case if currentClient != nil && currentClient != client && session != nil { // these conditions forbid reattaching to an existing session: - if client.Registered() || !bouncerAllowed || account == "" || account != currentClient.Account() || client.HasMode(modes.TLS) != currentClient.HasMode(modes.TLS) { - return errNicknameInUse + if registered || !bouncerAllowed || account == "" || account != currentClient.Account() || client.HasMode(modes.TLS) != currentClient.HasMode(modes.TLS) { + return "", errNicknameInUse } - if !currentClient.AddSession(session) { - return errNicknameInUse + reattachSuccessful, numSessions, lastSignoff := currentClient.AddSession(session) + if !reattachSuccessful { + return "", errNicknameInUse } + if numSessions == 1 { + invisible := client.HasMode(modes.Invisible) + operator := client.HasMode(modes.Operator) || client.HasMode(modes.LocalOperator) + client.server.stats.AddRegistered(invisible, operator) + } + session.lastSignoff = lastSignoff + // XXX SetNames only changes names if they are unset, so the realname change only + // takes effect on first attach to an always-on client (good), but the user/ident + // change is always a no-op (bad). we could make user/ident act the same way as + // realname, but then we'd have to send CHGHOST and i don't want to deal with that + // for performance reasons + currentClient.SetNames("user", realname, true) // successful reattach! - return nil + return newNick, nil } // analogous checks for skeletons skeletonHolder := clients.bySkeleton[newSkeleton] if skeletonHolder != nil && skeletonHolder != client { - return errNicknameInUse + return "", errNicknameInUse } if method == NickEnforcementStrict && reservedAccount != "" && reservedAccount != account { - return errNicknameReserved + return "", errNicknameReserved } clients.removeInternal(client) clients.byNick[newcfnick] = client clients.bySkeleton[newSkeleton] = client client.updateNick(newNick, newcfnick, newSkeleton) - return nil + return newNick, nil } func (clients *ClientManager) AllClients() (result []*Client) { diff --git a/irc/commands.go b/irc/commands.go index 859636b8..dd074019 100644 --- a/irc/commands.go +++ b/irc/commands.go @@ -54,6 +54,12 @@ func (cmd *Command) Run(server *Server, client *Client, session *Session, msg ir return cmd.handler(server, client, msg, rb) }() + // most servers do this only for PING/PONG, but we'll do it for any command: + if client.registered { + // touch even if `exiting`, so we record the time of a QUIT accurately + session.idletimer.Touch() + } + if exiting { return } @@ -63,11 +69,6 @@ func (cmd *Command) Run(server *Server, client *Client, session *Session, msg ir exiting = server.tryRegister(client, session) } - // most servers do this only for PING/PONG, but we'll do it for any command: - if client.registered { - session.idletimer.Touch() - } - if client.registered && !cmd.leaveClientIdle { client.Active(session) } @@ -109,7 +110,7 @@ func init() { }, "CHATHISTORY": { handler: chathistoryHandler, - minParams: 3, + minParams: 4, }, "DEBUG": { handler: debugHandler, diff --git a/irc/config.go b/irc/config.go index 1944f383..c7fe707c 100644 --- a/irc/config.go +++ b/irc/config.go @@ -61,6 +61,151 @@ type listenerConfig struct { ProxyBeforeTLS bool } +type PersistentStatus uint + +const ( + PersistentUnspecified PersistentStatus = iota + PersistentDisabled + PersistentOptIn + PersistentOptOut + PersistentMandatory +) + +func persistentStatusToString(status PersistentStatus) string { + switch status { + case PersistentUnspecified: + return "default" + case PersistentDisabled: + return "disabled" + case PersistentOptIn: + return "opt-in" + case PersistentOptOut: + return "opt-out" + case PersistentMandatory: + return "mandatory" + default: + return "" + } +} + +func persistentStatusFromString(status string) (PersistentStatus, error) { + switch strings.ToLower(status) { + case "default": + return PersistentUnspecified, nil + case "": + return PersistentDisabled, nil + case "opt-in": + return PersistentOptIn, nil + case "opt-out": + return PersistentOptOut, nil + case "mandatory": + return PersistentMandatory, nil + default: + b, err := utils.StringToBool(status) + if b { + return PersistentMandatory, err + } else { + return PersistentDisabled, err + } + } +} + +func (ps *PersistentStatus) UnmarshalYAML(unmarshal func(interface{}) error) error { + var orig string + var err error + if err = unmarshal(&orig); err != nil { + return err + } + result, err := persistentStatusFromString(orig) + if err == nil { + if result == PersistentUnspecified { + result = PersistentDisabled + } + *ps = result + } + return err +} + +func persistenceEnabled(serverSetting, clientSetting PersistentStatus) (enabled bool) { + if serverSetting == PersistentDisabled { + return false + } else if serverSetting == PersistentMandatory { + return true + } else if clientSetting == PersistentDisabled { + return false + } else if clientSetting == PersistentMandatory { + return true + } else if serverSetting == PersistentOptOut { + return true + } else { + return false + } +} + +type HistoryStatus uint + +const ( + HistoryDefault HistoryStatus = iota + HistoryDisabled + HistoryEphemeral + HistoryPersistent +) + +func historyStatusFromString(str string) (status HistoryStatus, err error) { + switch strings.ToLower(str) { + case "default": + return HistoryDefault, nil + case "ephemeral": + return HistoryEphemeral, nil + case "persistent": + return HistoryPersistent, nil + default: + b, err := utils.StringToBool(str) + if b { + return HistoryPersistent, err + } else { + return HistoryDisabled, err + } + } +} + +func historyStatusToString(status HistoryStatus) string { + switch status { + case HistoryDefault: + return "default" + case HistoryDisabled: + return "disabled" + case HistoryEphemeral: + return "ephemeral" + case HistoryPersistent: + return "persistent" + default: + return "" + } +} + +func historyEnabled(serverSetting PersistentStatus, localSetting HistoryStatus) (result HistoryStatus) { + if serverSetting == PersistentDisabled { + return HistoryDisabled + } else if serverSetting == PersistentMandatory { + return HistoryPersistent + } else if serverSetting == PersistentOptOut { + if localSetting == HistoryDefault { + return HistoryPersistent + } else { + return localSetting + } + } else if serverSetting == PersistentOptIn { + if localSetting >= HistoryEphemeral { + return localSetting + } else { + return HistoryDisabled + } + } else { + return HistoryDisabled + } +} + type AccountConfig struct { Registration AccountRegistrationConfig AuthenticationEnabled bool `yaml:"authentication-enabled"` @@ -79,7 +224,8 @@ type AccountConfig struct { NickReservation NickReservationConfig `yaml:"nick-reservation"` Bouncer struct { Enabled bool - AllowedByDefault bool `yaml:"allowed-by-default"` + AllowedByDefault bool `yaml:"allowed-by-default"` + AlwaysOn PersistentStatus `yaml:"always-on"` } VHosts VHostConfig } @@ -340,6 +486,8 @@ type Config struct { isupport isupport.List IPLimits connection_limits.LimiterConfig `yaml:"ip-limits"` Cloaks cloaks.CloakConfig `yaml:"ip-cloaking"` + SecureNetDefs []string `yaml:"secure-nets"` + secureNets []net.IPNet supportedCaps *caps.Set capValues caps.Values Casemapping Casemapping @@ -356,6 +504,14 @@ type Config struct { Datastore struct { Path string AutoUpgrade bool + MySQL struct { + Enabled bool + Host string + Port int + User string + Password string + HistoryDatabase string `yaml:"history-database"` + } } Accounts AccountConfig @@ -395,6 +551,18 @@ type Config struct { AutoresizeWindow time.Duration `yaml:"autoresize-window"` AutoreplayOnJoin int `yaml:"autoreplay-on-join"` ChathistoryMax int `yaml:"chathistory-maxmessages"` + ZNCMax int `yaml:"znc-maxmessages"` + Restrictions struct { + ExpireTime time.Duration `yaml:"expire-time"` + EnforceRegistrationDate bool `yaml:"enforce-registration-date"` + GracePeriod time.Duration `yaml:"grace-period"` + } + Persistent struct { + Enabled bool + UnregisteredChannels bool `yaml:"unregistered-channels"` + RegisteredChannels PersistentStatus `yaml:"registered-channels"` + DirectMessages PersistentStatus `yaml:"direct-messages"` + } } Filename string @@ -717,7 +885,10 @@ func LoadConfig(filename string) (config *Config, err error) { } if !config.Accounts.Bouncer.Enabled { + config.Accounts.Bouncer.AlwaysOn = PersistentDisabled config.Server.supportedCaps.Disable(caps.Bouncer) + } else if config.Accounts.Bouncer.AlwaysOn >= PersistentOptOut { + config.Accounts.Bouncer.AllowedByDefault = true } var newLogConfigs []logger.LoggingConfig @@ -786,6 +957,11 @@ func LoadConfig(filename string) (config *Config, err error) { return nil, fmt.Errorf("Could not parse proxy-allowed-from nets: %v", err.Error()) } + config.Server.secureNets, err = utils.ParseNetList(config.Server.SecureNetDefs) + if err != nil { + return nil, fmt.Errorf("Could not parse secure-nets: %v\n", err.Error()) + } + rawRegexp := config.Accounts.VHosts.ValidRegexpRaw if rawRegexp != "" { regexp, err := regexp.Compile(rawRegexp) @@ -882,6 +1058,16 @@ func LoadConfig(filename string) (config *Config, err error) { config.History.ClientLength = 0 } + if !config.History.Enabled || !config.History.Persistent.Enabled { + config.History.Persistent.UnregisteredChannels = false + config.History.Persistent.RegisteredChannels = PersistentDisabled + config.History.Persistent.DirectMessages = PersistentDisabled + } + + if config.History.ZNCMax == 0 { + config.History.ZNCMax = config.History.ChathistoryMax + } + config.Server.Cloaks.Initialize() if config.Server.Cloaks.Enabled { if config.Server.Cloaks.Secret == "" || config.Server.Cloaks.Secret == "siaELnk6Kaeo65K3RCrwJjlWaZ-Bt3WuZ2L8MXLbNb4" { diff --git a/irc/database.go b/irc/database.go index 12c9b83d..e23a6a77 100644 --- a/irc/database.go +++ b/irc/database.go @@ -36,22 +36,6 @@ type SchemaChange struct { // maps an initial version to a schema change capable of upgrading it var schemaChanges map[string]SchemaChange -type incompatibleSchemaError struct { - currentVersion string - requiredVersion string -} - -func IncompatibleSchemaError(currentVersion string) (result *incompatibleSchemaError) { - return &incompatibleSchemaError{ - currentVersion: currentVersion, - requiredVersion: latestDbSchema, - } -} - -func (err *incompatibleSchemaError) Error() string { - return fmt.Sprintf("Database requires update. Expected schema v%s, got v%s", err.requiredVersion, err.currentVersion) -} - // InitDB creates the database, implementing the `oragono initdb` command. func InitDB(path string) { _, err := os.Stat(path) @@ -129,7 +113,7 @@ func openDatabaseInternal(config *Config, allowAutoupgrade bool) (db *buntdb.DB, // successful autoupgrade, let's try this again: return openDatabaseInternal(config, false) } else { - err = IncompatibleSchemaError(version) + err = &utils.IncompatibleSchemaError{CurrentVersion: version, RequiredVersion: latestDbSchema} return } } @@ -173,7 +157,7 @@ func UpgradeDB(config *Config) (err error) { break } // unable to upgrade to the desired version, roll back - return IncompatibleSchemaError(version) + return &utils.IncompatibleSchemaError{CurrentVersion: version, RequiredVersion: latestDbSchema} } log.Println("attempting to update schema from version " + version) err := change.Changer(config, tx) diff --git a/irc/errors.go b/irc/errors.go index e90f86eb..de1e5b91 100644 --- a/irc/errors.go +++ b/irc/errors.go @@ -33,6 +33,7 @@ var ( errChannelNotOwnedByAccount = errors.New("Channel not owned by the specified account") errChannelTransferNotOffered = errors.New(`You weren't offered ownership of that channel`) errChannelAlreadyRegistered = errors.New("Channel is already registered") + errChannelNotRegistered = errors.New("Channel is not registered") errChannelNameInUse = errors.New(`Channel name in use`) errInvalidChannelName = errors.New(`Invalid channel name`) errMonitorLimitExceeded = errors.New("Monitor limit exceeded") @@ -40,6 +41,7 @@ var ( errNicknameInvalid = errors.New("invalid nickname") errNicknameInUse = errors.New("nickname in use") errNicknameReserved = errors.New("nickname is reserved") + errCantChangeNick = errors.New(`Always-on clients can't change nicknames`) errNoExistingBan = errors.New("Ban does not exist") errNoSuchChannel = errors.New(`No such channel`) errChannelPurged = errors.New(`This channel was purged by the server operators and cannot be used`) diff --git a/irc/gateways.go b/irc/gateways.go index 98b68978..3dec778c 100644 --- a/irc/gateways.go +++ b/irc/gateways.go @@ -61,7 +61,7 @@ func (wc *webircConfig) Populate() (err error) { func (client *Client) ApplyProxiedIP(session *Session, proxiedIP string, tls bool) (err error, quitMsg string) { // PROXY and WEBIRC are never accepted from a Tor listener, even if the address itself // is whitelisted: - if client.isTor { + if session.isTor { return errBadProxyLine, "" } diff --git a/irc/getters.go b/irc/getters.go index 6bbd6ed8..b15bce1f 100644 --- a/irc/getters.go +++ b/irc/getters.go @@ -91,21 +91,27 @@ func (client *Client) AllSessionData(currentSession *Session) (data []SessionDat return } -func (client *Client) AddSession(session *Session) (success bool) { +func (client *Client) AddSession(session *Session) (success bool, numSessions int, lastSignoff time.Time) { client.stateMutex.Lock() defer client.stateMutex.Unlock() // client may be dying and ineligible to receive another session if client.destroyed { - return false + return } // success, attach the new session to the client session.client = client newSessions := make([]*Session, len(client.sessions)+1) copy(newSessions, client.sessions) newSessions[len(newSessions)-1] = session + if len(client.sessions) == 0 && client.accountSettings.AutoreplayMissed { + // n.b. this is only possible if client is persistent and remained + // on the server with no sessions: + lastSignoff = client.lastSignoff + client.lastSignoff = time.Time{} + } client.sessions = newSessions - return true + return true, len(client.sessions), lastSignoff } func (client *Client) removeSession(session *Session) (success bool, length int) { @@ -189,6 +195,13 @@ func (client *Client) SetExitedSnomaskSent() { client.stateMutex.Unlock() } +func (client *Client) AlwaysOn() (alwaysOn bool) { + client.stateMutex.Lock() + alwaysOn = client.alwaysOn + client.stateMutex.Unlock() + return +} + // uniqueIdentifiers returns the strings for which the server enforces per-client // uniqueness/ownership; no two clients can have colliding casefolded nicks or // skeletons. @@ -264,23 +277,41 @@ func (client *Client) AccountName() string { return client.accountName } -func (client *Client) SetAccountName(account string) (changed bool) { - var casefoldedAccount string - var err error - if account != "" { - if casefoldedAccount, err = CasefoldName(account); err != nil { - return - } - } - +func (client *Client) Login(account ClientAccount) { + alwaysOn := persistenceEnabled(client.server.Config().Accounts.Bouncer.AlwaysOn, account.Settings.AlwaysOn) client.stateMutex.Lock() defer client.stateMutex.Unlock() - changed = client.account != casefoldedAccount - client.account = casefoldedAccount - client.accountName = account + client.account = account.NameCasefolded + client.accountName = account.Name + client.accountSettings = account.Settings + // check `registered` to avoid incorrectly marking a temporary (pre-reattach), + // SASL'ing client as always-on + if client.registered { + client.alwaysOn = alwaysOn + } + client.accountRegDate = account.RegisteredAt return } +func (client *Client) historyCutoff() (cutoff time.Time) { + client.stateMutex.Lock() + if client.account != "" { + cutoff = client.accountRegDate + } else { + cutoff = client.ctime + } + client.stateMutex.Unlock() + return +} + +func (client *Client) Logout() { + client.stateMutex.Lock() + client.account = "" + client.accountName = "" + client.alwaysOn = false + client.stateMutex.Unlock() +} + func (client *Client) AccountSettings() (result AccountSettings) { client.stateMutex.RLock() result = client.accountSettings @@ -289,8 +320,12 @@ func (client *Client) AccountSettings() (result AccountSettings) { } func (client *Client) SetAccountSettings(settings AccountSettings) { + alwaysOn := persistenceEnabled(client.server.Config().Accounts.Bouncer.AlwaysOn, settings.AlwaysOn) client.stateMutex.Lock() client.accountSettings = settings + if client.registered { + client.alwaysOn = alwaysOn + } client.stateMutex.Unlock() } @@ -309,11 +344,17 @@ func (client *Client) SetLanguages(languages []string) { func (client *Client) HasMode(mode modes.Mode) bool { // client.flags has its own synch - return client.flags.HasMode(mode) + return client.modes.HasMode(mode) } func (client *Client) SetMode(mode modes.Mode, on bool) bool { - return client.flags.SetMode(mode, on) + return client.modes.SetMode(mode, on) +} + +func (client *Client) SetRealname(realname string) { + client.stateMutex.Lock() + client.realname = realname + client.stateMutex.Unlock() } func (client *Client) Channels() (result []*Channel) { @@ -410,3 +451,17 @@ func (channel *Channel) HighestUserMode(client *Client) (result modes.Mode) { channel.stateMutex.RUnlock() return clientModes.HighestChannelUserMode() } + +func (channel *Channel) Settings() (result ChannelSettings) { + channel.stateMutex.RLock() + result = channel.settings + channel.stateMutex.RUnlock() + return result +} + +func (channel *Channel) SetSettings(settings ChannelSettings) { + channel.stateMutex.Lock() + channel.settings = settings + channel.stateMutex.Unlock() + channel.MarkDirty(IncludeSettings) +} diff --git a/irc/handlers.go b/irc/handlers.go index 8f3827b9..aa4b3457 100644 --- a/irc/handlers.go +++ b/irc/handlers.go @@ -531,64 +531,49 @@ func capHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Respo // CHATHISTORY BETWEEN [] // e.g., CHATHISTORY #ircv3 BETWEEN timestamp=YYYY-MM-DDThh:mm:ss.sssZ timestamp=YYYY-MM-DDThh:mm:ss.sssZ + 100 func chathistoryHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) (exiting bool) { - config := server.Config() - var items []history.Item - success := false - var hist *history.Buffer + unknown_command := false + var target string var channel *Channel + var sequence history.Sequence + var err error defer func() { // successful responses are sent as a chathistory or history batch - if success && 0 < len(items) { - if channel == nil { - client.replayPrivmsgHistory(rb, items, true) - } else { + if err == nil && 0 < len(items) { + if channel != nil { channel.replayHistoryItems(rb, items, false) + } else { + client.replayPrivmsgHistory(rb, items, target, true) } return } // errors are sent either without a batch, or in a draft/labeled-response batch as usual - // TODO: send `WARN CHATHISTORY MAX_MESSAGES_EXCEEDED` when appropriate - if hist == nil { - rb.Add(nil, server.name, "ERR", "CHATHISTORY", "NO_SUCH_CHANNEL") - } else if len(items) == 0 { - rb.Add(nil, server.name, "ERR", "CHATHISTORY", "NO_TEXT_TO_SEND") - } else if !success { - rb.Add(nil, server.name, "ERR", "CHATHISTORY", "NEED_MORE_PARAMS") + if unknown_command { + rb.Add(nil, server.name, "FAIL", "CHATHISTORY", "UNKNOWN_COMMAND", utils.SafeErrorParam(msg.Params[0]), client.t("Unknown command")) + } else if err == utils.ErrInvalidParams { + rb.Add(nil, server.name, "FAIL", "CHATHISTORY", "INVALID_PARAMETERS", msg.Params[0], client.t("Invalid parameters")) + } else if err != nil { + rb.Add(nil, server.name, "FAIL", "CHATHISTORY", "MESSAGE_ERROR", msg.Params[0], client.t("Messages could not be retrieved")) + } else if sequence == nil { + rb.Add(nil, server.name, "FAIL", "CHATHISTORY", "NO_SUCH_CHANNEL", utils.SafeErrorParam(msg.Params[1]), client.t("No such channel")) } }() - target := msg.Params[0] - channel = server.channels.Get(target) - if channel != nil && channel.hasClient(client) { - // "If [...] the user does not have permission to view the requested content, [...] - // NO_SUCH_CHANNEL SHOULD be returned" - hist = &channel.history - } else { - targetClient := server.clients.Get(target) - if targetClient != nil { - myAccount := client.Account() - targetAccount := targetClient.Account() - if myAccount != "" && targetAccount != "" && myAccount == targetAccount { - hist = &targetClient.history - } - } - } - if hist == nil { + config := server.Config() + maxChathistoryLimit := config.History.ChathistoryMax + if maxChathistoryLimit == 0 { return } - preposition := strings.ToLower(msg.Params[1]) - parseQueryParam := func(param string) (msgid string, timestamp time.Time, err error) { - err = errInvalidParams + err = utils.ErrInvalidParams pieces := strings.SplitN(param, "=", 2) if len(pieces) < 2 { return } identifier, value := strings.ToLower(pieces[0]), pieces[1] - if identifier == "id" { + if identifier == "msgid" { msgid, err = value, nil return } else if identifier == "timestamp" { @@ -598,10 +583,6 @@ func chathistoryHandler(server *Server, client *Client, msg ircmsg.IrcMessage, r return } - maxChathistoryLimit := config.History.ChathistoryMax - if maxChathistoryLimit == 0 { - return - } parseHistoryLimit := func(paramIndex int) (limit int) { if len(msg.Params) < (paramIndex + 1) { return maxChathistoryLimit @@ -613,140 +594,74 @@ func chathistoryHandler(server *Server, client *Client, msg ircmsg.IrcMessage, r return } - // TODO: as currently implemented, almost all of thes queries are worst-case O(n) - // in the number of stored history entries. Every one of them can be made O(1) - // if necessary, without too much difficulty. Some ideas: - // * Ensure that the ring buffer is sorted by time, enabling binary search for times - // * Maintain a map from msgid to position in the ring buffer - - if preposition == "between" { - if len(msg.Params) >= 5 { - startMsgid, startTimestamp, startErr := parseQueryParam(msg.Params[2]) - endMsgid, endTimestamp, endErr := parseQueryParam(msg.Params[3]) - ascending := msg.Params[4] == "+" - limit := parseHistoryLimit(5) - if startErr != nil || endErr != nil { - success = false - } else if startMsgid != "" && endMsgid != "" { - inInterval := false - matches := func(item history.Item) (result bool) { - result = inInterval - if item.HasMsgid(startMsgid) { - if ascending { - inInterval = true - } else { - inInterval = false - return false // interval is exclusive - } - } else if item.HasMsgid(endMsgid) { - if ascending { - inInterval = false - return false - } else { - inInterval = true - } - } - return - } - items = hist.Match(matches, ascending, limit) - success = true - } else if !startTimestamp.IsZero() && !endTimestamp.IsZero() { - items, _ = hist.Between(startTimestamp, endTimestamp, ascending, limit) - if !ascending { - history.Reverse(items) - } - success = true - } - // else: mismatched params, success = false, fail - } + preposition := strings.ToLower(msg.Params[0]) + target = msg.Params[1] + channel, sequence, err = server.GetHistorySequence(nil, client, target) + if err != nil || sequence == nil { return } - // before, after, latest, around - queryParam := msg.Params[2] - msgid, timestamp, err := parseQueryParam(queryParam) - limit := parseHistoryLimit(3) - before := false - switch preposition { - case "before": - before = true - fallthrough - case "after": - var matches history.Predicate - if err != nil { - break - } else if msgid != "" { - inInterval := false - matches = func(item history.Item) (result bool) { - result = inInterval - if item.HasMsgid(msgid) { - inInterval = true - } - return - } - } else { - matches = func(item history.Item) bool { - return before == item.Message.Time.Before(timestamp) - } - } - items = hist.Match(matches, !before, limit) - success = true - case "latest": - if queryParam == "*" { - items = hist.Latest(limit) - } else if err != nil { - break - } else { - var matches history.Predicate - if msgid != "" { - shouldStop := false - matches = func(item history.Item) bool { - if shouldStop { - return false - } - shouldStop = item.HasMsgid(msgid) - return !shouldStop - } - } else { - matches = func(item history.Item) bool { - return item.Message.Time.After(timestamp) - } - } - items = hist.Match(matches, false, limit) - } - success = true - case "around": - if err != nil { - break - } - var initialMatcher history.Predicate - if msgid != "" { - inInterval := false - initialMatcher = func(item history.Item) (result bool) { - if inInterval { - return true - } else { - inInterval = item.HasMsgid(msgid) - return inInterval - } - } - } else { - initialMatcher = func(item history.Item) (result bool) { - return item.Message.Time.Before(timestamp) - } - } - var halfLimit int - halfLimit = (limit + 1) / 2 - firstPass := hist.Match(initialMatcher, false, halfLimit) - if len(firstPass) > 0 { - timeWindowStart := firstPass[0].Message.Time - items = hist.Match(func(item history.Item) bool { - return item.Message.Time.Equal(timeWindowStart) || item.Message.Time.After(timeWindowStart) - }, true, limit) - } - success = true + roundUp := func(endpoint time.Time) (result time.Time) { + return endpoint.Truncate(time.Millisecond).Add(time.Millisecond) } + var start, end history.Selector + var limit int + switch preposition { + case "between": + start.Msgid, start.Time, err = parseQueryParam(msg.Params[2]) + if err != nil { + return + } + end.Msgid, end.Time, err = parseQueryParam(msg.Params[3]) + if err != nil { + return + } + // XXX preserve the ordering of the two parameters, since we might be going backwards, + // but round up the chronologically first one, whichever it is, to make it exclusive + if !start.Time.IsZero() && !end.Time.IsZero() { + if start.Time.Before(end.Time) { + start.Time = roundUp(start.Time) + } else { + end.Time = roundUp(end.Time) + } + } + limit = parseHistoryLimit(4) + case "before", "after", "around": + start.Msgid, start.Time, err = parseQueryParam(msg.Params[2]) + if err != nil { + return + } + if preposition == "after" && !start.Time.IsZero() { + start.Time = roundUp(start.Time) + } + if preposition == "before" { + end = start + start = history.Selector{} + } + limit = parseHistoryLimit(3) + case "latest": + if msg.Params[2] != "*" { + end.Msgid, end.Time, err = parseQueryParam(msg.Params[2]) + if err != nil { + return + } + if !end.Time.IsZero() { + end.Time = roundUp(end.Time) + } + start.Time = time.Now().UTC() + } + limit = parseHistoryLimit(3) + default: + unknown_command = true + return + } + + if preposition == "around" { + items, err = sequence.Around(start, limit) + } else { + items, _, err = sequence.Between(start, end, limit) + } return } @@ -1006,6 +921,7 @@ Get an explanation of , or "index" for a list of help topics.`), rb) // HISTORY [] // e.g., HISTORY #ubuntu 10 // HISTORY me 15 +// HISTORY #darwin 1h func historyHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool { config := server.Config() if !config.History.Enabled { @@ -1014,53 +930,55 @@ func historyHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *R } target := msg.Params[0] - var hist *history.Buffer - channel := server.channels.Get(target) - if channel != nil && channel.hasClient(client) { - hist = &channel.history - } else { - if strings.ToLower(target) == "me" { - hist = &client.history - } else { - targetClient := server.clients.Get(target) - if targetClient != nil { - myAccount, targetAccount := client.Account(), targetClient.Account() - if myAccount != "" && targetAccount != "" && myAccount == targetAccount { - hist = &targetClient.history - } + if strings.ToLower(target) == "me" { + target = "*" + } + channel, sequence, err := server.GetHistorySequence(nil, client, target) + + if sequence == nil || err != nil { + // whatever + rb.Add(nil, server.name, ERR_NOSUCHCHANNEL, client.Nick(), utils.SafeErrorParam(target), client.t("No such channel")) + return false + } + + var duration time.Duration + maxChathistoryLimit := config.History.ChathistoryMax + limit := 100 + if maxChathistoryLimit < limit { + limit = maxChathistoryLimit + } + if len(msg.Params) > 1 { + providedLimit, err := strconv.Atoi(msg.Params[1]) + if err == nil && providedLimit != 0 { + limit = providedLimit + if maxChathistoryLimit < limit { + limit = maxChathistoryLimit + } + } else if err != nil { + duration, err = time.ParseDuration(msg.Params[1]) + if err == nil { + limit = maxChathistoryLimit } } } - if hist == nil { - if channel == nil { - rb.Add(nil, server.name, ERR_NOSUCHCHANNEL, client.Nick(), utils.SafeErrorParam(target), client.t("No such channel")) - } else { - rb.Add(nil, server.name, ERR_NOTONCHANNEL, client.Nick(), target, client.t("You're not on that channel")) - } - return false - } - - limit := 10 - maxChathistoryLimit := config.History.ChathistoryMax - if len(msg.Params) > 1 { - providedLimit, err := strconv.Atoi(msg.Params[1]) - if providedLimit > maxChathistoryLimit { - providedLimit = maxChathistoryLimit - } - if err == nil && providedLimit != 0 { - limit = providedLimit - } - } - - items := hist.Latest(limit) - - if channel != nil { - channel.replayHistoryItems(rb, items, false) + var items []history.Item + if duration == 0 { + items, _, err = sequence.Between(history.Selector{}, history.Selector{}, limit) } else { - client.replayPrivmsgHistory(rb, items, true) + now := time.Now().UTC() + start := history.Selector{Time: now} + end := history.Selector{Time: now.Add(-duration)} + items, _, err = sequence.Between(start, end, limit) } + if err == nil { + if channel != nil { + channel.replayHistoryItems(rb, items, false) + } else { + client.replayPrivmsgHistory(rb, items, "", true) + } + } return false } @@ -1944,7 +1862,7 @@ func messageHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *R return false } - if client.isTor && utils.IsRestrictedCTCPMessage(message) { + if rb.session.isTor && utils.IsRestrictedCTCPMessage(message) { // note that error replies are never sent for NOTICE if histType != history.Notice { rb.Notice(client.t("CTCP messages are disabled over Tor")) @@ -2001,21 +1919,22 @@ func dispatchMessageToTarget(client *Client, tags map[string]string, histType hi } return } - tnick := user.Nick() + tDetails := user.Details() + tnick := tDetails.nick - nickMaskString := client.NickMaskString() - accountName := client.AccountName() + details := client.Details() + nickMaskString := details.nickMask + accountName := details.accountName // restrict messages appropriately when +R is set // intentionally make the sending user think the message went through fine - allowedPlusR := !user.HasMode(modes.RegisteredOnly) || client.LoggedIntoAccount() - allowedTor := !user.isTor || !message.IsRestrictedCTCPMessage() - if allowedPlusR && allowedTor { + allowedPlusR := !user.HasMode(modes.RegisteredOnly) || details.account != "" + if allowedPlusR { for _, session := range user.Sessions() { hasTagsCap := session.capabilities.Has(caps.MessageTags) // don't send TAGMSG at all if they don't have the tags cap if histType == history.Tagmsg && hasTagsCap { session.sendFromClientInternal(false, message.Time, message.Msgid, nickMaskString, accountName, tags, command, tnick) - } else if histType != history.Tagmsg { + } else if histType != history.Tagmsg && !(session.isTor && message.IsRestrictedCTCPMessage()) { tagsToSend := tags if !hasTagsCap { tagsToSend = nil @@ -2053,17 +1972,37 @@ func dispatchMessageToTarget(client *Client, tags map[string]string, histType hi rb.Add(nil, server.name, RPL_AWAY, client.Nick(), tnick, user.AwayMessage()) } + config := server.Config() + if !config.History.Enabled { + return + } item := history.Item{ Type: histType, Message: message, Nick: nickMaskString, AccountName: accountName, + Tags: tags, + } + if !item.IsStorable() { + return + } + targetedItem := item + targetedItem.Params[0] = tnick + cPersistent, cEphemeral := client.historyStatus(config) + tPersistent, tEphemeral := user.historyStatus(config) + // add to ephemeral history + if cEphemeral { + targetedItem.CfCorrespondent = tDetails.nickCasefolded + client.history.Add(targetedItem) + } + if tEphemeral { + item.CfCorrespondent = details.nickCasefolded + user.history.Add(item) + } + if cPersistent || tPersistent { + item.CfCorrespondent = "" + server.historyDB.AddDirectMessage(details.nickCasefolded, user.NickCasefolded(), cPersistent, tPersistent, targetedItem) } - // add to the target's history: - user.history.Add(item) - // add this to the client's history as well, recording the target: - item.Params[0] = tnick - client.history.Add(item) } } @@ -2375,11 +2314,7 @@ func sceneHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Res // SETNAME func setnameHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool { realname := msg.Params[0] - - client.stateMutex.Lock() - client.realname = realname - client.stateMutex.Unlock() - + client.SetRealname(realname) details := client.Details() // alert friends diff --git a/irc/help.go b/irc/help.go index 4f0a19da..6c9fa018 100644 --- a/irc/help.go +++ b/irc/help.go @@ -206,8 +206,9 @@ Get an explanation of , or "index" for a list of help topics.`, Replay message history. can be a channel name, "me" to replay direct message history, or a nickname to replay another client's direct message -history (they must be logged into the same account as you). At most [limit] -messages will be replayed.`, +history (they must be logged into the same account as you). [limit] can be +either an integer (the maximum number of messages to replay), or a time +duration like 10m or 1h (the time window within which to replay messages).`, }, "info": { text: `INFO diff --git a/irc/history/history.go b/irc/history/history.go index 6c71e104..b39b753a 100644 --- a/irc/history/history.go +++ b/irc/history/history.go @@ -6,7 +6,6 @@ package history import ( "github.com/oragono/oragono/irc/utils" "sync" - "sync/atomic" "time" ) @@ -43,9 +42,10 @@ type Item struct { // this is the uncasefolded account name, if there's no account it should be set to "*" AccountName string // for non-privmsg items, we may stuff some other data in here - Message utils.SplitMessage - Tags map[string]string - Params [1]string + Message utils.SplitMessage + Tags map[string]string + Params [1]string + CfCorrespondent string } // HasMsgid tests whether a message has the message id `msgid`. @@ -53,20 +53,30 @@ func (item *Item) HasMsgid(msgid string) bool { return item.Message.Msgid == msgid } -func (item *Item) isStorable() bool { - if item.Type == Tagmsg { +func (item *Item) IsStorable() bool { + switch item.Type { + case Tagmsg: for name := range item.Tags { if !transientTags[name] { return true } } return false // all tags were blacklisted - } else { + case Privmsg, Notice: + // don't store CTCP other than ACTION + return !item.Message.IsRestrictedCTCPMessage() + default: return true } } -type Predicate func(item Item) (matches bool) +type Predicate func(item *Item) (matches bool) + +func Reverse(results []Item) { + for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 { + results[i], results[j] = results[j], results[i] + } +} // Buffer is a ring buffer holding message/event history for a channel or user type Buffer struct { @@ -81,8 +91,6 @@ type Buffer struct { lastDiscarded time.Time - enabled uint32 - nowFunc func() time.Time } @@ -99,8 +107,6 @@ func (hist *Buffer) Initialize(size int, window time.Duration) { hist.window = window hist.maximumSize = size hist.nowFunc = time.Now - - hist.setEnabled(size) } // compute the initial size for the buffer, taking into account autoresize @@ -115,31 +121,8 @@ func (hist *Buffer) initialSize(size int, window time.Duration) (result int) { return } -func (hist *Buffer) setEnabled(size int) { - var enabled uint32 - if size != 0 { - enabled = 1 - } - atomic.StoreUint32(&hist.enabled, enabled) -} - -// Enabled returns whether the buffer is currently storing messages -// (a disabled buffer blackholes everything it sees) -func (list *Buffer) Enabled() bool { - return atomic.LoadUint32(&list.enabled) != 0 -} - // Add adds a history item to the buffer func (list *Buffer) Add(item Item) { - // fast path without a lock acquisition for when we are not storing history - if !list.Enabled() { - return - } - - if !item.isStorable() { - return - } - if item.Message.Time.IsZero() { item.Message.Time = time.Now().UTC() } @@ -147,6 +130,10 @@ func (list *Buffer) Add(item Item) { list.Lock() defer list.Unlock() + if len(list.buffer) == 0 { + return + } + list.maybeExpand() var pos int @@ -170,55 +157,100 @@ func (list *Buffer) Add(item Item) { list.buffer[pos] = item } -// Reverse reverses an []Item, in-place. -func Reverse(results []Item) { - for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 { - results[i], results[j] = results[j], results[i] +func (list *Buffer) lookup(msgid string) (result Item, found bool) { + predicate := func(item *Item) bool { + return item.HasMsgid(msgid) } + results := list.matchInternal(predicate, false, 1) + if len(results) != 0 { + return results[0], true + } + return } // Between returns all history items with a time `after` <= time <= `before`, // with an indication of whether the results are complete or are missing items // because some of that period was discarded. A zero value of `before` is considered // higher than all other times. -func (list *Buffer) Between(after, before time.Time, ascending bool, limit int) (results []Item, complete bool) { - if !list.Enabled() { - return - } +func (list *Buffer) betweenHelper(start, end Selector, cutoff time.Time, pred Predicate, limit int) (results []Item, complete bool, err error) { + var ascending bool + + defer func() { + if !ascending { + Reverse(results) + } + }() list.RLock() defer list.RUnlock() + if len(list.buffer) == 0 { + return + } + + after := start.Time + if start.Msgid != "" { + item, found := list.lookup(start.Msgid) + if !found { + return + } + after = item.Message.Time + } + before := end.Time + if end.Msgid != "" { + item, found := list.lookup(end.Msgid) + if !found { + return + } + before = item.Message.Time + } + + after, before, ascending = MinMaxAsc(after, before, cutoff) + complete = after.Equal(list.lastDiscarded) || after.After(list.lastDiscarded) - satisfies := func(item Item) bool { - return (after.IsZero() || item.Message.Time.After(after)) && (before.IsZero() || item.Message.Time.Before(before)) + satisfies := func(item *Item) bool { + return (after.IsZero() || item.Message.Time.After(after)) && + (before.IsZero() || item.Message.Time.Before(before)) && + (pred == nil || pred(item)) } - return list.matchInternal(satisfies, ascending, limit), complete + return list.matchInternal(satisfies, ascending, limit), complete, nil } -// Match returns all history items such that `predicate` returns true for them. -// Items are considered in reverse insertion order if `ascending` is false, or -// in insertion order if `ascending` is true, up to a total of `limit` matches -// if `limit` > 0 (unlimited otherwise). -// `predicate` MAY be a closure that maintains its own state across invocations; -// it MUST NOT acquire any locks or otherwise do anything weird. -// Results are always returned in insertion order. -func (list *Buffer) Match(predicate Predicate, ascending bool, limit int) (results []Item) { - if !list.Enabled() { - return +// implements history.Sequence, emulating a single history buffer (for a channel, +// a single user's DMs, or a DM conversation) +type bufferSequence struct { + list *Buffer + pred Predicate + cutoff time.Time +} + +func (list *Buffer) MakeSequence(correspondent string, cutoff time.Time) Sequence { + var pred Predicate + if correspondent != "" { + pred = func(item *Item) bool { + return item.CfCorrespondent == correspondent + } } + return &bufferSequence{ + list: list, + pred: pred, + cutoff: cutoff, + } +} - list.RLock() - defer list.RUnlock() +func (seq *bufferSequence) Between(start, end Selector, limit int) (results []Item, complete bool, err error) { + return seq.list.betweenHelper(start, end, seq.cutoff, seq.pred, limit) +} - return list.matchInternal(predicate, ascending, limit) +func (seq *bufferSequence) Around(start Selector, limit int) (results []Item, err error) { + return GenericAround(seq, start, limit) } // you must be holding the read lock to call this func (list *Buffer) matchInternal(predicate Predicate, ascending bool, limit int) (results []Item) { - if list.start == -1 { + if list.start == -1 || len(list.buffer) == 0 { return } @@ -232,7 +264,7 @@ func (list *Buffer) matchInternal(predicate Predicate, ascending bool, limit int } for { - if predicate(list.buffer[pos]) { + if predicate(&list.buffer[pos]) { results = append(results, list.buffer[pos]) } if pos == stop || (limit != 0 && len(results) == limit) { @@ -245,18 +277,14 @@ func (list *Buffer) matchInternal(predicate Predicate, ascending bool, limit int } } - // TODO sort by time instead? - if !ascending { - Reverse(results) - } return } -// Latest returns the items most recently added, up to `limit`. If `limit` is 0, +// latest returns the items most recently added, up to `limit`. If `limit` is 0, // it returns all items. -func (list *Buffer) Latest(limit int) (results []Item) { - matchAll := func(item Item) bool { return true } - return list.Match(matchAll, false, limit) +func (list *Buffer) latest(limit int) (results []Item) { + results, _, _ = list.betweenHelper(Selector{}, Selector{}, time.Time{}, nil, limit) + return } // LastDiscarded returns the latest time of any entry that was evicted @@ -355,8 +383,6 @@ func (list *Buffer) Resize(maximumSize int, window time.Duration) { func (list *Buffer) resize(size int) { newbuffer := make([]Item, size) - list.setEnabled(size) - if list.start == -1 { // indices are already correct and nothing needs to be copied } else if size == 0 { diff --git a/irc/history/history_test.go b/irc/history/history_test.go index e50cfb28..514827d3 100644 --- a/irc/history/history_test.go +++ b/irc/history/history_test.go @@ -14,19 +14,21 @@ const ( timeFormat = "2006-01-02 15:04:05Z" ) +func betweenTimestamps(buf *Buffer, start, end time.Time, limit int) (result []Item, complete bool) { + result, complete, _ = buf.betweenHelper(Selector{Time: start}, Selector{Time: end}, time.Time{}, nil, limit) + return +} + func TestEmptyBuffer(t *testing.T) { pastTime := easyParse(timeFormat) buf := NewHistoryBuffer(0, 0) - if buf.Enabled() { - t.Error("the buffer of size 0 must be considered disabled") - } buf.Add(Item{ Nick: "testnick", }) - since, complete := buf.Between(pastTime, time.Now(), false, 0) + since, complete := betweenTimestamps(buf, pastTime, time.Now(), 0) if len(since) != 0 { t.Error("shouldn't be able to add to disabled buf") } @@ -35,16 +37,13 @@ func TestEmptyBuffer(t *testing.T) { } buf.Resize(1, 0) - if !buf.Enabled() { - t.Error("the buffer of size 1 must be considered enabled") - } - since, complete = buf.Between(pastTime, time.Now(), false, 0) + since, complete = betweenTimestamps(buf, pastTime, time.Now(), 0) assertEqual(complete, true, t) assertEqual(len(since), 0, t) buf.Add(Item{ Nick: "testnick", }) - since, complete = buf.Between(pastTime, time.Now(), false, 0) + since, complete = betweenTimestamps(buf, pastTime, time.Now(), 0) if len(since) != 1 { t.Error("should be able to store items in a nonempty buffer") } @@ -58,7 +57,7 @@ func TestEmptyBuffer(t *testing.T) { buf.Add(Item{ Nick: "testnick2", }) - since, complete = buf.Between(pastTime, time.Now(), false, 0) + since, complete = betweenTimestamps(buf, pastTime, time.Now(), 0) if len(since) != 1 { t.Error("expect exactly 1 item") } @@ -68,8 +67,7 @@ func TestEmptyBuffer(t *testing.T) { if since[0].Nick != "testnick2" { t.Error("retrieved junk data") } - matchAll := func(item Item) bool { return true } - assertEqual(toNicks(buf.Match(matchAll, false, 0)), []string{"testnick2"}, t) + assertEqual(toNicks(buf.latest(0)), []string{"testnick2"}, t) } func toNicks(items []Item) (result []string) { @@ -110,27 +108,27 @@ func TestBuffer(t *testing.T) { buf.Add(easyItem("testnick2", "2006-01-03 15:04:05Z")) - since, complete := buf.Between(start, time.Now(), false, 0) + since, complete := betweenTimestamps(buf, start, time.Now(), 0) assertEqual(complete, true, t) assertEqual(toNicks(since), []string{"testnick0", "testnick1", "testnick2"}, t) // add another item, evicting the first buf.Add(easyItem("testnick3", "2006-01-04 15:04:05Z")) - since, complete = buf.Between(start, time.Now(), false, 0) + since, complete = betweenTimestamps(buf, start, time.Now(), 0) assertEqual(complete, false, t) assertEqual(toNicks(since), []string{"testnick1", "testnick2", "testnick3"}, t) // now exclude the time of the discarded entry; results should be complete again - since, complete = buf.Between(easyParse("2006-01-02 00:00:00Z"), time.Now(), false, 0) + since, complete = betweenTimestamps(buf, easyParse("2006-01-02 00:00:00Z"), time.Now(), 0) assertEqual(complete, true, t) assertEqual(toNicks(since), []string{"testnick1", "testnick2", "testnick3"}, t) - since, complete = buf.Between(easyParse("2006-01-02 00:00:00Z"), easyParse("2006-01-03 00:00:00Z"), false, 0) + since, complete = betweenTimestamps(buf, easyParse("2006-01-02 00:00:00Z"), easyParse("2006-01-03 00:00:00Z"), 0) assertEqual(complete, true, t) assertEqual(toNicks(since), []string{"testnick1"}, t) // shrink the buffer, cutting off testnick1 buf.Resize(2, 0) - since, complete = buf.Between(easyParse("2006-01-02 00:00:00Z"), time.Now(), false, 0) + since, complete = betweenTimestamps(buf, easyParse("2006-01-02 00:00:00Z"), time.Now(), 0) assertEqual(complete, false, t) assertEqual(toNicks(since), []string{"testnick2", "testnick3"}, t) @@ -138,18 +136,19 @@ func TestBuffer(t *testing.T) { buf.Add(easyItem("testnick4", "2006-01-05 15:04:05Z")) buf.Add(easyItem("testnick5", "2006-01-06 15:04:05Z")) buf.Add(easyItem("testnick6", "2006-01-07 15:04:05Z")) - since, complete = buf.Between(easyParse("2006-01-03 00:00:00Z"), time.Now(), false, 0) + since, complete = betweenTimestamps(buf, easyParse("2006-01-03 00:00:00Z"), time.Now(), 0) assertEqual(complete, true, t) assertEqual(toNicks(since), []string{"testnick2", "testnick3", "testnick4", "testnick5", "testnick6"}, t) // test ascending order - since, _ = buf.Between(easyParse("2006-01-03 00:00:00Z"), time.Now(), true, 2) + since, _ = betweenTimestamps(buf, easyParse("2006-01-03 00:00:00Z"), time.Time{}, 2) assertEqual(toNicks(since), []string{"testnick2", "testnick3"}, t) } func autoItem(id int, t time.Time) (result Item) { result.Message.Time = t result.Nick = strconv.Itoa(id) + result.Message.Msgid = result.Nick return } @@ -181,7 +180,7 @@ func TestAutoresize(t *testing.T) { now = now.Add(time.Minute * 10) id += 1 } - items := buf.Latest(0) + items := buf.latest(0) assertEqual(len(items), initialAutoSize, t) assertEqual(atoi(items[0].Nick), 40, t) assertEqual(atoi(items[len(items)-1].Nick), 71, t) @@ -195,7 +194,7 @@ func TestAutoresize(t *testing.T) { // ok, 5 items from the first batch are still in the 1-hour window; // we should overwrite until only those 5 are left, then start expanding // the buffer so that it retains those 5 and the 100 new items - items = buf.Latest(0) + items = buf.latest(0) assertEqual(len(items), 105, t) assertEqual(atoi(items[0].Nick), 67, t) assertEqual(atoi(items[len(items)-1].Nick), 171, t) @@ -207,7 +206,7 @@ func TestAutoresize(t *testing.T) { id += 1 } // should fill up to the maximum size of 128 and start overwriting - items = buf.Latest(0) + items = buf.latest(0) assertEqual(len(items), 128, t) assertEqual(atoi(items[0].Nick), 144, t) assertEqual(atoi(items[len(items)-1].Nick), 271, t) @@ -222,7 +221,7 @@ func TestEnabledByResize(t *testing.T) { buf.Resize(128, time.Hour) // add an item and test that it is stored and retrievable buf.Add(autoItem(0, now)) - items := buf.Latest(0) + items := buf.latest(0) assertEqual(len(items), 1, t) assertEqual(atoi(items[0].Nick), 0, t) } @@ -232,13 +231,13 @@ func TestDisabledByResize(t *testing.T) { // enabled autoresizing buffer buf := NewHistoryBuffer(128, time.Hour) buf.Add(autoItem(0, now)) - items := buf.Latest(0) + items := buf.latest(0) assertEqual(len(items), 1, t) assertEqual(atoi(items[0].Nick), 0, t) // disable as during a rehash, confirm that nothing can be retrieved buf.Resize(0, time.Hour) - items = buf.Latest(0) + items = buf.latest(0) assertEqual(len(items), 0, t) } @@ -252,3 +251,25 @@ func TestRoundUp(t *testing.T) { assertEqual(roundUpToPowerOfTwo(1025), 2048, t) assertEqual(roundUpToPowerOfTwo(269435457), 536870912, t) } + +func BenchmarkInsert(b *testing.B) { + buf := NewHistoryBuffer(1024, 0) + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf.Add(Item{}) + } +} + +func BenchmarkMatch(b *testing.B) { + buf := NewHistoryBuffer(1024, 0) + var now time.Time + for i := 0; i < 1024; i += 1 { + buf.Add(autoItem(i, now)) + now = now.Add(time.Second) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf.lookup("512") + } +} diff --git a/irc/history/queries.go b/irc/history/queries.go new file mode 100644 index 00000000..771a1df0 --- /dev/null +++ b/irc/history/queries.go @@ -0,0 +1,71 @@ +// Copyright (c) 2020 Shivaram Lingamneni +// released under the MIT license + +package history + +import ( + "time" +) + +// Selector represents a parameter to a CHATHISTORY command; +// at most one of Msgid or Time may be nonzero +type Selector struct { + Msgid string + Time time.Time +} + +// Sequence is an abstract sequence of history entries that can be queried; +// it encapsulates restrictions such as registration time cutoffs, or +// only looking at a single "query buffer" (DMs with a particular correspondent) +type Sequence interface { + Between(start, end Selector, limit int) (results []Item, complete bool, err error) + Around(start Selector, limit int) (results []Item, err error) +} + +// This is a bad, slow implementation of CHATHISTORY AROUND using the BETWEEN semantics +func GenericAround(seq Sequence, start Selector, limit int) (results []Item, err error) { + var halfLimit int + halfLimit = (limit + 1) / 2 + initialResults, _, err := seq.Between(Selector{}, start, halfLimit) + if err != nil { + return + } else if len(initialResults) == 0 { + // TODO: this fails if we're doing an AROUND on the first message in the buffer + // would be nice to fix this but whatever + return + } + newStart := Selector{Time: initialResults[0].Message.Time} + results, _, err = seq.Between(newStart, Selector{}, limit) + return +} + +// MinMaxAsc converts CHATHISTORY arguments into time intervals, handling the most +// general case (BETWEEN going forwards or backwards) natively and the other ordering +// queries (AFTER, BEFORE, LATEST) as special cases. +func MinMaxAsc(after, before, cutoff time.Time) (min, max time.Time, ascending bool) { + startIsZero, endIsZero := after.IsZero(), before.IsZero() + if !startIsZero && endIsZero { + // AFTER + ascending = true + } else if startIsZero && !endIsZero { + // BEFORE + ascending = false + } else if !startIsZero && !endIsZero { + if before.Before(after) { + // BETWEEN going backwards + before, after = after, before + ascending = false + } else { + // BETWEEN going forwards + ascending = true + } + } else if startIsZero && endIsZero { + // LATEST + ascending = false + } + if after.IsZero() || after.Before(cutoff) { + // this may result in an impossible query, which is fine + after = cutoff + } + return after, before, ascending +} diff --git a/irc/idletimer.go b/irc/idletimer.go index e4dc6a4f..fe7e7321 100644 --- a/irc/idletimer.go +++ b/irc/idletimer.go @@ -52,6 +52,7 @@ type IdleTimer struct { quitTimeout time.Duration state TimerState timer *time.Timer + lastTouch time.Time } // Initialize sets up an IdleTimer and starts counting idle time; @@ -61,9 +62,11 @@ func (it *IdleTimer) Initialize(session *Session) { it.registerTimeout = RegisterTimeout it.idleTimeout, it.quitTimeout = it.recomputeDurations() registered := session.client.Registered() + now := time.Now().UTC() it.Lock() defer it.Unlock() + it.lastTouch = now if registered { it.state = TimerActive } else { @@ -82,7 +85,7 @@ func (it *IdleTimer) recomputeDurations() (idleTimeout, quitTimeout time.Duratio } idleTimeout = DefaultIdleTimeout - if it.session.client.isTor { + if it.session.isTor { idleTimeout = TorIdleTimeout } @@ -92,10 +95,12 @@ func (it *IdleTimer) recomputeDurations() (idleTimeout, quitTimeout time.Duratio func (it *IdleTimer) Touch() { idleTimeout, quitTimeout := it.recomputeDurations() + now := time.Now().UTC() it.Lock() defer it.Unlock() it.idleTimeout, it.quitTimeout = idleTimeout, quitTimeout + it.lastTouch = now // a touch transitions TimerUnregistered or TimerIdle into TimerActive if it.state != TimerDead { it.state = TimerActive @@ -103,6 +108,13 @@ func (it *IdleTimer) Touch() { } } +func (it *IdleTimer) LastTouch() (result time.Time) { + it.Lock() + result = it.lastTouch + it.Unlock() + return +} + func (it *IdleTimer) processTimeout() { idleTimeout, quitTimeout := it.recomputeDurations() @@ -322,9 +334,6 @@ const ( // BrbDead is the state of a client after its timeout has expired; it will be removed // and therefore new sessions cannot be attached to it BrbDead - // BrbSticky allows a client to remain online without sessions, with no timeout. - // This is not used yet. - BrbSticky ) type BrbTimer struct { @@ -345,16 +354,16 @@ func (bt *BrbTimer) Initialize(client *Client) { // attempts to enable BRB for a client, returns whether it succeeded func (bt *BrbTimer) Enable() (success bool, duration time.Duration) { - if !bt.client.Registered() || bt.client.ResumeID() == "" { - return - } - // TODO make this configurable duration = ResumeableTotalTimeout bt.client.stateMutex.Lock() defer bt.client.stateMutex.Unlock() + if !bt.client.registered || bt.client.alwaysOn || bt.client.resumeID == "" { + return + } + switch bt.state { case BrbDisabled, BrbEnabled: bt.state = BrbEnabled @@ -366,8 +375,6 @@ func (bt *BrbTimer) Enable() (success bool, duration time.Duration) { bt.brbAt = time.Now().UTC() } success = true - case BrbSticky: - success = true default: // BrbDead success = false @@ -416,6 +423,10 @@ func (bt *BrbTimer) processTimeout() { bt.client.stateMutex.Lock() defer bt.client.stateMutex.Unlock() + if bt.client.alwaysOn { + return + } + switch bt.state { case BrbDisabled, BrbEnabled: if len(bt.client.sessions) == 0 { @@ -432,16 +443,3 @@ func (bt *BrbTimer) processTimeout() { } bt.resetTimeout() } - -// sets a client to be "sticky", i.e., indefinitely exempt from removal for -// lack of sessions -func (bt *BrbTimer) SetSticky() (success bool) { - bt.client.stateMutex.Lock() - defer bt.client.stateMutex.Unlock() - if bt.state != BrbDead { - success = true - bt.state = BrbSticky - } - bt.resetTimeout() - return -} diff --git a/irc/mysql/history.go b/irc/mysql/history.go new file mode 100644 index 00000000..5482b5e9 --- /dev/null +++ b/irc/mysql/history.go @@ -0,0 +1,535 @@ +package mysql + +import ( + "bytes" + "database/sql" + "fmt" + "runtime/debug" + "strconv" + "sync" + "time" + + _ "github.com/go-sql-driver/mysql" + "github.com/oragono/oragono/irc/history" + "github.com/oragono/oragono/irc/logger" + "github.com/oragono/oragono/irc/utils" +) + +const ( + // latest schema of the db + latestDbSchema = "1" + keySchemaVersion = "db.version" + cleanupRowLimit = 50 + cleanupPauseTime = 10 * time.Minute +) + +type MySQL struct { + db *sql.DB + logger *logger.Manager + + insertHistory *sql.Stmt + insertSequence *sql.Stmt + insertConversation *sql.Stmt + + stateMutex sync.Mutex + expireTime time.Duration +} + +func (mysql *MySQL) Initialize(logger *logger.Manager, expireTime time.Duration) { + mysql.logger = logger + mysql.expireTime = expireTime +} + +func (mysql *MySQL) SetExpireTime(expireTime time.Duration) { + mysql.stateMutex.Lock() + mysql.expireTime = expireTime + mysql.stateMutex.Unlock() +} + +func (mysql *MySQL) getExpireTime() (expireTime time.Duration) { + mysql.stateMutex.Lock() + expireTime = mysql.expireTime + mysql.stateMutex.Unlock() + return +} + +func (mysql *MySQL) Open(username, password, host string, port int, database string) (err error) { + // TODO: timeouts! + var address string + if port != 0 { + address = fmt.Sprintf("tcp(%s:%d)", host, port) + } + + mysql.db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@%s/%s", username, password, address, database)) + if err != nil { + return err + } + + err = mysql.fixSchemas() + if err != nil { + return err + } + + err = mysql.prepareStatements() + if err != nil { + return err + } + + go mysql.cleanupLoop() + + return nil +} + +func (mysql *MySQL) fixSchemas() (err error) { + _, err = mysql.db.Exec(`CREATE TABLE IF NOT EXISTS metadata ( + key_name VARCHAR(32) primary key, + value VARCHAR(32) NOT NULL + ) CHARSET=ascii COLLATE=ascii_bin;`) + if err != nil { + return err + } + + var schema string + err = mysql.db.QueryRow(`select value from metadata where key_name = ?;`, keySchemaVersion).Scan(&schema) + if err == sql.ErrNoRows { + err = mysql.createTables() + if err != nil { + return + } + _, err = mysql.db.Exec(`insert into metadata (key_name, value) values (?, ?);`, keySchemaVersion, latestDbSchema) + if err != nil { + return + } + } else if err == nil && schema != latestDbSchema { + // TODO figure out what to do about schema changes + return &utils.IncompatibleSchemaError{CurrentVersion: schema, RequiredVersion: latestDbSchema} + } else { + return err + } + + return nil +} + +func (mysql *MySQL) createTables() (err error) { + _, err = mysql.db.Exec(`CREATE TABLE history ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + data BLOB NOT NULL, + msgid BINARY(16) NOT NULL, + KEY (msgid(4)) + ) CHARSET=ascii COLLATE=ascii_bin;`) + if err != nil { + return err + } + + _, err = mysql.db.Exec(`CREATE TABLE sequence ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + target VARBINARY(64) NOT NULL, + nanotime BIGINT UNSIGNED NOT NULL, + history_id BIGINT NOT NULL, + KEY (target, nanotime), + KEY (history_id) + ) CHARSET=ascii COLLATE=ascii_bin;`) + if err != nil { + return err + } + + _, err = mysql.db.Exec(`CREATE TABLE conversations ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + lower_target VARBINARY(64) NOT NULL, + upper_target VARBINARY(64) NOT NULL, + nanotime BIGINT UNSIGNED NOT NULL, + history_id BIGINT NOT NULL, + KEY (lower_target, upper_target, nanotime), + KEY (history_id) + ) CHARSET=ascii COLLATE=ascii_bin;`) + if err != nil { + return err + } + + return nil +} + +func (mysql *MySQL) cleanupLoop() { + defer func() { + if r := recover(); r != nil { + mysql.logger.Error("mysql", + fmt.Sprintf("Panic in cleanup routine: %v\n%s", r, debug.Stack())) + time.Sleep(cleanupPauseTime) + go mysql.cleanupLoop() + } + }() + + for { + expireTime := mysql.getExpireTime() + if expireTime != 0 { + for { + startTime := time.Now() + rowsDeleted, err := mysql.doCleanup(expireTime) + elapsed := time.Now().Sub(startTime) + mysql.logError("error during row cleanup", err) + // keep going as long as we're accomplishing significant work + // (don't busy-wait on small numbers of rows expiring): + if rowsDeleted < (cleanupRowLimit / 10) { + break + } + // crude backpressure mechanism: if the database is slow, + // give it time to process other queries + time.Sleep(elapsed) + } + } + time.Sleep(cleanupPauseTime) + } +} + +func (mysql *MySQL) doCleanup(age time.Duration) (count int, err error) { + ids, maxNanotime, err := mysql.selectCleanupIDs(age) + if len(ids) == 0 { + mysql.logger.Debug("mysql", "found no rows to clean up") + return + } + + mysql.logger.Debug("mysql", fmt.Sprintf("deleting %d history rows, max age %s", len(ids), utils.NanoToTimestamp(maxNanotime))) + + // can't use ? binding for a variable number of arguments, build the IN clause manually + var inBuf bytes.Buffer + inBuf.WriteByte('(') + for i, id := range ids { + if i != 0 { + inBuf.WriteRune(',') + } + inBuf.WriteString(strconv.FormatInt(int64(id), 10)) + } + inBuf.WriteRune(')') + + _, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM conversations WHERE history_id in %s;`, inBuf.Bytes())) + if err != nil { + return + } + _, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM sequence WHERE history_id in %s;`, inBuf.Bytes())) + if err != nil { + return + } + _, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM history WHERE id in %s;`, inBuf.Bytes())) + if err != nil { + return + } + + count = len(ids) + return +} + +func (mysql *MySQL) selectCleanupIDs(age time.Duration) (ids []uint64, maxNanotime int64, err error) { + rows, err := mysql.db.Query(` + SELECT history.id, sequence.nanotime + FROM history + LEFT JOIN sequence ON history.id = sequence.history_id + ORDER BY history.id LIMIT ?;`, cleanupRowLimit) + if err != nil { + return + } + defer rows.Close() + + // a history ID may have 0-2 rows in sequence: 1 for a channel entry, + // 2 for a DM, 0 if the data is inconsistent. therefore, deduplicate + // and delete anything that doesn't have a sequence entry: + idset := make(map[uint64]struct{}, cleanupRowLimit) + threshold := time.Now().Add(-age).UnixNano() + for rows.Next() { + var id uint64 + var nanotime sql.NullInt64 + err = rows.Scan(&id, &nanotime) + if err != nil { + return + } + if !nanotime.Valid || nanotime.Int64 < threshold { + idset[id] = struct{}{} + if nanotime.Valid && nanotime.Int64 > maxNanotime { + maxNanotime = nanotime.Int64 + } + } + } + ids = make([]uint64, len(idset)) + i := 0 + for id := range idset { + ids[i] = id + i++ + } + return +} + +func (mysql *MySQL) prepareStatements() (err error) { + mysql.insertHistory, err = mysql.db.Prepare(`INSERT INTO history + (data, msgid) VALUES (?, ?);`) + if err != nil { + return + } + mysql.insertSequence, err = mysql.db.Prepare(`INSERT INTO sequence + (target, nanotime, history_id) VALUES (?, ?, ?);`) + if err != nil { + return + } + mysql.insertConversation, err = mysql.db.Prepare(`INSERT INTO conversations + (lower_target, upper_target, nanotime, history_id) VALUES (?, ?, ?, ?);`) + if err != nil { + return + } + + return +} + +func (mysql *MySQL) logError(context string, err error) (quit bool) { + if err != nil { + mysql.logger.Error("mysql", context, err.Error()) + return true + } + return false +} + +func (mysql *MySQL) AddChannelItem(target string, item history.Item) (err error) { + if mysql.db == nil { + return + } + + if target == "" { + return utils.ErrInvalidParams + } + + id, err := mysql.insertBase(item) + if err != nil { + return + } + + err = mysql.insertSequenceEntry(target, item.Message.Time, id) + return +} + +func (mysql *MySQL) insertSequenceEntry(target string, messageTime time.Time, id int64) (err error) { + _, err = mysql.insertSequence.Exec(target, messageTime.UnixNano(), id) + mysql.logError("could not insert sequence entry", err) + return +} + +func (mysql *MySQL) insertConversationEntry(sender, recipient string, messageTime time.Time, id int64) (err error) { + lower, higher := stringMinMax(sender, recipient) + _, err = mysql.insertConversation.Exec(lower, higher, messageTime.UnixNano(), id) + mysql.logError("could not insert conversations entry", err) + return +} + +func (mysql *MySQL) insertBase(item history.Item) (id int64, err error) { + value, err := marshalItem(&item) + if mysql.logError("could not marshal item", err) { + return + } + + msgidBytes, err := decodeMsgid(item.Message.Msgid) + if mysql.logError("could not decode msgid", err) { + return + } + + result, err := mysql.insertHistory.Exec(value, msgidBytes) + if mysql.logError("could not insert item", err) { + return + } + id, err = result.LastInsertId() + if mysql.logError("could not insert item", err) { + return + } + + return +} + +func stringMinMax(first, second string) (min, max string) { + if first < second { + return first, second + } else { + return second, first + } +} + +func (mysql *MySQL) AddDirectMessage(sender, recipient string, senderPersistent, recipientPersistent bool, item history.Item) (err error) { + if mysql.db == nil { + return + } + + if !(senderPersistent || recipientPersistent) { + return + } + + if sender == "" || recipient == "" { + return utils.ErrInvalidParams + } + + id, err := mysql.insertBase(item) + if err != nil { + return + } + + if senderPersistent { + mysql.insertSequenceEntry(sender, item.Message.Time, id) + if err != nil { + return + } + } + + if recipientPersistent && sender != recipient { + err = mysql.insertSequenceEntry(recipient, item.Message.Time, id) + if err != nil { + return + } + } + + err = mysql.insertConversationEntry(sender, recipient, item.Message.Time, id) + + return +} + +func (mysql *MySQL) msgidToTime(msgid string) (result time.Time, err error) { + // in theory, we could optimize out a roundtrip to the database by using a subquery instead: + // sequence.nanotime > ( + // SELECT sequence.nanotime FROM sequence, history + // WHERE sequence.history_id = history.id AND history.msgid = ? + // LIMIT 1) + // however, this doesn't handle the BETWEEN case with one or two msgids, where we + // don't initially know whether the interval is going forwards or backwards. to simplify + // the logic, resolve msgids to timestamps "manually" in all cases, using a separate query. + decoded, err := decodeMsgid(msgid) + if err != nil { + return + } + row := mysql.db.QueryRow(` + SELECT sequence.nanotime FROM sequence + INNER JOIN history ON history.id = sequence.history_id + WHERE history.msgid = ? LIMIT 1;`, decoded) + var nanotime int64 + err = row.Scan(&nanotime) + if mysql.logError("could not resolve msgid to time", err) { + return + } + result = time.Unix(0, nanotime) + return +} + +func (mysql *MySQL) selectItems(query string, args ...interface{}) (results []history.Item, err error) { + rows, err := mysql.db.Query(query, args...) + if mysql.logError("could not select history items", err) { + return + } + + defer rows.Close() + + for rows.Next() { + var blob []byte + var item history.Item + err = rows.Scan(&blob) + if mysql.logError("could not scan history item", err) { + return + } + err = unmarshalItem(blob, &item) + if mysql.logError("could not unmarshal history item", err) { + return + } + results = append(results, item) + } + return +} + +func (mysql *MySQL) BetweenTimestamps(sender, recipient string, after, before, cutoff time.Time, limit int) (results []history.Item, err error) { + useSequence := true + var lowerTarget, upperTarget string + if sender != "" { + lowerTarget, upperTarget = stringMinMax(sender, recipient) + useSequence = false + } + + table := "sequence" + if !useSequence { + table = "conversations" + } + + after, before, ascending := history.MinMaxAsc(after, before, cutoff) + direction := "ASC" + if !ascending { + direction = "DESC" + } + + var queryBuf bytes.Buffer + + args := make([]interface{}, 0, 6) + fmt.Fprintf(&queryBuf, + "SELECT history.data from history INNER JOIN %[1]s ON history.id = %[1]s.history_id WHERE", table) + if useSequence { + fmt.Fprintf(&queryBuf, " sequence.target = ?") + args = append(args, recipient) + } else { + fmt.Fprintf(&queryBuf, " conversations.lower_target = ? AND conversations.upper_target = ?") + args = append(args, lowerTarget) + args = append(args, upperTarget) + } + if !after.IsZero() { + fmt.Fprintf(&queryBuf, " AND %s.nanotime > ?", table) + args = append(args, after.UnixNano()) + } + if !before.IsZero() { + fmt.Fprintf(&queryBuf, " AND %s.nanotime < ?", table) + args = append(args, before.UnixNano()) + } + fmt.Fprintf(&queryBuf, " ORDER BY %[1]s.nanotime %[2]s LIMIT ?;", table, direction) + args = append(args, limit) + + results, err = mysql.selectItems(queryBuf.String(), args...) + if err == nil && !ascending { + history.Reverse(results) + } + return +} + +func (mysql *MySQL) Close() { + // closing the database will close our prepared statements as well + if mysql.db != nil { + mysql.db.Close() + } + mysql.db = nil +} + +// implements history.Sequence, emulating a single history buffer (for a channel, +// a single user's DMs, or a DM conversation) +type mySQLHistorySequence struct { + mysql *MySQL + sender string + recipient string + cutoff time.Time +} + +func (s *mySQLHistorySequence) Between(start, end history.Selector, limit int) (results []history.Item, complete bool, err error) { + startTime := start.Time + if start.Msgid != "" { + startTime, err = s.mysql.msgidToTime(start.Msgid) + if err != nil { + return nil, false, err + } + } + endTime := end.Time + if end.Msgid != "" { + endTime, err = s.mysql.msgidToTime(end.Msgid) + if err != nil { + return nil, false, err + } + } + + results, err = s.mysql.BetweenTimestamps(s.sender, s.recipient, startTime, endTime, s.cutoff, limit) + return results, (err == nil), err +} + +func (s *mySQLHistorySequence) Around(start history.Selector, limit int) (results []history.Item, err error) { + return history.GenericAround(s, start, limit) +} + +func (mysql *MySQL) MakeSequence(sender, recipient string, cutoff time.Time) history.Sequence { + return &mySQLHistorySequence{ + sender: sender, + recipient: recipient, + mysql: mysql, + cutoff: cutoff, + } +} diff --git a/irc/mysql/serialization.go b/irc/mysql/serialization.go new file mode 100644 index 00000000..bc3dbcf9 --- /dev/null +++ b/irc/mysql/serialization.go @@ -0,0 +1,23 @@ +package mysql + +import ( + "encoding/json" + + "github.com/oragono/oragono/irc/history" + "github.com/oragono/oragono/irc/utils" +) + +// 123 / '{' is the magic number that means JSON; +// if we want to do a binary encoding later, we just have to add different magic version numbers + +func marshalItem(item *history.Item) (result []byte, err error) { + return json.Marshal(item) +} + +func unmarshalItem(data []byte, result *history.Item) (err error) { + return json.Unmarshal(data, result) +} + +func decodeMsgid(msgid string) ([]byte, error) { + return utils.B32Encoder.DecodeString(msgid) +} diff --git a/irc/nickname.go b/irc/nickname.go index d7fb775b..71f7629d 100644 --- a/irc/nickname.go +++ b/irc/nickname.go @@ -43,13 +43,15 @@ func performNickChange(server *Server, client *Client, target *Client, session * hadNick := target.HasNick() origNickMask := target.NickMaskString() details := target.Details() - err := client.server.clients.SetNick(target, session, nickname) + assignedNickname, err := client.server.clients.SetNick(target, session, nickname) if err == errNicknameInUse { rb.Add(nil, server.name, ERR_NICKNAMEINUSE, currentNick, nickname, client.t("Nickname is already in use")) } else if err == errNicknameReserved { rb.Add(nil, server.name, ERR_NICKNAMEINUSE, currentNick, nickname, client.t("Nickname is reserved by a different account")) } else if err == errNicknameInvalid { rb.Add(nil, server.name, ERR_ERRONEUSNICKNAME, currentNick, utils.SafeErrorParam(nickname), client.t("Erroneous nickname")) + } else if err == errCantChangeNick { + rb.Add(nil, server.name, ERR_NICKNAMEINUSE, currentNick, utils.SafeErrorParam(nickname), client.t(err.Error())) } else if err != nil { rb.Add(nil, server.name, ERR_UNKNOWNERROR, currentNick, "NICK", fmt.Sprintf(client.t("Could not set or change nickname: %s"), err.Error())) } @@ -64,26 +66,26 @@ func performNickChange(server *Server, client *Client, target *Client, session * AccountName: details.accountName, Message: message, } - histItem.Params[0] = nickname + histItem.Params[0] = assignedNickname - client.server.logger.Debug("nick", fmt.Sprintf("%s changed nickname to %s [%s]", origNickMask, nickname, client.NickCasefolded())) + client.server.logger.Debug("nick", fmt.Sprintf("%s changed nickname to %s [%s]", origNickMask, assignedNickname, client.NickCasefolded())) if hadNick { if client == target { - target.server.snomasks.Send(sno.LocalNicks, fmt.Sprintf(ircfmt.Unescape("$%s$r changed nickname to %s"), details.nick, nickname)) + target.server.snomasks.Send(sno.LocalNicks, fmt.Sprintf(ircfmt.Unescape("$%s$r changed nickname to %s"), details.nick, assignedNickname)) } else { - target.server.snomasks.Send(sno.LocalNicks, fmt.Sprintf(ircfmt.Unescape("Operator %s changed nickname of $%s$r to %s"), client.Nick(), details.nick, nickname)) + target.server.snomasks.Send(sno.LocalNicks, fmt.Sprintf(ircfmt.Unescape("Operator %s changed nickname of $%s$r to %s"), client.Nick(), details.nick, assignedNickname)) } target.server.whoWas.Append(details.WhoWas) - rb.AddFromClient(message.Time, message.Msgid, origNickMask, details.accountName, nil, "NICK", nickname) + rb.AddFromClient(message.Time, message.Msgid, origNickMask, details.accountName, nil, "NICK", assignedNickname) for session := range target.Friends() { if session != rb.session { - session.sendFromClientInternal(false, message.Time, message.Msgid, origNickMask, details.accountName, nil, "NICK", nickname) + session.sendFromClientInternal(false, message.Time, message.Msgid, origNickMask, details.accountName, nil, "NICK", assignedNickname) } } } for _, channel := range client.Channels() { - channel.history.Add(histItem) + channel.AddHistoryItem(histItem) } if target.Registered() { diff --git a/irc/nickserv.go b/irc/nickserv.go index 16177c75..102efe04 100644 --- a/irc/nickserv.go +++ b/irc/nickserv.go @@ -217,7 +217,7 @@ information on the settings and their possible values, see HELP SET.`, helpStrings: []string{ `Syntax $bSET $b -Set modifies your account settings. The following settings are available:`, +SET modifies your account settings. The following settings are available:`, `$bENFORCE$b 'enforce' lets you specify a custom enforcement mechanism for your registered @@ -247,6 +247,22 @@ lines for join and part. This provides more information about the context of messages, but may be spammy. Your options are 'always', 'never', and the default of 'commands-only' (the messages will be replayed in /HISTORY output, but not during autoreplay).`, + `$bALWAYS-ON$b +'always-on' controls whether your nickname/identity will remain active +even while you are disconnected from the server. Your options are 'true', +'false', and 'default' (use the server default value).`, + `$bAUTOREPLAY-MISSED$b +'autoreplay-missed' is only effective for always-on clients. If enabled, +if you have at most one active session, the server will remember the time +you disconnect and then replay missed messages to you when you reconnect. +Your options are 'on' and 'off'.`, + `$bDM-HISTORY$b +'dm-history' is only effective for always-on clients. It lets you control +how the history of your direct messages is stored. Your options are: +1. 'off' [no history] +2. 'ephemeral' [a limited amount of temporary history, not stored on disk] +3. 'on' [history stored in a permanent database, if available] +4. 'default' [use the server default]`, }, authRequired: true, enabled: servCmdRequiresAccreg, @@ -349,6 +365,31 @@ func displaySetting(settingName string, settings AccountSettings, client *Client nsNotice(rb, client.t("Bouncer functionality is currently enabled for your account")) } } + case "always-on": + stored := settings.AlwaysOn + actual := client.AlwaysOn() + nsNotice(rb, fmt.Sprintf(client.t("Your stored always-on setting is: %s"), persistentStatusToString(stored))) + if actual { + nsNotice(rb, client.t("Given current server settings, your client is always-on")) + } else { + nsNotice(rb, client.t("Given current server settings, your client is not always-on")) + } + case "autoreplay-missed": + stored := settings.AutoreplayMissed + if stored { + if client.AlwaysOn() { + nsNotice(rb, client.t("Autoreplay of missed messages is enabled")) + } else { + nsNotice(rb, client.t("You have enabled autoreplay of missed messages, but you can't receive them because your client isn't set to always-on")) + } + } else { + nsNotice(rb, client.t("Your account is not configured to receive autoreplayed missed messages")) + } + case "dm-history": + effectiveValue := historyEnabled(config.History.Persistent.DirectMessages, settings.DMHistory) + csNotice(rb, fmt.Sprintf(client.t("Your stored direct message history setting is: %s"), historyStatusToString(settings.DMHistory))) + csNotice(rb, fmt.Sprintf(client.t("Given current server settings, your direct message history setting is: %s"), historyStatusToString(effectiveValue))) + default: nsNotice(rb, client.t("No such setting")) } @@ -429,6 +470,37 @@ func nsSetHandler(server *Server, client *Client, command string, params []strin return } } + case "always-on": + var newValue PersistentStatus + newValue, err = persistentStatusFromString(params[1]) + // "opt-in" and "opt-out" don't make sense as user preferences + if err == nil && newValue != PersistentOptIn && newValue != PersistentOptOut { + munger = func(in AccountSettings) (out AccountSettings, err error) { + out = in + out.AlwaysOn = newValue + return + } + } + case "autoreplay-missed": + var newValue bool + newValue, err = utils.StringToBool(params[1]) + if err == nil { + munger = func(in AccountSettings) (out AccountSettings, err error) { + out = in + out.AutoreplayMissed = newValue + return + } + } + case "dm-history": + var newValue HistoryStatus + newValue, err = historyStatusFromString(params[1]) + if err == nil { + munger = func(in AccountSettings) (out AccountSettings, err error) { + out = in + out.DMHistory = newValue + return + } + } default: err = errInvalidParams } @@ -480,6 +552,9 @@ func nsGhostHandler(server *Server, client *Client, command string, params []str } else if ghost == client { nsNotice(rb, client.t("You can't GHOST yourself (try /QUIT instead)")) return + } else if ghost.AlwaysOn() { + nsNotice(rb, client.t("You can't GHOST an always-on client")) + return } authorized := false diff --git a/irc/responsebuffer.go b/irc/responsebuffer.go index e4daa5ca..419baae8 100644 --- a/irc/responsebuffer.go +++ b/irc/responsebuffer.go @@ -58,6 +58,10 @@ func NewResponseBuffer(session *Session) *ResponseBuffer { } func (rb *ResponseBuffer) AddMessage(msg ircmsg.IrcMessage) { + if rb == nil { + return + } + if rb.finalized { rb.target.server.logger.Error("internal", "message added to finalized ResponseBuffer, undefined behavior") debug.PrintStack() @@ -80,12 +84,20 @@ func (rb *ResponseBuffer) setNestedBatchTag(msg *ircmsg.IrcMessage) { // Add adds a standard new message to our queue. func (rb *ResponseBuffer) Add(tags map[string]string, prefix string, command string, params ...string) { + if rb == nil { + return + } + rb.AddMessage(ircmsg.MakeMessage(tags, prefix, command, params...)) } // Broadcast adds a standard new message to our queue, then sends an unlabeled copy // to all other sessions. func (rb *ResponseBuffer) Broadcast(tags map[string]string, prefix string, command string, params ...string) { + if rb == nil { + return + } + // can't reuse the IrcMessage object because of tag pollution :-\ rb.Add(tags, prefix, command, params...) for _, session := range rb.session.client.Sessions() { @@ -97,6 +109,10 @@ func (rb *ResponseBuffer) Broadcast(tags map[string]string, prefix string, comma // AddFromClient adds a new message from a specific client to our queue. func (rb *ResponseBuffer) AddFromClient(time time.Time, msgid string, fromNickMask string, fromAccount string, tags map[string]string, command string, params ...string) { + if rb == nil { + return + } + msg := ircmsg.MakeMessage(nil, fromNickMask, command, params...) if rb.session.capabilities.Has(caps.MessageTags) { msg.UpdateTags(tags) @@ -118,10 +134,14 @@ func (rb *ResponseBuffer) AddFromClient(time time.Time, msgid string, fromNickMa // AddSplitMessageFromClient adds a new split message from a specific client to our queue. func (rb *ResponseBuffer) AddSplitMessageFromClient(fromNickMask string, fromAccount string, tags map[string]string, command string, target string, message utils.SplitMessage) { + if rb == nil { + return + } + if message.Is512() { rb.AddFromClient(message.Time, message.Msgid, fromNickMask, fromAccount, tags, command, target, message.Message) } else { - if message.IsMultiline() && rb.session.capabilities.Has(caps.Multiline) { + if rb.session.capabilities.Has(caps.Multiline) { batch := rb.session.composeMultilineBatch(fromNickMask, fromAccount, tags, command, target, message) rb.setNestedBatchTag(&batch[0]) rb.setNestedBatchTag(&batch[len(batch)-1]) @@ -165,6 +185,10 @@ func (rb *ResponseBuffer) sendBatchEnd(blocking bool) { // Starts a nested batch (see the ResponseBuffer struct definition for a description of // how this works) func (rb *ResponseBuffer) StartNestedBatch(batchType string, params ...string) (batchID string) { + if rb == nil { + return + } + batchID = rb.session.generateBatchID() msgParams := make([]string, len(params)+2) msgParams[0] = "+" + batchID @@ -194,6 +218,10 @@ func (rb *ResponseBuffer) EndNestedBatch(batchID string) { // Convenience to start a nested batch for history lines, at the highest level // supported by the client (`history`, `chathistory`, or no batch, in descending order). func (rb *ResponseBuffer) StartNestedHistoryBatch(params ...string) (batchID string) { + if rb == nil { + return + } + var batchType string if rb.session.capabilities.Has(caps.EventPlayback) { batchType = "history" @@ -210,6 +238,10 @@ func (rb *ResponseBuffer) StartNestedHistoryBatch(params ...string) (batchID str // Afterwards, the buffer is in an undefined state and MUST NOT be used further. // If `blocking` is true you MUST be sending to the client from its own goroutine. func (rb *ResponseBuffer) Send(blocking bool) error { + if rb == nil { + return nil + } + return rb.flushInternal(true, blocking) } @@ -218,6 +250,10 @@ func (rb *ResponseBuffer) Send(blocking bool) error { // to ensure that the final `BATCH -` message is sent. // If `blocking` is true you MUST be sending to the client from its own goroutine. func (rb *ResponseBuffer) Flush(blocking bool) error { + if rb == nil { + return nil + } + return rb.flushInternal(false, blocking) } @@ -292,5 +328,9 @@ func (rb *ResponseBuffer) flushInternal(final bool, blocking bool) error { // Notice sends the client the given notice from the server. func (rb *ResponseBuffer) Notice(text string) { - rb.Add(nil, rb.target.server.name, "NOTICE", rb.target.nick, text) + if rb == nil { + return + } + + rb.Add(nil, rb.target.server.name, "NOTICE", rb.target.Nick(), text) } diff --git a/irc/server.go b/irc/server.go index abf6f530..54f490d5 100644 --- a/irc/server.go +++ b/irc/server.go @@ -24,8 +24,10 @@ import ( "github.com/goshuirc/irc-go/ircfmt" "github.com/oragono/oragono/irc/caps" "github.com/oragono/oragono/irc/connection_limits" + "github.com/oragono/oragono/irc/history" "github.com/oragono/oragono/irc/logger" "github.com/oragono/oragono/irc/modes" + "github.com/oragono/oragono/irc/mysql" "github.com/oragono/oragono/irc/sno" "github.com/tidwall/buntdb" ) @@ -84,6 +86,7 @@ type Server struct { signals chan os.Signal snomasks SnoManager store *buntdb.DB + historyDB mysql.MySQL torLimiter connection_limits.TorLimiter whoWas WhoWasList stats Stats @@ -122,7 +125,7 @@ func NewServer(config *Config, logger *logger.Manager) (*Server, error) { server.monitorManager.Initialize() server.snomasks.Initialize() - if err := server.applyConfig(config, true); err != nil { + if err := server.applyConfig(config); err != nil { return nil, err } @@ -143,6 +146,8 @@ func (server *Server) Shutdown() { if err := server.store.Close(); err != nil { server.logger.Error("shutdown", fmt.Sprintln("Could not close datastore:", err)) } + + server.historyDB.Close() } // Run starts the server. @@ -316,7 +321,7 @@ func (server *Server) tryRegister(c *Client, session *Session) (exiting bool) { // client MUST send PASS if necessary, or authenticate with SASL if necessary, // before completing the other registration commands - authOutcome := c.isAuthorized(server.Config()) + authOutcome := c.isAuthorized(server.Config(), session.isTor) var quitMessage string switch authOutcome { case authFailPass: @@ -376,7 +381,7 @@ func (server *Server) playRegistrationBurst(session *Session) { // continue registration d := c.Details() server.logger.Info("localconnect", fmt.Sprintf("Client connected [%s] [u:%s] [r:%s]", d.nick, d.username, d.realname)) - server.snomasks.Send(sno.LocalConnects, fmt.Sprintf("Client connected [%s] [u:%s] [h:%s] [ip:%s] [r:%s]", d.nick, d.username, c.RawHostname(), c.IPString(), d.realname)) + server.snomasks.Send(sno.LocalConnects, fmt.Sprintf("Client connected [%s] [u:%s] [h:%s] [ip:%s] [r:%s]", d.nick, d.username, session.rawHostname, session.IP().String(), d.realname)) // send welcome text //NOTE(dan): we specifically use the NICK here instead of the nickmask @@ -550,7 +555,7 @@ func (server *Server) rehash() error { return fmt.Errorf("Error loading config file config: %s", err.Error()) } - err = server.applyConfig(config, false) + err = server.applyConfig(config) if err != nil { return fmt.Errorf("Error applying config changes: %s", err.Error()) } @@ -558,7 +563,10 @@ func (server *Server) rehash() error { return nil } -func (server *Server) applyConfig(config *Config, initial bool) (err error) { +func (server *Server) applyConfig(config *Config) (err error) { + oldConfig := server.Config() + initial := oldConfig == nil + if initial { server.configFilename = config.Filename server.name = config.Server.Name @@ -568,7 +576,7 @@ func (server *Server) applyConfig(config *Config, initial bool) (err error) { // enforce configs that can't be changed after launch: if server.name != config.Server.Name { return fmt.Errorf("Server name cannot be changed after launching the server, rehash aborted") - } else if server.Config().Datastore.Path != config.Datastore.Path { + } else if oldConfig.Datastore.Path != config.Datastore.Path { return fmt.Errorf("Datastore path cannot be changed after launching the server, rehash aborted") } else if globalCasemappingSetting != config.Server.Casemapping { return fmt.Errorf("Casemapping cannot be changed after launching the server, rehash aborted") @@ -576,7 +584,6 @@ func (server *Server) applyConfig(config *Config, initial bool) (err error) { } server.logger.Info("server", "Using config file", server.configFilename) - oldConfig := server.Config() // first, reload config sections for functionality implemented in subpackages: wasLoggingRawIO := !initial && server.logger.IsLoggingRawIO() @@ -609,14 +616,13 @@ func (server *Server) applyConfig(config *Config, initial bool) (err error) { if !oldConfig.Channels.Registration.Enabled { server.channels.loadRegisteredChannels(config) } - // resize history buffers as needed if oldConfig.History != config.History { for _, channel := range server.channels.Channels() { - channel.history.Resize(config.History.ChannelLength, config.History.AutoresizeWindow) + channel.resizeHistory(config) } for _, client := range server.clients.AllClients() { - client.history.Resize(config.History.ClientLength, config.History.AutoresizeWindow) + client.resizeHistory(config) } } } @@ -658,6 +664,10 @@ func (server *Server) applyConfig(config *Config, initial bool) (err error) { if err := server.loadDatastore(config); err != nil { return err } + } else { + if config.Datastore.MySQL.Enabled { + server.historyDB.SetExpireTime(config.History.Restrictions.ExpireTime) + } } server.setupPprofListener(config) @@ -778,6 +788,15 @@ func (server *Server) loadDatastore(config *Config) error { server.channels.Initialize(server) server.accounts.Initialize(server) + if config.Datastore.MySQL.Enabled { + server.historyDB.Initialize(server.logger, config.History.Restrictions.ExpireTime) + err = server.historyDB.Open(config.Datastore.MySQL.User, config.Datastore.MySQL.Password, config.Datastore.MySQL.Host, config.Datastore.MySQL.Port, config.Datastore.MySQL.HistoryDatabase) + if err != nil { + server.logger.Error("internal", "could not connect to mysql", err.Error()) + return err + } + } + return nil } @@ -835,6 +854,72 @@ func (server *Server) setupListeners(config *Config) (err error) { return } +// Gets the abstract sequence from which we're going to query history; +// we may already know the channel we're querying, or we may have +// to look it up via a string target. This function is responsible for +// privilege checking. +func (server *Server) GetHistorySequence(providedChannel *Channel, client *Client, target string) (channel *Channel, sequence history.Sequence, err error) { + config := server.Config() + var sender, recipient string + var hist *history.Buffer + if target == "*" { + if client.AlwaysOn() { + recipient = client.NickCasefolded() + } else { + hist = &client.history + } + } else { + channel = providedChannel + if channel == nil { + channel = server.channels.Get(target) + } + if channel != nil { + if !channel.hasClient(client) { + err = errInsufficientPrivs + return + } + persistent, ephemeral, cfTarget := channel.historyStatus(config) + if persistent { + recipient = cfTarget + } else if ephemeral { + hist = &channel.history + } else { + return + } + } else { + sender = client.NickCasefolded() + var cfTarget string + cfTarget, err = CasefoldName(target) + if err != nil { + return + } + recipient = cfTarget + if !client.AlwaysOn() { + hist = &client.history + } + } + } + + var cutoff time.Time + if config.History.Restrictions.ExpireTime != 0 { + cutoff = time.Now().UTC().Add(-config.History.Restrictions.ExpireTime) + } + if config.History.Restrictions.EnforceRegistrationDate { + regCutoff := client.historyCutoff() + regCutoff.Add(-config.History.Restrictions.GracePeriod) + // take the earlier of the two cutoffs + if regCutoff.After(cutoff) { + cutoff = regCutoff + } + } + if hist != nil { + sequence = hist.MakeSequence(recipient, cutoff) + } else if recipient != "" { + sequence = server.historyDB.MakeSequence(sender, recipient, cutoff) + } + return +} + // elistMatcher takes and matches ELIST conditions type elistMatcher struct { MinClientsActive bool diff --git a/irc/stats.go b/irc/stats.go index 5643ad87..a54f7269 100644 --- a/irc/stats.go +++ b/irc/stats.go @@ -26,15 +26,33 @@ func (s *Stats) Add() { s.mutex.Unlock() } +// Activates a registered client, e.g., for the initial attach to a persistent client +func (s *Stats) AddRegistered(invisible, operator bool) { + s.mutex.Lock() + if invisible { + s.Invisible += 1 + } + if operator { + s.Operators += 1 + } + s.Total += 1 + s.setMax() + s.mutex.Unlock() +} + // Transition a client from unregistered to registered func (s *Stats) Register() { s.mutex.Lock() s.Unknown -= 1 s.Total += 1 + s.setMax() + s.mutex.Unlock() +} + +func (s *Stats) setMax() { if s.Max < s.Total { s.Max = s.Total } - s.mutex.Unlock() } // Modify the Invisible count diff --git a/irc/utils/args.go b/irc/utils/args.go index dd2c8b64..98a11ed8 100644 --- a/irc/utils/args.go +++ b/irc/utils/args.go @@ -5,7 +5,13 @@ package utils import ( "errors" + "fmt" "strings" + "time" +) + +const ( + IRCv3TimestampFormat = "2006-01-02T15:04:05.000Z" ) var ( @@ -45,9 +51,9 @@ func ArgsToStrings(maxLength int, arguments []string, delim string) []string { func StringToBool(str string) (result bool, err error) { switch strings.ToLower(str) { - case "on", "true", "t", "yes", "y": + case "on", "true", "t", "yes", "y", "disabled": result = true - case "off", "false", "f", "no", "n": + case "off", "false", "f", "no", "n", "enabled": result = false default: err = ErrInvalidParams @@ -63,3 +69,16 @@ func SafeErrorParam(param string) string { } return param } + +type IncompatibleSchemaError struct { + CurrentVersion string + RequiredVersion string +} + +func (err *IncompatibleSchemaError) Error() string { + return fmt.Sprintf("Database requires update. Expected schema v%s, got v%s", err.RequiredVersion, err.CurrentVersion) +} + +func NanoToTimestamp(nanotime int64) string { + return time.Unix(0, nanotime).Format(IRCv3TimestampFormat) +} diff --git a/irc/utils/net.go b/irc/utils/net.go index 3eb13192..a3a3b01b 100644 --- a/irc/utils/net.go +++ b/irc/utils/net.go @@ -18,14 +18,6 @@ var ( validHostnameLabelRegexp = regexp.MustCompile(`^[0-9A-Za-z.\-]+$`) ) -// AddrIsLocal returns whether the address is from a trusted local connection (loopback or unix). -func AddrIsLocal(addr net.Addr) bool { - if tcpaddr, ok := addr.(*net.TCPAddr); ok { - return tcpaddr.IP.IsLoopback() - } - return AddrIsUnix(addr) -} - // AddrToIP returns the IP address for a net.Addr; unix domain sockets are treated as IPv4 loopback func AddrToIP(addr net.Addr) net.IP { if tcpaddr, ok := addr.(*net.TCPAddr); ok { diff --git a/irc/utils/text.go b/irc/utils/text.go index de240643..5d46b140 100644 --- a/irc/utils/text.go +++ b/irc/utils/text.go @@ -23,9 +23,9 @@ type MessagePair struct { // SplitMessage represents a message that's been split for sending. // Two possibilities: // (a) Standard message that can be relayed on a single 512-byte line -// (MessagePair contains the message, Wrapped == nil) +// (MessagePair contains the message, Split == nil) // (b) multiline message that was split on the client side -// (Message == "", Wrapped contains the split lines) +// (Message == "", Split contains the split lines) type SplitMessage struct { Message string Msgid string @@ -36,7 +36,7 @@ type SplitMessage struct { func MakeMessage(original string) (result SplitMessage) { result.Message = original result.Msgid = GenerateSecretToken() - result.Time = time.Now().UTC() + result.SetTime() return } @@ -52,7 +52,8 @@ func (sm *SplitMessage) Append(message string, concat bool) { } func (sm *SplitMessage) SetTime() { - sm.Time = time.Now().UTC() + // strip the monotonic time, it's a potential source of problems: + sm.Time = time.Now().UTC().Round(0) } func (sm *SplitMessage) LenLines() int { @@ -88,10 +89,6 @@ func (sm *SplitMessage) IsRestrictedCTCPMessage() bool { return false } -func (sm *SplitMessage) IsMultiline() bool { - return sm.Message == "" && len(sm.Split) != 0 -} - func (sm *SplitMessage) Is512() bool { return sm.Message != "" } diff --git a/irc/znc.go b/irc/znc.go index 8e18449d..aedfa3a4 100644 --- a/irc/znc.go +++ b/irc/znc.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" "time" + + "github.com/oragono/oragono/irc/history" ) type zncCommandHandler func(client *Client, command string, params []string, rb *ResponseBuffer) @@ -89,10 +91,8 @@ func zncPlaybackHandler(client *Client, command string, params []string, rb *Res // 3.3 When the client sends a subsequent redundant JOIN line for those // channels; redundant JOIN is a complete no-op so we won't replay twice - config := client.server.Config() if params[1] == "*" { - items, _ := client.history.Between(after, before, false, config.History.ChathistoryMax) - client.replayPrivmsgHistory(rb, items, true) + zncPlayPrivmsgs(client, rb, after, before) } else { targets = make(StringSet) // TODO actually handle nickname targets @@ -116,3 +116,15 @@ func zncPlaybackHandler(client *Client, command string, params []string, rb *Res } } } + +func zncPlayPrivmsgs(client *Client, rb *ResponseBuffer, after, before time.Time) { + _, sequence, _ := client.server.GetHistorySequence(nil, client, "*") + if sequence == nil { + return + } + zncMax := client.server.Config().History.ZNCMax + items, _, err := sequence.Between(history.Selector{Time: after}, history.Selector{Time: before}, zncMax) + if err == nil { + client.replayPrivmsgHistory(rb, items, "", true) + } +} diff --git a/oragono.yaml b/oragono.yaml index 306610c0..49a9d1fe 100644 --- a/oragono.yaml +++ b/oragono.yaml @@ -245,6 +245,15 @@ server: # all users will receive simply `netname` as their cloaked hostname. num-bits: 80 + # secure-nets identifies IPs and CIDRs which are secure at layer 3, + # for example, because they are on a trusted internal LAN or a VPN. + # plaintext connections from these IPs and CIDRs will be considered + # secure (clients will receive the +Z mode and be allowed to resume + # or reattach to secure connections). note that loopback IPs are always + # considered secure: + secure-nets: + # - "10.0.0.0/8" + # account options accounts: @@ -351,6 +360,11 @@ accounts: # via nickserv allowed-by-default: true + # whether to allow clients that remain on the server even + # when they have no active connections. The possible values are: + # "disabled", "opt-in", "opt-out", or "mandatory". + always-on: "disabled" + # vhosts controls the assignment of vhosts (strings displayed in place of the user's # hostname/IP) by the HostServ service vhosts: @@ -585,6 +599,16 @@ datastore: # up, and if the upgrade fails, the original database will be restored. autoupgrade: true + # connection information for MySQL (currently only used for persistent history): + mysql: + enabled: false + host: "localhost" + # port is unnecessary for connections via unix domain socket: + #port: 3306 + user: "oragono" + password: "KOHw8WSaRwaoo-avo0qVpQ" + history-database: "oragono_history" + # languages config languages: # whether to load languages @@ -657,7 +681,7 @@ fakelag: # message history tracking, for the RESUME extension and possibly other uses in future history: # should we store messages for later playback? - # the current implementation stores messages in RAM only; they do not persist + # by default, messages are stored in RAM only; they do not persist # across server restarts. however, you should not enable this unless you understand # how it interacts with the GDPR and/or any data privacy laws that apply # in your country and the countries of your users. @@ -683,3 +707,41 @@ history: # maximum number of CHATHISTORY messages that can be # requested at once (0 disables support for CHATHISTORY) chathistory-maxmessages: 100 + + # maximum number of messages that can be replayed at once during znc emulation + # (znc.in/playback, or automatic replay on initial reattach to a persistent client): + znc-maxmessages: 2048 + + # options to delete old messages, or prevent them from being retrieved + restrictions: + # if this is set, messages older than this cannot be retrieved by anyone + # (and will eventually be deleted from persistent storage, if that's enabled) + #expire-time: 168h # 7 days + + # if this is set, logged-in users cannot retrieve messages older than their + # account registration date, and logged-out users cannot retrieve messages + # older than their sign-on time (modulo grace-period, see below): + enforce-registration-date: false + + # but if this is set, you can retrieve messages that are up to `grace-period` + # older than the above cutoff time. this is recommended to allow logged-out + # users to do session resumption / query history after disconnections. + grace-period: 1h + + # options to store history messages in a persistent database (currently only MySQL): + persistent: + enabled: false + + # store unregistered channel messages in the persistent database? + unregistered-channels: false + + # for a registered channel, the channel owner can potentially customize + # the history storage setting. as the server operator, your options are + # 'disabled' (no persistent storage, regardless of per-channel setting), + # 'opt-in', 'opt-out', and 'mandatory' (force persistent storage, ignoring + # per-channel setting): + registered-channels: "opt-out" + + # direct messages are only stored in the database for persistent clients; + # you can control how they are stored here (same options as above) + direct-messages: "opt-out"