GitHub/ergo
3
0
Fork 0
mirror of https://github.com/ergochat/ergo.git synced 2025-11-04 07:47:25 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1004 from slingamn/vanish.4

Browse Source 
fix #858 and #383
This commit is contained in:
Shivaram Lingamneni 2020-05-14 06:54:24 -07:00 committed by GitHub
commit a46c0eed9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 821 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
conventional.yaml
View File

@ -259,6 +259,10 @@ server:
secure-nets:
# - "10.0.0.0/8"
# oragono will write files to disk under certain circumstances, e.g.,
# CPU profiling or data export. by default, these files will be written
# to the working directory. set this to customize:
# output-path: "/home/oragono/out"
# account options
accounts:
@ -556,6 +560,7 @@ oper-classes:
- "samode"
- "vhosts"
- "chanreg"
- "history"
# ircd operators
opers:
@ -751,7 +756,8 @@ roleplay:
# add the real nickname, in parentheses, to the end of every roleplay message?
add-suffix: true
# message history tracking, for the RESUME extension and possibly other uses in future
# history message storage: this is used by CHATHISTORY, HISTORY, znc.in/playback,
# various autoreplay features, and the resume extension
history:
# should we store messages for later playback?
# by default, messages are stored in RAM only; they do not persist
@ -820,3 +826,13 @@ history:
# if you enable this, strict nickname reservation is strongly recommended
# as well.
direct-messages: "opt-out"
# options to control how messages are stored and deleted:
retention:
# allow users to delete their own messages from history?
allow-individual-delete: false
# if persistent history is enabled, create additional index tables,
# allowing deletion of JSON export of an account's messages. this
# may be needed for compliance with data privacy regulations.
enable-account-indexing: false

24
irc/accounts.go
View File

@ -1103,6 +1103,30 @@ func (am *AccountManager) LoadAccount(accountName string) (result ClientAccount,
return
}
// look up the unfolded version of an account name, possibly after deletion
func (am *AccountManager) AccountToAccountName(account string) (result string) {
casefoldedAccount, err := CasefoldName(account)
if err != nil {
return
}
unregisteredKey := fmt.Sprintf(keyAccountUnregistered, casefoldedAccount)
accountNameKey := fmt.Sprintf(keyAccountName, casefoldedAccount)
am.server.store.View(func(tx *buntdb.Tx) error {
if name, err := tx.Get(accountNameKey); err == nil {
result = name
return nil
}
if name, err := tx.Get(unregisteredKey); err == nil {
result = name
}
return nil
})
return
}
func (am *AccountManager) deserializeRawAccount(raw rawClientAccount, cfName string) (result ClientAccount, err error) {
result.Name = raw.Name
result.NameCasefolded = cfName

48
irc/channel.go
View File

@ -20,10 +20,6 @@ import (
"github.com/oragono/oragono/irc/utils"
)
const (
histServMask = "HistServ!HistServ@localhost"
)
type ChannelSettings struct {
History HistoryStatus
}
@ -641,14 +637,14 @@ func channelHistoryStatus(config *Config, registered bool, storedStatus HistoryS
}
}
func (channel *Channel) AddHistoryItem(item history.Item) (err error) {
func (channel *Channel) AddHistoryItem(item history.Item, account string) (err error) {
if !item.IsStorable() {
return
}
status, target := channel.historyStatus(channel.server.Config())
if status == HistoryPersistent {
err = channel.server.historyDB.AddChannelItem(target, item)
err = channel.server.historyDB.AddChannelItem(target, item, account)
} else if status == HistoryEphemeral {
channel.history.Add(item)
}
@ -746,7 +742,7 @@ func (channel *Channel) Join(client *Client, key string, isSajoin bool, rb *Resp
Message: message,
}
histItem.Params[0] = details.realname
channel.AddHistoryItem(histItem)
channel.AddHistoryItem(histItem, details.account)
}
client.addChannel(channel, rb == nil)
@ -902,7 +898,7 @@ func (channel *Channel) Part(client *Client, message string, rb *ResponseBuffer)
Nick: details.nickMask,
AccountName: details.accountName,
Message: splitMessage,
})
}, details.account)
client.server.logger.Debug("part", fmt.Sprintf("%s left channel %s", details.nick, chname))
}
@ -1165,7 +1161,7 @@ func (channel *Channel) SetTopic(client *Client, topic string, rb *ResponseBuffe
Nick: details.nickMask,
AccountName: details.accountName,
Message: message,
})
}, details.account)
channel.MarkDirty(IncludeTopic)
}
@ -1222,8 +1218,7 @@ func (channel *Channel) SendSplitMessage(command string, minPrefixMode modes.Mod
return
}
nickmask := client.NickMaskString()
account := client.AccountName()
details := client.Details()
chname := channel.Name()
// STATUSMSG targets are prefixed with the supplied min-prefix, e.g., @#channel
@ -1238,9 +1233,9 @@ func (channel *Channel) SendSplitMessage(command string, minPrefixMode modes.Mod
tagsToUse = clientOnlyTags
}
if histType == history.Tagmsg && rb.session.capabilities.Has(caps.MessageTags) {
rb.AddFromClient(message.Time, message.Msgid, nickmask, account, tagsToUse, command, chname)
rb.AddFromClient(message.Time, message.Msgid, details.nickMask, details.accountName, tagsToUse, command, chname)
} else {
rb.AddSplitMessageFromClient(nickmask, account, tagsToUse, command, chname, message)
rb.AddSplitMessageFromClient(details.nickMask, details.accountName, tagsToUse, command, chname, message)
}
}
// send echo-message to other connected sessions
@ -1253,9 +1248,9 @@ func (channel *Channel) SendSplitMessage(command string, minPrefixMode modes.Mod
tagsToUse = clientOnlyTags
}
if histType == history.Tagmsg && session.capabilities.Has(caps.MessageTags) {
session.sendFromClientInternal(false, message.Time, message.Msgid, nickmask, account, tagsToUse, command, chname)
session.sendFromClientInternal(false, message.Time, message.Msgid, details.nickMask, details.accountName, tagsToUse, command, chname)
} else if histType != history.Tagmsg {
session.sendSplitMsgFromClientInternal(false, nickmask, account, tagsToUse, command, chname, message)
session.sendSplitMsgFromClientInternal(false, details.nickMask, details.accountName, tagsToUse, command, chname, message)
}
}
@ -1282,9 +1277,9 @@ func (channel *Channel) SendSplitMessage(command string, minPrefixMode modes.Mod
}
if histType == history.Tagmsg {
session.sendFromClientInternal(false, message.Time, message.Msgid, nickmask, account, tagsToUse, command, chname)
session.sendFromClientInternal(false, message.Time, message.Msgid, details.nickMask, details.accountName, tagsToUse, command, chname)
} else {
session.sendSplitMsgFromClientInternal(false, nickmask, account, tagsToUse, command, chname, message)
session.sendSplitMsgFromClientInternal(false, details.nickMask, details.accountName, tagsToUse, command, chname, message)
}
}
}
@ -1294,10 +1289,10 @@ func (channel *Channel) SendSplitMessage(command string, minPrefixMode modes.Mod
channel.AddHistoryItem(history.Item{
Type: histType,
Message: message,
Nick: nickmask,
AccountName: account,
Nick: details.nickMask,
AccountName: details.accountName,
Tags: clientOnlyTags,
})
}, details.account)
}
}
@ -1391,28 +1386,27 @@ func (channel *Channel) Kick(client *Client, target *Client, comment string, rb
}
message := utils.MakeMessage(comment)
clientMask := client.NickMaskString()
clientAccount := client.AccountName()
details := client.Details()
targetNick := target.Nick()
chname := channel.Name()
for _, member := range channel.Members() {
for _, session := range member.Sessions() {
if session != rb.session {
session.sendFromClientInternal(false, message.Time, message.Msgid, clientMask, clientAccount, nil, "KICK", chname, targetNick, comment)
session.sendFromClientInternal(false, message.Time, message.Msgid, details.nickMask, details.accountName, nil, "KICK", chname, targetNick, comment)
}
}
}
rb.Add(nil, clientMask, "KICK", chname, targetNick, comment)
rb.AddFromClient(message.Time, message.Msgid, details.nickMask, details.accountName, nil, "KICK", chname, targetNick, comment)
histItem := history.Item{
Type: history.Kick,
Nick: clientMask,
AccountName: target.AccountName(),
Nick: details.nickMask,
AccountName: details.accountName,
Message: message,
}
histItem.Params[0] = targetNick
channel.AddHistoryItem(histItem)
channel.AddHistoryItem(histItem, details.account)
channel.Quit(target)
}

6
irc/chanserv.go
View File

@ -255,7 +255,7 @@ func csAmodeHandler(server *Server, client *Client, command string, params []str
if member.Account() == change.Arg {
applied, change := channel.applyModeToMember(client, change, rb)
if applied {
announceCmodeChanges(channel, modes.ModeChanges{change}, chanservMask, "*", rb)
announceCmodeChanges(channel, modes.ModeChanges{change}, chanservMask, "*", "", rb)
}
}
}
@ -302,7 +302,7 @@ func csOpHandler(server *Server, client *Client, command string, params []string
},
rb)
if applied {
announceCmodeChanges(channelInfo, modes.ModeChanges{change}, chanservMask, "*", rb)
announceCmodeChanges(channelInfo, modes.ModeChanges{change}, chanservMask, "*", "", rb)
}
csNotice(rb, fmt.Sprintf(client.t("Successfully op'd in channel %s"), channelName))
@ -354,7 +354,7 @@ func csRegisterHandler(server *Server, client *Client, command string, params []
},
rb)
if applied {
announceCmodeChanges(channelInfo, modes.ModeChanges{change}, chanservMask, "*", rb)
announceCmodeChanges(channelInfo, modes.ModeChanges{change}, chanservMask, "*", "", rb)
}
}

2
irc/client.go
View File

@ -1277,7 +1277,7 @@ func (client *Client) destroy(session *Session) {
// use a defer here to avoid writing to mysql while holding the destroy semaphore:
defer func() {
for _, channel := range channels {
channel.AddHistoryItem(quitItem)
channel.AddHistoryItem(quitItem, details.account)
}
}()

11
irc/config.go
View File

@ -13,6 +13,7 @@ import (
"log"
"net"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
@ -511,6 +512,7 @@ type Config struct {
supportedCaps *caps.Set
capValues caps.Values
Casemapping Casemapping
OutputPath string `yaml:"output-path"`
}
Roleplay struct {
@ -590,6 +592,10 @@ type Config struct {
RegisteredChannels PersistentStatus `yaml:"registered-channels"`
DirectMessages PersistentStatus `yaml:"direct-messages"`
}
Retention struct {
AllowIndividualDelete bool `yaml:"allow-individual-delete"`
EnableAccountIndexing bool `yaml:"enable-account-indexing"`
}
}
Filename string
@ -1111,6 +1117,7 @@ func LoadConfig(filename string) (config *Config, err error) {
config.Roleplay.addSuffix = utils.BoolDefaultTrue(config.Roleplay.AddSuffix)
config.Datastore.MySQL.ExpireTime = time.Duration(config.History.Restrictions.ExpireTime)
config.Datastore.MySQL.TrackAccountMessages = config.History.Retention.EnableAccountIndexing
config.Server.Cloaks.Initialize()
if config.Server.Cloaks.Enabled {
@ -1133,6 +1140,10 @@ func LoadConfig(filename string) (config *Config, err error) {
return config, nil
}
func (config *Config) getOutputPath(filename string) string {
return filepath.Join(config.Server.OutputPath, filename)
}
// setISupport sets up our RPL_ISUPPORT reply.
func (config *Config) generateISupport() (err error) {
maxTargetsString := strconv.Itoa(maxTargets)

57
irc/handlers.go
View File

@ -676,7 +676,7 @@ func debugHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Res
rb.Notice(fmt.Sprintf("num goroutines: %d", count))
case "PROFILEHEAP":
profFile := "oragono.mprof"
profFile := server.Config().getOutputPath("oragono.mprof")
file, err := os.Create(profFile)
if err != nil {
rb.Notice(fmt.Sprintf("error: %s", err))
@ -687,7 +687,7 @@ func debugHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Res
rb.Notice(fmt.Sprintf("written to %s", profFile))
case "STARTCPUPROFILE":
profFile := "oragono.prof"
profFile := server.Config().getOutputPath("oragono.prof")
file, err := os.Create(profFile)
if err != nil {
rb.Notice(fmt.Sprintf("error: %s", err))
@ -934,50 +934,17 @@ func historyHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *R
return false
}
target := msg.Params[0]
if strings.ToLower(target) == "me" {
target = "*"
}
channel, sequence, err := server.GetHistorySequence(nil, client, target)
items, channel, err := easySelectHistory(server, client, msg.Params)
if sequence == nil || err != nil {
// whatever
rb.Add(nil, server.name, ERR_NOSUCHCHANNEL, client.Nick(), utils.SafeErrorParam(target), client.t("No such channel"))
if err == errNoSuchChannel {
rb.Add(nil, server.name, ERR_NOSUCHCHANNEL, client.Nick(), utils.SafeErrorParam(msg.Params[0]), client.t("No such channel"))
return false
} else if err != nil {
rb.Add(nil, server.name, ERR_UNKNOWNERROR, client.Nick(), msg.Command, client.t("Could not retrieve history"))
return false
}
var duration time.Duration
maxChathistoryLimit := config.History.ChathistoryMax
limit := 100
if maxChathistoryLimit < limit {
limit = maxChathistoryLimit
}
if len(msg.Params) > 1 {
providedLimit, err := strconv.Atoi(msg.Params[1])
if err == nil && providedLimit != 0 {
limit = providedLimit
if maxChathistoryLimit < limit {
limit = maxChathistoryLimit
}
} else if err != nil {
duration, err = time.ParseDuration(msg.Params[1])
if err == nil {
limit = maxChathistoryLimit
}
}
}
var items []history.Item
if duration == 0 {
items, _, err = sequence.Between(history.Selector{}, history.Selector{}, limit)
} else {
now := time.Now().UTC()
start := history.Selector{Time: now}
end := history.Selector{Time: now.Add(-duration)}
items, _, err = sequence.Between(start, end, limit)
}
if err == nil && len(items) != 0 {
if len(items) != 0 {
if channel != nil {
channel.replayHistoryItems(rb, items, false)
} else {
@ -1533,12 +1500,12 @@ func cmodeHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *Res
// process mode changes, include list operations (an empty set of changes does a list)
applied := channel.ApplyChannelModeChanges(client, msg.Command == "SAMODE", changes, rb)
details := client.Details()
announceCmodeChanges(channel, applied, details.nickMask, details.accountName, rb)
announceCmodeChanges(channel, applied, details.nickMask, details.accountName, details.account, rb)
return false
}
func announceCmodeChanges(channel *Channel, applied modes.ModeChanges, source, accountName string, rb *ResponseBuffer) {
func announceCmodeChanges(channel *Channel, applied modes.ModeChanges, source, accountName, account string, rb *ResponseBuffer) {
// send out changes
if len(applied) > 0 {
message := utils.MakeMessage("")
@ -1560,7 +1527,7 @@ func announceCmodeChanges(channel *Channel, applied modes.ModeChanges, source, a
Nick: source,
AccountName: accountName,
Message: message,
})
}, account)
}
}

26
irc/history/history.go
View File

@ -284,6 +284,32 @@ func (list *Buffer) matchInternal(predicate Predicate, ascending bool, limit int
return
}
// Delete deletes messages matching some predicate.
func (list *Buffer) Delete(predicate Predicate) (count int) {
list.Lock()
defer list.Unlock()
if list.start == -1 || len(list.buffer) == 0 {
return
}
pos := list.start
stop := list.prev(list.end)
for {
if predicate(&list.buffer[pos]) {
list.buffer[pos] = Item{}
count++
}
if pos == stop {
break
}
pos = list.next(pos)
}
return
}
// latest returns the items most recently added, up to `limit`. If `limit` is 0,
// it returns all items.
func (list *Buffer) latest(limit int) (results []Item) {

244
irc/histserv.go Normal file
View File

@ -0,0 +1,244 @@
// Copyright (c) 2020 Shivaram Lingamneni <slingamn@cs.stanford.edu>
// released under the MIT license
package irc
import (
"bufio"
"fmt"
"os"
"runtime/debug"
"strconv"
"strings"
"time"
"github.com/oragono/oragono/irc/history"
"github.com/oragono/oragono/irc/utils"
)
const (
histservHelp = `HistServ provides commands related to history.`
histServMask = "HistServ!HistServ@localhost"
)
func histservEnabled(config *Config) bool {
return config.History.Enabled
}
func historyComplianceEnabled(config *Config) bool {
return config.History.Enabled && config.History.Persistent.Enabled && config.History.Retention.EnableAccountIndexing
}
var (
histservCommands = map[string]*serviceCommand{
"forget": {
handler: histservForgetHandler,
help: `Syntax: $bFORGET <account>$b
FORGET deletes all history messages sent by an account.`,
helpShort: `$bFORGET$b deletes all history messages sent by an account.`,
capabs: []string{"history"},
enabled: histservEnabled,
minParams: 1,
maxParams: 1,
},
"delete": {
handler: histservDeleteHandler,
help: `Syntax: $bDELETE [target] <msgid>$b
DELETE deletes an individual message by its msgid. The target is a channel
name or nickname; depending on the history implementation, this may or may not
be necessary to locate the message.`,
helpShort: `$bDELETE$b deletes an individual message by its msgid.`,
enabled: histservEnabled,
minParams: 1,
maxParams: 2,
},
"export": {
handler: histservExportHandler,
help: `Syntax: $bEXPORT <account>$b
EXPORT exports all messages sent by an account as JSON. This can be used at
the request of the account holder.`,
helpShort: `$bEXPORT$b exports all messages sent by an account as JSON.`,
enabled: historyComplianceEnabled,
capabs: []string{"history"},
minParams: 1,
maxParams: 1,
},
"play": {
handler: histservPlayHandler,
help: `Syntax: $bPLAY <target> [limit]$b
PLAY plays back history messages, rendering them into direct messages from
HistServ. 'target' is a channel name (or 'me' for direct messages), and 'limit'
is a message count or a time duration. Note that message playback may be
incomplete or degraded, relative to direct playback from /HISTORY or
CHATHISTORY.`,
helpShort: `$bPLAY$b plays back history messages.`,
enabled: histservEnabled,
minParams: 1,
maxParams: 2,
},
}
)
// histNotice sends the client a notice from HistServ
func histNotice(rb *ResponseBuffer, text string) {
rb.Add(nil, histServMask, "NOTICE", rb.target.Nick(), text)
}
func histservForgetHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) {
accountName := server.accounts.AccountToAccountName(params[0])
if accountName == "" {
histNotice(rb, client.t("Could not look up account name, proceeding anyway"))
accountName = params[0]
}
server.ForgetHistory(accountName)
histNotice(rb, fmt.Sprintf(client.t("Enqueued account %s for message deletion"), accountName))
}
func histservDeleteHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) {
var target, msgid string
if len(params) == 1 {
msgid = params[0]
} else {
target, msgid = params[0], params[1]
}
accountName := "*"
hasPrivs := client.HasRoleCapabs("history")
if !hasPrivs {
accountName = client.AccountName()
if !(server.Config().History.Retention.AllowIndividualDelete && accountName != "*") {
hsNotice(rb, client.t("Insufficient privileges"))
return
}
}
err := server.DeleteMessage(target, msgid, accountName)
if err == nil {
hsNotice(rb, client.t("Successfully deleted message"))
} else {
if hasPrivs {
hsNotice(rb, fmt.Sprintf(client.t("Error deleting message: %v"), err))
} else {
hsNotice(rb, client.t("Could not delete message"))
}
}
}
func histservExportHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) {
cfAccount, err := CasefoldName(params[0])
if err != nil {
histNotice(rb, client.t("Invalid account name"))
return
}
config := server.Config()
// don't include the account name in the filename because of escaping concerns
filename := fmt.Sprintf("%s-%s.json", utils.GenerateSecretToken(), time.Now().UTC().Format(IRCv3TimestampFormat))
pathname := config.getOutputPath(filename)
outfile, err := os.Create(pathname)
if err != nil {
hsNotice(rb, fmt.Sprintf(client.t("Error opening export file: %v"), err))
} else {
hsNotice(rb, fmt.Sprintf(client.t("Started exporting data for account %[1]s to file %[2]s"), cfAccount, filename))
}
go histservExportAndNotify(server, cfAccount, outfile, filename, client.Nick())
}
func histservExportAndNotify(server *Server, cfAccount string, outfile *os.File, filename, alertNick string) {
defer func() {
if r := recover(); r != nil {
server.logger.Error("history",
fmt.Sprintf("Panic in history export routine: %v\n%s", r, debug.Stack()))
}
}()
defer outfile.Close()
writer := bufio.NewWriter(outfile)
defer writer.Flush()
server.historyDB.Export(cfAccount, writer)
client := server.clients.Get(alertNick)
if client != nil && client.HasRoleCapabs("history") {
client.Send(nil, histServMask, "NOTICE", client.Nick(), fmt.Sprintf(client.t("Data export for %[1]s completed and written to %[2]s"), cfAccount, filename))
}
}
func histservPlayHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) {
items, _, err := easySelectHistory(server, client, params)
if err != nil {
hsNotice(rb, client.t("Could not retrieve history"))
return
}
playMessage := func(timestamp time.Time, nick, message string) {
hsNotice(rb, fmt.Sprintf("%s <%s> %s", timestamp.Format("15:04:05"), stripMaskFromNick(nick), message))
}
for _, item := range items {
// TODO: support a few more of these, maybe JOIN/PART/QUIT
if item.Type != history.Privmsg && item.Type != history.Notice {
continue
}
if len(item.Message.Split) == 0 {
playMessage(item.Message.Time, item.Nick, item.Message.Message)
} else {
for _, pair := range item.Message.Split {
playMessage(item.Message.Time, item.Nick, pair.Message)
}
}
}
hsNotice(rb, client.t("End of history playback"))
}
// handles parameter parsing and history queries for /HISTORY and /HISTSERV PLAY
func easySelectHistory(server *Server, client *Client, params []string) (items []history.Item, channel *Channel, err error) {
target := params[0]
if strings.ToLower(target) == "me" {
target = "*"
}
channel, sequence, err := server.GetHistorySequence(nil, client, target)
if sequence == nil || err != nil {
return nil, nil, errNoSuchChannel
}
var duration time.Duration
maxChathistoryLimit := server.Config().History.ChathistoryMax
limit := 100
if maxChathistoryLimit < limit {
limit = maxChathistoryLimit
}
if len(params) > 1 {
providedLimit, err := strconv.Atoi(params[1])
if err == nil && providedLimit != 0 {
limit = providedLimit
if maxChathistoryLimit < limit {
limit = maxChathistoryLimit
}
} else if err != nil {
duration, err = time.ParseDuration(params[1])
if err == nil {
limit = maxChathistoryLimit
}
}
}
if duration == 0 {
items, _, err = sequence.Between(history.Selector{}, history.Selector{}, limit)
} else {
now := time.Now().UTC()
start := history.Selector{Time: now}
end := history.Selector{Time: now.Add(-duration)}
items, _, err = sequence.Between(start, end, limit)
}
return
}

3
irc/mysql/config.go
View File

@ -18,5 +18,6 @@ type Config struct {
Timeout time.Duration
// XXX these are copied from elsewhere in the config:
ExpireTime time.Duration
ExpireTime time.Duration
TrackAccountMessages bool
}

391
irc/mysql/history.go
View File

@ -7,7 +7,10 @@ import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"runtime/debug"
"sync"
"sync/atomic"
@ -19,6 +22,10 @@ import (
"github.com/oragono/oragono/irc/utils"
)
var (
ErrDisallowed = errors.New("disallowed")
)
const (
// maximum length in bytes of any message target (nickname or channel name) in its
// canonicalized (i.e., casefolded) state:
@ -27,30 +34,46 @@ const (
// latest schema of the db
latestDbSchema = "2"
keySchemaVersion = "db.version"
cleanupRowLimit = 50
cleanupPauseTime = 10 * time.Minute
// minor version indicates rollback-safe upgrades, i.e.,
// you can downgrade oragono and everything will work
latestDbMinorVersion = "1"
keySchemaMinorVersion = "db.minorversion"
cleanupRowLimit = 50
cleanupPauseTime = 10 * time.Minute
)
type MySQL struct {
timeout int64
db *sql.DB
logger *logger.Manager
type e struct{}
insertHistory *sql.Stmt
insertSequence *sql.Stmt
insertConversation *sql.Stmt
type MySQL struct {
timeout int64
trackAccountMessages uint32
db *sql.DB
logger *logger.Manager
insertHistory *sql.Stmt
insertSequence *sql.Stmt
insertConversation *sql.Stmt
insertAccountMessage *sql.Stmt
stateMutex sync.Mutex
config Config
wakeForgetter chan e
}
func (mysql *MySQL) Initialize(logger *logger.Manager, config Config) {
mysql.logger = logger
mysql.wakeForgetter = make(chan e, 1)
mysql.SetConfig(config)
}
func (mysql *MySQL) SetConfig(config Config) {
atomic.StoreInt64(&mysql.timeout, int64(config.Timeout))
var trackAccountMessages uint32
if config.TrackAccountMessages {
trackAccountMessages = 1
}
atomic.StoreUint32(&mysql.trackAccountMessages, trackAccountMessages)
mysql.stateMutex.Lock()
mysql.config = config
mysql.stateMutex.Unlock()
@ -85,6 +108,7 @@ func (m *MySQL) Open() (err error) {
}
go m.cleanupLoop()
go m.forgetLoop()
return nil
}
@ -109,14 +133,35 @@ func (mysql *MySQL) fixSchemas() (err error) {
if err != nil {
return
}
_, err = mysql.db.Exec(`insert into metadata (key_name, value) values (?, ?);`, keySchemaMinorVersion, latestDbMinorVersion)
if err != nil {
return
}
return
} else if err == nil && schema != latestDbSchema {
// TODO figure out what to do about schema changes
return &utils.IncompatibleSchemaError{CurrentVersion: schema, RequiredVersion: latestDbSchema}
} else {
} else if err != nil {
return err
}
return nil
var minorVersion string
err = mysql.db.QueryRow(`select value from metadata where key_name = ?;`, keySchemaMinorVersion).Scan(&minorVersion)
if err == sql.ErrNoRows {
// XXX for now, the only minor version upgrade is the account tracking tables
err = mysql.createComplianceTables()
if err != nil {
return
}
_, err = mysql.db.Exec(`insert into metadata (key_name, value) values (?, ?);`, keySchemaMinorVersion, latestDbMinorVersion)
if err != nil {
return
}
} else if err == nil && minorVersion != latestDbMinorVersion {
// TODO: if minorVersion < latestDbMinorVersion, upgrade,
// if latestDbMinorVersion < minorVersion, ignore because backwards compatible
}
return
}
func (mysql *MySQL) createTables() (err error) {
@ -155,6 +200,32 @@ func (mysql *MySQL) createTables() (err error) {
return err
}
err = mysql.createComplianceTables()
if err != nil {
return err
}
return nil
}
func (mysql *MySQL) createComplianceTables() (err error) {
_, err = mysql.db.Exec(fmt.Sprintf(`CREATE TABLE account_messages (
history_id BIGINT UNSIGNED NOT NULL PRIMARY KEY,
account VARBINARY(%[1]d) NOT NULL,
KEY (account, history_id)
) CHARSET=ascii COLLATE=ascii_bin;`, MaxTargetLength))
if err != nil {
return err
}
_, err = mysql.db.Exec(fmt.Sprintf(`CREATE TABLE forget (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
account VARBINARY(%[1]d) NOT NULL
) CHARSET=ascii COLLATE=ascii_bin;`, MaxTargetLength))
if err != nil {
return err
}
return nil
}
@ -191,7 +262,10 @@ func (mysql *MySQL) cleanupLoop() {
}
func (mysql *MySQL) doCleanup(age time.Duration) (count int, err error) {
ids, maxNanotime, err := mysql.selectCleanupIDs(age)
ctx, cancel := context.WithTimeout(context.Background(), cleanupPauseTime)
defer cancel()
ids, maxNanotime, err := mysql.selectCleanupIDs(ctx, age)
if len(ids) == 0 {
mysql.logger.Debug("mysql", "found no rows to clean up")
return
@ -199,6 +273,10 @@ func (mysql *MySQL) doCleanup(age time.Duration) (count int, err error) {
mysql.logger.Debug("mysql", fmt.Sprintf("deleting %d history rows, max age %s", len(ids), utils.NanoToTimestamp(maxNanotime)))
return len(ids), mysql.deleteHistoryIDs(ctx, ids)
}
func (mysql *MySQL) deleteHistoryIDs(ctx context.Context, ids []uint64) (err error) {
// can't use ? binding for a variable number of arguments, build the IN clause manually
var inBuf bytes.Buffer
inBuf.WriteByte('(')
@ -210,25 +288,30 @@ func (mysql *MySQL) doCleanup(age time.Duration) (count int, err error) {
}
inBuf.WriteRune(')')
_, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM conversations WHERE history_id in %s;`, inBuf.Bytes()))
_, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM conversations WHERE history_id in %s;`, inBuf.Bytes()))
if err != nil {
return
}
_, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM sequence WHERE history_id in %s;`, inBuf.Bytes()))
_, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM sequence WHERE history_id in %s;`, inBuf.Bytes()))
if err != nil {
return
}
_, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM history WHERE id in %s;`, inBuf.Bytes()))
if mysql.isTrackingAccountMessages() {
_, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM account_messages WHERE history_id in %s;`, inBuf.Bytes()))
if err != nil {
return
}
}
_, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM history WHERE id in %s;`, inBuf.Bytes()))
if err != nil {
return
}
count = len(ids)
return
}
func (mysql *MySQL) selectCleanupIDs(age time.Duration) (ids []uint64, maxNanotime int64, err error) {
rows, err := mysql.db.Query(`
func (mysql *MySQL) selectCleanupIDs(ctx context.Context, age time.Duration) (ids []uint64, maxNanotime int64, err error) {
rows, err := mysql.db.QueryContext(ctx, `
SELECT history.id, sequence.nanotime
FROM history
LEFT JOIN sequence ON history.id = sequence.history_id
@ -266,6 +349,109 @@ func (mysql *MySQL) selectCleanupIDs(age time.Duration) (ids []uint64, maxNanoti
return
}
// wait for forget queue items and process them one by one
func (mysql *MySQL) forgetLoop() {
defer func() {
if r := recover(); r != nil {
mysql.logger.Error("mysql",
fmt.Sprintf("Panic in forget routine: %v\n%s", r, debug.Stack()))
time.Sleep(cleanupPauseTime)
go mysql.forgetLoop()
}
}()
for {
for {
found, err := mysql.doForget()
mysql.logError("error processing forget", err)
if err != nil {
time.Sleep(cleanupPauseTime)
}
if !found {
break
}
}
<-mysql.wakeForgetter
}
}
// dequeue an item from the forget queue and process it
func (mysql *MySQL) doForget() (found bool, err error) {
id, account, err := func() (id int64, account string, err error) {
ctx, cancel := context.WithTimeout(context.Background(), cleanupPauseTime)
defer cancel()
row := mysql.db.QueryRowContext(ctx,
`SELECT forget.id, forget.account FROM forget LIMIT 1;`)
err = row.Scan(&id, &account)
if err == sql.ErrNoRows {
return 0, "", nil
}
return
}()
if err != nil || account == "" {
return false, err
}
found = true
var count int
for {
start := time.Now()
count, err = mysql.doForgetIteration(account)
elapsed := time.Since(start)
if err != nil {
return true, err
}
if count == 0 {
break
}
time.Sleep(elapsed)
}
mysql.logger.Debug("mysql", "forget complete for account", account)
ctx, cancel := context.WithTimeout(context.Background(), cleanupPauseTime)
defer cancel()
_, err = mysql.db.ExecContext(ctx, `DELETE FROM forget where id = ?;`, id)
return
}
func (mysql *MySQL) doForgetIteration(account string) (count int, err error) {
ctx, cancel := context.WithTimeout(context.Background(), cleanupPauseTime)
defer cancel()
rows, err := mysql.db.QueryContext(ctx, `
SELECT account_messages.history_id
FROM account_messages
WHERE account_messages.account = ?
LIMIT ?;`, account, cleanupRowLimit)
if err != nil {
return
}
defer rows.Close()
var ids []uint64
for rows.Next() {
var id uint64
err = rows.Scan(&id)
if err != nil {
return
}
ids = append(ids, id)
}
if len(ids) == 0 {
return
}
mysql.logger.Debug("mysql", fmt.Sprintf("deleting %d history rows from account %s", len(ids), account))
err = mysql.deleteHistoryIDs(ctx, ids)
return len(ids), err
}
func (mysql *MySQL) prepareStatements() (err error) {
mysql.insertHistory, err = mysql.db.Prepare(`INSERT INTO history
(data, msgid) VALUES (?, ?);`)
@ -282,6 +468,11 @@ func (mysql *MySQL) prepareStatements() (err error) {
if err != nil {
return
}
mysql.insertAccountMessage, err = mysql.db.Prepare(`INSERT INTO account_messages
(history_id, account) VALUES (?, ?);`)
if err != nil {
return
}
return
}
@ -290,6 +481,10 @@ func (mysql *MySQL) getTimeout() time.Duration {
return time.Duration(atomic.LoadInt64(&mysql.timeout))
}
func (mysql *MySQL) isTrackingAccountMessages() bool {
return atomic.LoadUint32(&mysql.trackAccountMessages) != 0
}
func (mysql *MySQL) logError(context string, err error) (quit bool) {
if err != nil {
mysql.logger.Error("mysql", context, err.Error())
@ -298,7 +493,27 @@ func (mysql *MySQL) logError(context string, err error) (quit bool) {
return false
}
func (mysql *MySQL) AddChannelItem(target string, item history.Item) (err error) {
func (mysql *MySQL) Forget(account string) {
if mysql.db == nil || account == "" {
return
}
ctx, cancel := context.WithTimeout(context.Background(), mysql.getTimeout())
defer cancel()
_, err := mysql.db.ExecContext(ctx, `INSERT INTO forget (account) VALUES (?);`, account)
if mysql.logError("can't insert into forget table", err) {
return
}
// wake up the forget goroutine if it's blocked:
select {
case mysql.wakeForgetter <- e{}:
default:
}
}
func (mysql *MySQL) AddChannelItem(target string, item history.Item, account string) (err error) {
if mysql.db == nil {
return
}
@ -316,6 +531,15 @@ func (mysql *MySQL) AddChannelItem(target string, item history.Item) (err error)
}
err = mysql.insertSequenceEntry(ctx, target, item.Message.Time.UnixNano(), id)
if err != nil {
return
}
err = mysql.insertAccountMessageEntry(ctx, id, account)
if err != nil {
return
}
return
}
@ -354,6 +578,15 @@ func (mysql *MySQL) insertBase(ctx context.Context, item history.Item) (id int64
return
}
func (mysql *MySQL) insertAccountMessageEntry(ctx context.Context, id int64, account string) (err error) {
if account == "" || !mysql.isTrackingAccountMessages() {
return
}
_, err = mysql.insertAccountMessage.ExecContext(ctx, id, account)
mysql.logError("could not insert account-message entry", err)
return
}
func (mysql *MySQL) AddDirectMessage(sender, senderAccount, recipient, recipientAccount string, item history.Item) (err error) {
if mysql.db == nil {
return
@ -399,10 +632,102 @@ func (mysql *MySQL) AddDirectMessage(sender, senderAccount, recipient, recipient
}
}
err = mysql.insertAccountMessageEntry(ctx, id, senderAccount)
if err != nil {
return
}
return
}
func (mysql *MySQL) msgidToTime(ctx context.Context, msgid string) (result time.Time, err error) {
// note that accountName is the unfolded name
func (mysql *MySQL) DeleteMsgid(msgid, accountName string) (err error) {
if mysql.db == nil {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), mysql.getTimeout())
defer cancel()
_, id, data, err := mysql.lookupMsgid(ctx, msgid, true)
if err != nil {
return
}
if accountName != "*" {
var item history.Item
err = unmarshalItem(data, &item)
// delete if the entry is corrupt
if err == nil && item.AccountName != accountName {
return ErrDisallowed
}
}
err = mysql.deleteHistoryIDs(ctx, []uint64{id})
mysql.logError("couldn't delete msgid", err)
return
}
func (mysql *MySQL) Export(account string, writer io.Writer) {
if mysql.db == nil {
return
}
var err error
var lastSeen uint64
for {
rows := func() (count int) {
ctx, cancel := context.WithTimeout(context.Background(), cleanupPauseTime)
defer cancel()
rows, rowsErr := mysql.db.QueryContext(ctx, `
SELECT account_messages.history_id, history.data, sequence.target FROM account_messages
INNER JOIN history ON history.id = account_messages.history_id
INNER JOIN sequence ON account_messages.history_id = sequence.history_id
WHERE account_messages.account = ? AND account_messages.history_id > ?
LIMIT ?`, account, lastSeen, cleanupRowLimit)
if rowsErr != nil {
err = rowsErr
return
}
defer rows.Close()
for rows.Next() {
var id uint64
var blob, jsonBlob []byte
var target string
var item history.Item
err = rows.Scan(&id, &blob, &target)
if err != nil {
return
}
err = unmarshalItem(blob, &item)
if err != nil {
return
}
item.CfCorrespondent = target
jsonBlob, err = json.Marshal(item)
if err != nil {
return
}
count++
if lastSeen < id {
lastSeen = id
}
writer.Write(jsonBlob)
writer.Write([]byte{'\n'})
}
return
}()
if rows == 0 || err != nil {
break
}
}
mysql.logError("could not export history", err)
return
}
func (mysql *MySQL) lookupMsgid(ctx context.Context, msgid string, includeData bool) (result time.Time, id uint64, data []byte, err error) {
// in theory, we could optimize out a roundtrip to the database by using a subquery instead:
// sequence.nanotime > (
// SELECT sequence.nanotime FROM sequence, history
@ -415,15 +740,27 @@ func (mysql *MySQL) msgidToTime(ctx context.Context, msgid string) (result time.
if err != nil {
return
}
row := mysql.db.QueryRowContext(ctx, `
SELECT sequence.nanotime FROM sequence
cols := `sequence.nanotime`
if includeData {
cols = `sequence.nanotime, sequence.history_id, history.data`
}
row := mysql.db.QueryRowContext(ctx, fmt.Sprintf(`
SELECT %s FROM sequence
INNER JOIN history ON history.id = sequence.history_id
WHERE history.msgid = ? LIMIT 1;`, decoded)
WHERE history.msgid = ? LIMIT 1;`, cols), decoded)
var nanotime int64
err = row.Scan(&nanotime)
if mysql.logError("could not resolve msgid to time", err) {
if !includeData {
err = row.Scan(&nanotime)
} else {
err = row.Scan(&nanotime, &id, &data)
}
if err != sql.ErrNoRows {
mysql.logError("could not resolve msgid to time", err)
}
if err != nil {
return
}
result = time.Unix(0, nanotime).UTC()
return
}
@ -519,14 +856,14 @@ func (s *mySQLHistorySequence) Between(start, end history.Selector, limit int) (
startTime := start.Time
if start.Msgid != "" {
startTime, err = s.mysql.msgidToTime(ctx, start.Msgid)
startTime, _, _, err = s.mysql.lookupMsgid(ctx, start.Msgid, false)
if err != nil {
return nil, false, err
}
}
endTime := end.Time
if end.Msgid != "" {
endTime, err = s.mysql.msgidToTime(ctx, end.Msgid)
endTime, _, _, err = s.mysql.lookupMsgid(ctx, end.Msgid, false)
if err != nil {
return nil, false, err
}

2
irc/nickname.go
View File

@ -80,7 +80,7 @@ func performNickChange(server *Server, client *Client, target *Client, session *
}
for _, channel := range client.Channels() {
channel.AddHistoryItem(histItem)
channel.AddHistoryItem(histItem, details.account)
}
if target.Registered() {

2
irc/roleplay.go
View File

@ -91,7 +91,7 @@ func sendRoleplayMessage(server *Server, client *Client, source string, targetSt
Type: history.Privmsg,
Message: splitMessage,
Nick: source,
})
}, client.Account())
} else {
target, err := CasefoldName(targetString)
user := server.clients.Get(target)

70
irc/server.go
View File

@ -879,6 +879,76 @@ func (server *Server) GetHistorySequence(providedChannel *Channel, client *Clien
return
}
func (server *Server) ForgetHistory(accountName string) {
// sanity check
if accountName == "*" {
return
}
config := server.Config()
if !config.History.Enabled {
return
}
if cfAccount, err := CasefoldName(accountName); err == nil {
server.historyDB.Forget(cfAccount)
}
persistent := config.History.Persistent
if persistent.Enabled && persistent.UnregisteredChannels && persistent.RegisteredChannels == PersistentMandatory && persistent.DirectMessages == PersistentMandatory {
return
}
predicate := func(item *history.Item) bool { return item.AccountName == accountName }
for _, channel := range server.channels.Channels() {
channel.history.Delete(predicate)
}
for _, client := range server.clients.AllClients() {
client.history.Delete(predicate)
}
}
// deletes a message. target is a hint about what buffer it's in (not required for
// persistent history, where all the msgids are indexed together). if accountName
// is anything other than "*", it must match the recorded AccountName of the message
func (server *Server) DeleteMessage(target, msgid, accountName string) (err error) {
config := server.Config()
var hist *history.Buffer
if target != "" {
if target[0] == '#' {
channel := server.channels.Get(target)
if channel != nil {
if status, _ := channel.historyStatus(config); status == HistoryEphemeral {
hist = &channel.history
}
}
} else {
client := server.clients.Get(target)
if client != nil {
if status, _ := client.historyStatus(config); status == HistoryEphemeral {
hist = &client.history
}
}
}
}
if hist == nil {
err = server.historyDB.DeleteMsgid(msgid, accountName)
} else {
count := hist.Delete(func(item *history.Item) bool {
return item.Message.Msgid == msgid && (accountName == "*" || item.AccountName == accountName)
})
if count == 0 {
err = errNoop
}
}
return
}
// elistMatcher takes and matches ELIST conditions
type elistMatcher struct {
MinClientsActive bool

7
irc/services.go
View File

@ -82,6 +82,13 @@ var OragonoServices = map[string]*ircService{
Commands: hostservCommands,
HelpBanner: hostservHelp,
},
"histserv": {
Name: "HistServ",
ShortName: "HISTSERV",
CommandAliases: []string{"HISTSERV"},
Commands: histservCommands,
HelpBanner: histservHelp,
},
}
// all service commands at the protocol level, by uppercase command name

18
oragono.yaml
View File

@ -280,6 +280,10 @@ server:
secure-nets:
# - "10.0.0.0/8"
# oragono will write files to disk under certain circumstances, e.g.,
# CPU profiling or data export. by default, these files will be written
# to the working directory. set this to customize:
# output-path: "/home/oragono/out"
# account options
accounts:
@ -577,6 +581,7 @@ oper-classes:
- "samode"
- "vhosts"
- "chanreg"
- "history"
# ircd operators
opers:
@ -772,7 +777,8 @@ roleplay:
# add the real nickname, in parentheses, to the end of every roleplay message?
add-suffix: true
# message history tracking, for the RESUME extension and possibly other uses in future
# history message storage: this is used by CHATHISTORY, HISTORY, znc.in/playback,
# various autoreplay features, and the resume extension
history:
# should we store messages for later playback?
# by default, messages are stored in RAM only; they do not persist
@ -841,3 +847,13 @@ history:
# if you enable this, strict nickname reservation is strongly recommended
# as well.
direct-messages: "opt-out"
# options to control how messages are stored and deleted:
retention:
# allow users to delete their own messages from history?
allow-individual-delete: false
# if persistent history is enabled, create additional index tables,
# allowing deletion of JSON export of an account's messages. this
# may be needed for compliance with data privacy regulations.
enable-account-indexing: false