From ae69ef5cd68ee35f15697d5ca1455737271ca698 Mon Sep 17 00:00:00 2001 From: Daniel Oaks Date: Sat, 17 Sep 2016 21:23:04 +1000 Subject: [PATCH] db: Remove SQLite db, hopefully looking up clients still works.Channel persistence is broken by this, will fix it later. --- CHANGELOG.md | 2 + irc/channel.go | 43 +++++++------- irc/client_lookup_set.go | 117 ++++++++------------------------------- irc/config.go | 6 +- irc/database.go | 53 +++++------------- irc/server.go | 66 +++++++++++----------- oragono.go | 8 +-- 7 files changed, 98 insertions(+), 197 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02db8f9b..3244095f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Initial release of Oragono! ### Added * Added YAML config file format. +* Added buntdb key-value store for persistent data. * Added native SSL/TLS support (thanks to @edmand). * Added ability to generate testing certificates from the command line. * Added support for looking up usernames with [ident](https://tools.ietf.org/html/rfc1413) on client connection. @@ -35,6 +36,7 @@ Initial release of Oragono! ### Removed * Removed gitconfig configuration format [replaced with YAML]. +* Removed sqlite database [replaced with buntdb key-value store]. * Removed `THEATER` command (it broke and I'm not that interested in putting the work in to get it working again with the aim of this project. PRs accepted). ### Fixed diff --git a/irc/channel.go b/irc/channel.go index a121847f..a95bebc7 100644 --- a/irc/channel.go +++ b/irc/channel.go @@ -424,26 +424,31 @@ func (channel *Channel) applyModeMask(client *Client, mode ChannelMode, op ModeO } func (channel *Channel) Persist() (err error) { - if channel.flags[Persistent] { - //TODO(dan): Save topicSetBy/topicSetTime and createdTime - _, err = channel.server.db.Exec(` - INSERT OR REPLACE INTO channel - (name, flags, key, topic, user_limit, ban_list, except_list, - invite_list) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - channel.name.String(), channel.flags.String(), channel.key, - channel.topic, channel.userLimit, channel.lists[BanMask].String(), - channel.lists[ExceptMask].String(), channel.lists[InviteMask].String()) - } else { - _, err = channel.server.db.Exec(` - DELETE FROM channel WHERE name = ?`, channel.name.String()) - } - - if err != nil { - Log.error.Println("Channel.Persist:", channel, err) - } - return + + //TODO(dan): Fix persistence + /* + if channel.flags[Persistent] { + //TODO(dan): Save topicSetBy/topicSetTime and createdTime + _, err = channel.server.db.Exec(` + INSERT OR REPLACE INTO channel + (name, flags, key, topic, user_limit, ban_list, except_list, + invite_list) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + channel.name.String(), channel.flags.String(), channel.key, + channel.topic, channel.userLimit, channel.lists[BanMask].String(), + channel.lists[ExceptMask].String(), channel.lists[InviteMask].String()) + } else { + _, err = channel.server.db.Exec(` + DELETE FROM channel WHERE name = ?`, channel.name.String()) + } + + if err != nil { + Log.error.Println("Channel.Persist:", channel, err) + } + + return + */ } func (channel *Channel) Notice(client *Client, message string) { diff --git a/irc/client_lookup_set.go b/irc/client_lookup_set.go index e26d7725..b270601d 100644 --- a/irc/client_lookup_set.go +++ b/irc/client_lookup_set.go @@ -4,33 +4,23 @@ package irc import ( - "database/sql" "errors" - "log" "regexp" "strings" + + "github.com/DanielOaks/girc-go/ircmatch" ) var ( ErrNickMissing = errors.New("nick missing") ErrNicknameInUse = errors.New("nickname in use") ErrNicknameMismatch = errors.New("nickname mismatch") - wildMaskExpr = regexp.MustCompile(`\*|\?`) - likeQuoter = strings.NewReplacer( - `\`, `\\`, - `%`, `\%`, - `_`, `\_`, - `*`, `%`, - `?`, `_`) ) -func HasWildcards(mask string) bool { - return wildMaskExpr.MatchString(mask) -} - func ExpandUserHost(userhost Name) (expanded Name) { expanded = userhost // fill in missing wildcards for nicks + //TODO(dan): this would fail with dan@lol, do we want to accommodate that? if !strings.Contains(expanded.String(), "!") { expanded += "!*" } @@ -40,19 +30,13 @@ func ExpandUserHost(userhost Name) (expanded Name) { return } -func QuoteLike(userhost Name) string { - return likeQuoter.Replace(userhost.String()) -} - type ClientLookupSet struct { byNick map[Name]*Client - db *ClientDB } func NewClientLookupSet() *ClientLookupSet { return &ClientLookupSet{ byNick: make(map[Name]*Client), - db: NewClientDB(), } } @@ -68,7 +52,6 @@ func (clients *ClientLookupSet) Add(client *Client) error { return ErrNicknameInUse } clients.byNick[client.Nick().ToLower()] = client - clients.db.Add(client) return nil } @@ -80,101 +63,47 @@ func (clients *ClientLookupSet) Remove(client *Client) error { return ErrNicknameMismatch } delete(clients.byNick, client.nick.ToLower()) - clients.db.Remove(client) return nil } func (clients *ClientLookupSet) FindAll(userhost Name) (set ClientSet) { - userhost = ExpandUserHost(userhost) set = make(ClientSet) - rows, err := clients.db.db.Query( - `SELECT nickname FROM client WHERE userhost LIKE ? ESCAPE '\'`, - QuoteLike(userhost)) - if err != nil { - Log.error.Println("ClientLookupSet.FindAll.Query:", err) - return - } - for rows.Next() { - var sqlNickname string - err := rows.Scan(&sqlNickname) - if err != nil { - Log.error.Println("ClientLookupSet.FindAll.Scan:", err) - return + + userhost = ExpandUserHost(userhost) + matcher := ircmatch.MakeMatch(userhost.String()) + + var casemappedNickMask string + for _, client := range clients.byNick { + casemappedNickMask = NewName(client.nickMaskString).String() + if matcher.Match(casemappedNickMask) { + set.Add(client) } - nickname := Name(sqlNickname) - client := clients.Get(nickname) - if client == nil { - Log.error.Println("ClientLookupSet.FindAll: missing client:", nickname) - continue - } - set.Add(client) } - return + + return set } func (clients *ClientLookupSet) Find(userhost Name) *Client { userhost = ExpandUserHost(userhost) - row := clients.db.db.QueryRow( - `SELECT nickname FROM client WHERE userhost LIKE ? ESCAPE '\' LIMIT 1`, - QuoteLike(userhost)) - var nickname Name - err := row.Scan(&nickname) - if err != nil { - Log.error.Println("ClientLookupSet.Find:", err) - return nil - } - return clients.Get(nickname) -} + matcher := ircmatch.MakeMatch(userhost.String()) -// -// client db -// - -type ClientDB struct { - db *sql.DB -} - -func NewClientDB() *ClientDB { - db := &ClientDB{ - db: OpenDB(":memory:"), - } - stmts := []string{ - `CREATE TABLE client ( - nickname TEXT NOT NULL COLLATE NOCASE UNIQUE, - userhost TEXT NOT NULL COLLATE NOCASE, - UNIQUE (nickname, userhost) ON CONFLICT REPLACE)`, - `CREATE UNIQUE INDEX idx_nick ON client (nickname COLLATE NOCASE)`, - `CREATE UNIQUE INDEX idx_uh ON client (userhost COLLATE NOCASE)`, - } - for _, stmt := range stmts { - _, err := db.db.Exec(stmt) - if err != nil { - log.Fatal("NewClientDB: ", stmt, err) + var casemappedNickMask string + for _, client := range clients.byNick { + casemappedNickMask = NewName(client.nickMaskString).String() + if matcher.Match(casemappedNickMask) { + return client } } - return db -} -func (db *ClientDB) Add(client *Client) { - _, err := db.db.Exec(`INSERT INTO client (nickname, userhost) VALUES (?, ?)`, - client.Nick().String(), client.UserHost().String()) - if err != nil { - Log.error.Println("ClientDB.Add:", err) - } -} - -func (db *ClientDB) Remove(client *Client) { - _, err := db.db.Exec(`DELETE FROM client WHERE nickname = ?`, - client.Nick().String()) - if err != nil { - Log.error.Println("ClientDB.Remove:", err) - } + return nil } // // usermask to regexp // +//TODO(dan): move this over to generally using glob syntax instead? +// kinda more expected in normal ban/etc masks, though regex is useful (probably as an extban?) type UserMaskSet struct { masks map[Name]bool regexp *regexp.Regexp diff --git a/irc/config.go b/irc/config.go index cf335d6d..74110317 100644 --- a/irc/config.go +++ b/irc/config.go @@ -84,8 +84,7 @@ type Config struct { } Datastore struct { - Path string - SQLitePath string `yaml:"sqlite-path"` + Path string } Registration struct { @@ -152,9 +151,6 @@ func LoadConfig(filename string) (config *Config, err error) { if config.Datastore.Path == "" { return nil, errors.New("Datastore path missing") } - if config.Datastore.SQLitePath == "" { - return nil, errors.New("SQLite database path missing") - } if len(config.Server.Listen) == 0 { return nil, errors.New("Server listening addresses missing") } diff --git a/irc/database.go b/irc/database.go index 407a2a79..298d0fa8 100644 --- a/irc/database.go +++ b/irc/database.go @@ -4,79 +4,52 @@ package irc import ( - "database/sql" "encoding/base64" "fmt" "log" "os" - _ "github.com/mattn/go-sqlite3" "github.com/tidwall/buntdb" ) const ( + // 'version' of the database schema + keySchemaVersion = "db.version" // key for the primary salt used by the ircd keySalt = "crypto.salt" ) -func InitDB(buntpath string, path string) { +// InitDB creates the database. +func InitDB(path string) { // prepare kvstore db - os.Remove(buntpath) - store, err := buntdb.Open(buntpath) + //TODO(dan): fail if already exists instead? don't want to overwrite good data + os.Remove(path) + store, err := buntdb.Open(path) if err != nil { log.Fatal(fmt.Sprintf("Failed to open datastore: %s", err.Error())) } defer store.Close() err = store.Update(func(tx *buntdb.Tx) error { + // set base db salt salt, err := NewSalt() encodedSalt := base64.StdEncoding.EncodeToString(salt) if err != nil { log.Fatal("Could not generate cryptographically-secure salt for the user:", err.Error()) } tx.Set(keySalt, encodedSalt, nil) + + // set schema version + tx.Set(keySchemaVersion, "1", nil) return nil }) if err != nil { log.Fatal("Could not save bunt store:", err.Error()) } - - // prepare SQLite db - os.Remove(path) - db := OpenDB(path) - defer db.Close() - _, err = db.Exec(` - CREATE TABLE channel ( - name TEXT NOT NULL UNIQUE, - flags TEXT DEFAULT '', - key TEXT DEFAULT '', - topic TEXT DEFAULT '', - user_limit INTEGER DEFAULT 0, - ban_list TEXT DEFAULT '', - except_list TEXT DEFAULT '', - invite_list TEXT DEFAULT '')`) - if err != nil { - log.Fatal("initdb error: ", err) - } } +// UpgradeDB upgrades the datastore to the latest schema. func UpgradeDB(path string) { - db := OpenDB(path) - alter := `ALTER TABLE channel ADD COLUMN %s TEXT DEFAULT ''` - cols := []string{"ban_list", "except_list", "invite_list"} - for _, col := range cols { - _, err := db.Exec(fmt.Sprintf(alter, col)) - if err != nil { - log.Fatal("updatedb error: ", err) - } - } -} - -func OpenDB(path string) *sql.DB { - db, err := sql.Open("sqlite3", path) - if err != nil { - log.Fatal("open db error: ", err) - } - return db + return } diff --git a/irc/server.go b/irc/server.go index 7c6f284e..dd8fd6ed 100644 --- a/irc/server.go +++ b/irc/server.go @@ -8,7 +8,6 @@ package irc import ( "bufio" "crypto/tls" - "database/sql" "encoding/base64" "fmt" "log" @@ -39,7 +38,6 @@ type Server struct { clients *ClientLookupSet commands chan Command ctime time.Time - db *sql.DB store buntdb.DB idle chan *Client limits Limits @@ -79,7 +77,6 @@ func NewServer(config *Config) *Server { clients: NewClientLookupSet(), commands: make(chan Command), ctime: time.Now(), - db: OpenDB(config.Datastore.SQLitePath), idle: make(chan *Client), limits: Limits{ AwayLen: config.Limits.AwayLen, @@ -216,35 +213,38 @@ func loadChannelList(channel *Channel, list string, maskMode ChannelMode) { } func (server *Server) loadChannels() { - rows, err := server.db.Query(` - SELECT name, flags, key, topic, user_limit, ban_list, except_list, - invite_list - FROM channel`) - if err != nil { - log.Fatal("error loading channels: ", err) - } - for rows.Next() { - var name, flags, key, topic string - var userLimit uint64 - var banList, exceptList, inviteList string - err = rows.Scan(&name, &flags, &key, &topic, &userLimit, &banList, - &exceptList, &inviteList) - if err != nil { - log.Println("Server.loadChannels:", err) - continue - } + //TODO(dan): Fix channel persistence + /* + rows, err := server.db.Query(` + SELECT name, flags, key, topic, user_limit, ban_list, except_list, + invite_list + FROM channel`) + if err != nil { + log.Fatal("error loading channels: ", err) + } + for rows.Next() { + var name, flags, key, topic string + var userLimit uint64 + var banList, exceptList, inviteList string + err = rows.Scan(&name, &flags, &key, &topic, &userLimit, &banList, + &exceptList, &inviteList) + if err != nil { + log.Println("Server.loadChannels:", err) + continue + } - channel := NewChannel(server, NewName(name), false) - for _, flag := range flags { - channel.flags[ChannelMode(flag)] = true - } - channel.key = key - channel.topic = topic - channel.userLimit = userLimit - loadChannelList(channel, banList, BanMask) - loadChannelList(channel, exceptList, ExceptMask) - loadChannelList(channel, inviteList, InviteMask) - } + channel := NewChannel(server, NewName(name), false) + for _, flag := range flags { + channel.flags[ChannelMode(flag)] = true + } + channel.key = key + channel.topic = topic + channel.userLimit = userLimit + loadChannelList(channel, banList, BanMask) + loadChannelList(channel, exceptList, ExceptMask) + loadChannelList(channel, inviteList, InviteMask) + } + */ } func (server *Server) Shutdown() { @@ -253,9 +253,6 @@ func (server *Server) Shutdown() { client.Notice("Server is shutting down") } - if err := server.db.Close(); err != nil { - Log.error.Println("Server.Shutdown db.Close: error:", err) - } if err := server.store.Close(); err != nil { Log.error.Println("Server.Shutdown store.Close: error:", err) } @@ -263,7 +260,6 @@ func (server *Server) Shutdown() { func (server *Server) Run() { // defer closing db/store - defer server.db.Close() defer server.store.Close() done := false diff --git a/oragono.go b/oragono.go index 362d2ab2..f6e57d7b 100644 --- a/oragono.go +++ b/oragono.go @@ -54,11 +54,11 @@ Options: fmt.Print("\n") fmt.Println(encoded) } else if arguments["initdb"].(bool) { - irc.InitDB(config.Datastore.Path, config.Datastore.SQLitePath) - log.Println("databases initialized: ", config.Datastore.Path, config.Datastore.SQLitePath) + irc.InitDB(config.Datastore.Path) + log.Println("database initialized: ", config.Datastore.Path) } else if arguments["upgradedb"].(bool) { - irc.UpgradeDB(config.Datastore.SQLitePath) - log.Println("database upgraded: ", config.Datastore.SQLitePath) + irc.UpgradeDB(config.Datastore.Path) + log.Println("database upgraded: ", config.Datastore.Path) } else if arguments["mkcerts"].(bool) { log.Println("making self-signed certificates")