From 69fd3ac3244344d1c1398e288935b66c8f7cfcd4 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Mon, 16 Apr 2018 16:28:31 -0400 Subject: [PATCH 1/2] implement database auto-upgrades (#243) --- irc/config.go | 3 +- irc/database.go | 139 ++++++++++++++++++++++++++++++++++++++++-------- irc/server.go | 12 ++--- oragono.go | 5 +- oragono.yaml | 4 ++ 5 files changed, 131 insertions(+), 32 deletions(-) diff --git a/irc/config.go b/irc/config.go index ada0a578..3a1282ac 100644 --- a/irc/config.go +++ b/irc/config.go @@ -229,7 +229,8 @@ type Config struct { } Datastore struct { - Path string + Path string + AutoUpgrade *bool } Accounts AccountConfig diff --git a/irc/database.go b/irc/database.go index 1e1a3987..d92c2027 100644 --- a/irc/database.go +++ b/irc/database.go @@ -8,9 +8,11 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" "log" "os" "strings" + "time" "github.com/oragono/oragono/irc/modes" "github.com/oragono/oragono/irc/passwd" @@ -38,6 +40,22 @@ type SchemaChange struct { // maps an initial version to a schema change capable of upgrading it var schemaChanges map[string]SchemaChange +type incompatibleSchemaError struct { + currentVersion string + requiredVersion string +} + +func IncompatibleSchemaError(currentVersion string) (result *incompatibleSchemaError) { + return &incompatibleSchemaError{ + currentVersion: currentVersion, + requiredVersion: latestDbSchema, + } +} + +func (err *incompatibleSchemaError) Error() string { + return fmt.Sprintf("Database requires update. Expected schema v%s, got v%s", err.requiredVersion, err.currentVersion) +} + // InitDB creates the database. func InitDB(path string) { // prepare kvstore db @@ -69,36 +87,107 @@ func InitDB(path string) { } // OpenDatabase returns an existing database, performing a schema version check. -func OpenDatabase(path string) (*buntdb.DB, error) { - // open data store - db, err := buntdb.Open(path) +func OpenDatabase(config *Config) (*buntdb.DB, error) { + allowAutoupgrade := true + if config.Datastore.AutoUpgrade != nil { + allowAutoupgrade = *config.Datastore.AutoUpgrade + } + return openDatabaseInternal(config, allowAutoupgrade) +} + +// open the database, giving it at most one chance to auto-upgrade the schema +func openDatabaseInternal(config *Config, allowAutoupgrade bool) (db *buntdb.DB, err error) { + db, err = buntdb.Open(config.Datastore.Path) if err != nil { - return nil, err + return } - // check db version - err = db.View(func(tx *buntdb.Tx) error { - version, _ := tx.Get(keySchemaVersion) - if version != latestDbSchema { - return fmt.Errorf("Database must be updated. Expected schema v%s, got v%s", latestDbSchema, version) + defer func() { + if err != nil && db != nil { + db.Close() + db = nil } - return nil - }) + }() + // read the current version string + var version string + err = db.View(func(tx *buntdb.Tx) error { + version, err = tx.Get(keySchemaVersion) + return err + }) if err != nil { - // close the db - db.Close() - return nil, err + return } - return db, nil + if version == latestDbSchema { + // success + return + } + + // XXX quiesce the DB so we can be sure it's safe to make a backup copy + db.Close() + db = nil + if allowAutoupgrade { + err = performAutoUpgrade(version, config) + if err != nil { + return + } + // successful autoupgrade, let's try this again: + return openDatabaseInternal(config, false) + } else { + err = IncompatibleSchemaError(version) + return + } +} + +// implementation of `cp` (go should really provide this...) +func cpFile(src string, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + closeError := out.Close() + if err == nil { + err = closeError + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + return +} + +func performAutoUpgrade(currentVersion string, config *Config) (err error) { + path := config.Datastore.Path + log.Printf("attempting to auto-upgrade schema from version %s to %s\n", currentVersion, latestDbSchema) + timestamp := time.Now().UTC().Format("2006-01-02-15:04:05.000Z") + backupPath := fmt.Sprintf("%s.v%s.%s.bak", path, currentVersion, timestamp) + log.Printf("making a backup of current database at %s\n", backupPath) + err = cpFile(path, backupPath) + if err != nil { + return err + } + + err = UpgradeDB(config) + if err != nil { + // database upgrade is a single transaction, so we don't need to restore the backup; + // we can just delete it + os.Remove(backupPath) + } + return err } // UpgradeDB upgrades the datastore to the latest schema. -func UpgradeDB(config *Config) { +func UpgradeDB(config *Config) (err error) { store, err := buntdb.Open(config.Datastore.Path) if err != nil { - log.Fatal(fmt.Sprintf("Failed to open datastore: %s", err.Error())) + return err } defer store.Close() @@ -108,9 +197,14 @@ func UpgradeDB(config *Config) { version, _ = tx.Get(keySchemaVersion) change, schemaNeedsChange := schemaChanges[version] if !schemaNeedsChange { - break + if version == latestDbSchema { + // success! + break + } + // unable to upgrade to the desired version, roll back + return IncompatibleSchemaError(version) } - log.Println("attempting to update store from version " + version) + log.Println("attempting to update schema from version " + version) err := change.Changer(config, tx) if err != nil { return err @@ -119,16 +213,15 @@ func UpgradeDB(config *Config) { if err != nil { return err } - log.Println("successfully updated store to version " + change.TargetVersion) + log.Println("successfully updated schema to version " + change.TargetVersion) } return nil }) if err != nil { - log.Fatal("Could not update datastore:", err.Error()) + log.Println("database upgrade failed and was rolled back") } - - return + return err } func schemaChangeV1toV2(config *Config, tx *buntdb.Tx) error { diff --git a/irc/server.go b/irc/server.go index e6a5570b..9032c3ff 100644 --- a/irc/server.go +++ b/irc/server.go @@ -127,7 +127,6 @@ type Server struct { signals chan os.Signal snomasks *SnoManager store *buntdb.DB - storeFilename string stsEnabled bool webirc []webircConfig whoWas *WhoWasList @@ -746,7 +745,7 @@ func (server *Server) applyConfig(config *Config, initial bool) error { return fmt.Errorf("Maximum line length (linelen) cannot be changed after launching the server, rehash aborted") } else if server.name != config.Server.Name { return fmt.Errorf("Server name cannot be changed after launching the server, rehash aborted") - } else if server.storeFilename != config.Datastore.Path { + } else if server.config.Datastore.Path != config.Datastore.Path { return fmt.Errorf("Datastore path cannot be changed after launching the server, rehash aborted") } } @@ -966,10 +965,9 @@ func (server *Server) applyConfig(config *Config, initial bool) error { server.config = config server.configurableStateMutex.Unlock() - server.storeFilename = config.Datastore.Path - server.logger.Info("rehash", "Using datastore", server.storeFilename) + server.logger.Info("rehash", "Using datastore", config.Datastore.Path) if initial { - if err := server.loadDatastore(server.storeFilename); err != nil { + if err := server.loadDatastore(config); err != nil { return err } } @@ -1066,11 +1064,11 @@ func (server *Server) loadMOTD(motdPath string, useFormatting bool) error { return nil } -func (server *Server) loadDatastore(datastorePath string) error { +func (server *Server) loadDatastore(config *Config) error { // open the datastore and load server state for which it (rather than config) // is the source of truth - db, err := OpenDatabase(datastorePath) + db, err := OpenDatabase(config) if err == nil { server.store = db } else { diff --git a/oragono.go b/oragono.go index f0f99b40..61547131 100644 --- a/oragono.go +++ b/oragono.go @@ -84,7 +84,10 @@ Options: log.Println("database initialized: ", config.Datastore.Path) } } else if arguments["upgradedb"].(bool) { - irc.UpgradeDB(config) + err = irc.UpgradeDB(config) + if err != nil { + log.Fatal("Error while upgrading db:", err.Error()) + } if !arguments["--quiet"].(bool) { log.Println("database upgraded: ", config.Datastore.Path) } diff --git a/oragono.yaml b/oragono.yaml index 6c5aa757..7fca18a7 100644 --- a/oragono.yaml +++ b/oragono.yaml @@ -341,6 +341,10 @@ debug: datastore: # path to the datastore path: ircd.db + # if the database schema requires an upgrade, `autoupgrade` (which defaults to true) + # will attempt to perform it automatically on startup. the database will be backed + # up, and if the upgrade fails, the original database will be restored. + autoupgrade: true # languages config languages: From 3db71415c95c73d11c4971588e043d85310d9317 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Fri, 20 Apr 2018 03:57:48 -0400 Subject: [PATCH 2/2] review fixes --- irc/config.go | 2 +- irc/database.go | 33 +++------------------------------ irc/utils/os.go | 31 +++++++++++++++++++++++++++++++ oragono.yaml | 5 +++-- 4 files changed, 38 insertions(+), 33 deletions(-) create mode 100644 irc/utils/os.go diff --git a/irc/config.go b/irc/config.go index 3a1282ac..3d1b6504 100644 --- a/irc/config.go +++ b/irc/config.go @@ -230,7 +230,7 @@ type Config struct { Datastore struct { Path string - AutoUpgrade *bool + AutoUpgrade bool } Accounts AccountConfig diff --git a/irc/database.go b/irc/database.go index d92c2027..54f9f469 100644 --- a/irc/database.go +++ b/irc/database.go @@ -8,7 +8,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" "log" "os" "strings" @@ -16,6 +15,7 @@ import ( "github.com/oragono/oragono/irc/modes" "github.com/oragono/oragono/irc/passwd" + "github.com/oragono/oragono/irc/utils" "github.com/tidwall/buntdb" ) @@ -88,11 +88,7 @@ func InitDB(path string) { // OpenDatabase returns an existing database, performing a schema version check. func OpenDatabase(config *Config) (*buntdb.DB, error) { - allowAutoupgrade := true - if config.Datastore.AutoUpgrade != nil { - allowAutoupgrade = *config.Datastore.AutoUpgrade - } - return openDatabaseInternal(config, allowAutoupgrade) + return openDatabaseInternal(config, config.Datastore.AutoUpgrade) } // open the database, giving it at most one chance to auto-upgrade the schema @@ -140,36 +136,13 @@ func openDatabaseInternal(config *Config, allowAutoupgrade bool) (db *buntdb.DB, } } -// implementation of `cp` (go should really provide this...) -func cpFile(src string, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - closeError := out.Close() - if err == nil { - err = closeError - } - }() - if _, err = io.Copy(out, in); err != nil { - return - } - return -} - func performAutoUpgrade(currentVersion string, config *Config) (err error) { path := config.Datastore.Path log.Printf("attempting to auto-upgrade schema from version %s to %s\n", currentVersion, latestDbSchema) timestamp := time.Now().UTC().Format("2006-01-02-15:04:05.000Z") backupPath := fmt.Sprintf("%s.v%s.%s.bak", path, currentVersion, timestamp) log.Printf("making a backup of current database at %s\n", backupPath) - err = cpFile(path, backupPath) + err = utils.CopyFile(path, backupPath) if err != nil { return err } diff --git a/irc/utils/os.go b/irc/utils/os.go new file mode 100644 index 00000000..ef3eb5d1 --- /dev/null +++ b/irc/utils/os.go @@ -0,0 +1,31 @@ +// Copyright (c) 2018 Shivaram Lingamneni + +package utils + +import ( + "io" + "os" +) + +// implementation of `cp` (go should really provide this...) +func CopyFile(src string, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + closeError := out.Close() + if err == nil { + err = closeError + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + return +} diff --git a/oragono.yaml b/oragono.yaml index 7fca18a7..9bb17b45 100644 --- a/oragono.yaml +++ b/oragono.yaml @@ -341,8 +341,9 @@ debug: datastore: # path to the datastore path: ircd.db - # if the database schema requires an upgrade, `autoupgrade` (which defaults to true) - # will attempt to perform it automatically on startup. the database will be backed + + # if the database schema requires an upgrade, `autoupgrade` will attempt to + # perform it automatically on startup. the database will be backed # up, and if the upgrade fails, the original database will be restored. autoupgrade: true