Merge pull request #246 from slingamn/db_autoupgrade.1

implement database auto-upgrades (#243)

irc/config.go

@@ -230,6 +230,7 @@ type Config struct {
 
 	Datastore struct {
 		Path        string
+		AutoUpgrade bool
 	}
 
 	Accounts AccountConfig

irc/database.go

@@ -11,9 +11,11 @@ import (
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/oragono/oragono/irc/modes"
 	"github.com/oragono/oragono/irc/passwd"
+	"github.com/oragono/oragono/irc/utils"
 
 	"github.com/tidwall/buntdb"
 )
@@ -38,6 +40,22 @@ type SchemaChange struct {
 // maps an initial version to a schema change capable of upgrading it
 var schemaChanges map[string]SchemaChange
 
+type incompatibleSchemaError struct {
+	currentVersion  string
+	requiredVersion string
+}
+
+func IncompatibleSchemaError(currentVersion string) (result *incompatibleSchemaError) {
+	return &incompatibleSchemaError{
+		currentVersion:  currentVersion,
+		requiredVersion: latestDbSchema,
+	}
+}
+
+func (err *incompatibleSchemaError) Error() string {
+	return fmt.Sprintf("Database requires update. Expected schema v%s, got v%s", err.requiredVersion, err.currentVersion)
+}
+
 // InitDB creates the database.
 func InitDB(path string) {
 	// prepare kvstore db
@@ -69,36 +87,80 @@ func InitDB(path string) {
 }
 
 // OpenDatabase returns an existing database, performing a schema version check.
-func OpenDatabase(path string) (*buntdb.DB, error) {
+func OpenDatabase(config *Config) (*buntdb.DB, error) {
-	// open data store
+	return openDatabaseInternal(config, config.Datastore.AutoUpgrade)
-	db, err := buntdb.Open(path)
+}
+
+// open the database, giving it at most one chance to auto-upgrade the schema
+func openDatabaseInternal(config *Config, allowAutoupgrade bool) (db *buntdb.DB, err error) {
+	db, err = buntdb.Open(config.Datastore.Path)
 	if err != nil {
-		return nil, err
+		return
 	}
 
-	// check db version
+	defer func() {
-	err = db.View(func(tx *buntdb.Tx) error {
+		if err != nil && db != nil {
-		version, _ := tx.Get(keySchemaVersion)
-		if version != latestDbSchema {
-			return fmt.Errorf("Database must be updated. Expected schema v%s, got v%s", latestDbSchema, version)
-		}
-		return nil
-	})
-
-	if err != nil {
-		// close the db
 			db.Close()
-		return nil, err
+			db = nil
+		}
+	}()
+
+	// read the current version string
+	var version string
+	err = db.View(func(tx *buntdb.Tx) error {
+		version, err = tx.Get(keySchemaVersion)
+		return err
+	})
+	if err != nil {
+		return
 	}
 
-	return db, nil
+	if version == latestDbSchema {
+		// success
+		return
+	}
+
+	// XXX quiesce the DB so we can be sure it's safe to make a backup copy
+	db.Close()
+	db = nil
+	if allowAutoupgrade {
+		err = performAutoUpgrade(version, config)
+		if err != nil {
+			return
+		}
+		// successful autoupgrade, let's try this again:
+		return openDatabaseInternal(config, false)
+	} else {
+		err = IncompatibleSchemaError(version)
+		return
+	}
+}
+
+func performAutoUpgrade(currentVersion string, config *Config) (err error) {
+	path := config.Datastore.Path
+	log.Printf("attempting to auto-upgrade schema from version %s to %s\n", currentVersion, latestDbSchema)
+	timestamp := time.Now().UTC().Format("2006-01-02-15:04:05.000Z")
+	backupPath := fmt.Sprintf("%s.v%s.%s.bak", path, currentVersion, timestamp)
+	log.Printf("making a backup of current database at %s\n", backupPath)
+	err = utils.CopyFile(path, backupPath)
+	if err != nil {
+		return err
+	}
+
+	err = UpgradeDB(config)
+	if err != nil {
+		// database upgrade is a single transaction, so we don't need to restore the backup;
+		// we can just delete it
+		os.Remove(backupPath)
+	}
+	return err
 }
 
 // UpgradeDB upgrades the datastore to the latest schema.
-func UpgradeDB(config *Config) {
+func UpgradeDB(config *Config) (err error) {
 	store, err := buntdb.Open(config.Datastore.Path)
 	if err != nil {
-		log.Fatal(fmt.Sprintf("Failed to open datastore: %s", err.Error()))
+		return err
 	}
 	defer store.Close()
 
@@ -108,9 +170,14 @@ func UpgradeDB(config *Config) {
 			version, _ = tx.Get(keySchemaVersion)
 			change, schemaNeedsChange := schemaChanges[version]
 			if !schemaNeedsChange {
+				if version == latestDbSchema {
+					// success!
 					break
 				}
-			log.Println("attempting to update store from version " + version)
+				// unable to upgrade to the desired version, roll back
+				return IncompatibleSchemaError(version)
+			}
+			log.Println("attempting to update schema from version " + version)
 			err := change.Changer(config, tx)
 			if err != nil {
 				return err
@@ -119,16 +186,15 @@ func UpgradeDB(config *Config) {
 			if err != nil {
 				return err
 			}
-			log.Println("successfully updated store to version " + change.TargetVersion)
+			log.Println("successfully updated schema to version " + change.TargetVersion)
 		}
 		return nil
 	})
 
 	if err != nil {
-		log.Fatal("Could not update datastore:", err.Error())
+		log.Println("database upgrade failed and was rolled back")
 	}
-
+	return err
-	return
 }
 
 func schemaChangeV1toV2(config *Config, tx *buntdb.Tx) error {

irc/server.go

@@ -127,7 +127,6 @@ type Server struct {
 	signals                    chan os.Signal
 	snomasks                   *SnoManager
 	store                      *buntdb.DB
-	storeFilename              string
 	stsEnabled                 bool
 	webirc                     []webircConfig
 	whoWas                     *WhoWasList
@@ -753,7 +752,7 @@ func (server *Server) applyConfig(config *Config, initial bool) error {
 			return fmt.Errorf("Maximum line length (linelen) cannot be changed after launching the server, rehash aborted")
 		} else if server.name != config.Server.Name {
 			return fmt.Errorf("Server name cannot be changed after launching the server, rehash aborted")
-		} else if server.storeFilename != config.Datastore.Path {
+		} else if server.config.Datastore.Path != config.Datastore.Path {
 			return fmt.Errorf("Datastore path cannot be changed after launching the server, rehash aborted")
 		}
 	}
@@ -973,10 +972,9 @@ func (server *Server) applyConfig(config *Config, initial bool) error {
 	server.config = config
 	server.configurableStateMutex.Unlock()
 
-	server.storeFilename = config.Datastore.Path
+	server.logger.Info("rehash", "Using datastore", config.Datastore.Path)
-	server.logger.Info("rehash", "Using datastore", server.storeFilename)
 	if initial {
-		if err := server.loadDatastore(server.storeFilename); err != nil {
+		if err := server.loadDatastore(config); err != nil {
 			return err
 		}
 	}
@@ -1073,11 +1071,11 @@ func (server *Server) loadMOTD(motdPath string, useFormatting bool) error {
 	return nil
 }
 
-func (server *Server) loadDatastore(datastorePath string) error {
+func (server *Server) loadDatastore(config *Config) error {
 	// open the datastore and load server state for which it (rather than config)
 	// is the source of truth
 
-	db, err := OpenDatabase(datastorePath)
+	db, err := OpenDatabase(config)
 	if err == nil {
 		server.store = db
 	} else {

irc/utils/os.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2018 Shivaram Lingamneni
+
+package utils
+
+import (
+	"io"
+	"os"
+)
+
+// implementation of `cp` (go should really provide this...)
+func CopyFile(src string, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+	defer func() {
+		closeError := out.Close()
+		if err == nil {
+			err = closeError
+		}
+	}()
+	if _, err = io.Copy(out, in); err != nil {
+		return
+	}
+	return
+}

oragono.go

@@ -84,7 +84,10 @@ Options:
 			log.Println("database initialized: ", config.Datastore.Path)
 		}
 	} else if arguments["upgradedb"].(bool) {
-		irc.UpgradeDB(config)
+		err = irc.UpgradeDB(config)
+		if err != nil {
+			log.Fatal("Error while upgrading db:", err.Error())
+		}
 		if !arguments["--quiet"].(bool) {
 			log.Println("database upgraded: ", config.Datastore.Path)
 		}

oragono.yaml

@@ -342,6 +342,11 @@ datastore:
     # path to the datastore
     path: ircd.db
 
+    # if the database schema requires an upgrade, `autoupgrade` will attempt to
+    # perform it automatically on startup. the database will be backed
+    # up, and if the upgrade fails, the original database will be restored.
+    autoupgrade: true
+
 # languages config
 languages:
     # whether to load languages