Fix various migration issues (#5723)

* Indicate while backing up database
* Close migrate connection to db before optimising
* Don't vacuum post-migration

In most cases is probably not needed and can be an optonal user-initiated step

* Ensure connection close on NewMigrator error
* Perform post-migration using migrator connection

Flush WAL file at end of migration

internal/manager/task/migrate.go

@@ -42,8 +42,8 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
 
 	logger.Infof("Migrating database from %d to %d", schemaInfo.CurrentSchemaVersion, schemaInfo.RequiredSchemaVersion)
 
-	// set the number of tasks = required steps + optimise
+	// set the number of tasks = backup + required steps + optimise
-	progress.SetTotal(int(schemaInfo.StepsRequired + 1))
+	progress.SetTotal(int(schemaInfo.StepsRequired + 2))
 
 	database := s.Database
 
@@ -61,12 +61,20 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
 		}
 	}
 
+	progress.ExecuteTask("Backing up database", func() {
+		defer progress.Increment()
+
 		// perform database backup
-	if err := database.Backup(backupPath); err != nil {
+		err = database.Backup(backupPath)
+	})
+
+	if err != nil {
 		return fmt.Errorf("error backing up database: %s", err)
 	}
 
-	if err := s.runMigrations(ctx, progress); err != nil {
+	err = s.runMigrations(ctx, progress)
+
+	if err != nil {
 		errStr := fmt.Sprintf("error performing migration: %s", err)
 
 		// roll back to the backed up version
@@ -87,6 +95,11 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error
 		}
 	}
 
+	// reinitialise the database
+	if err := database.ReInitialise(); err != nil {
+		return fmt.Errorf("error reinitialising database: %s", err)
+	}
+
 	logger.Infof("Database migration complete")
 
 	return nil
@@ -124,6 +137,8 @@ func (s *MigrateJob) runMigrations(ctx context.Context, progress *job.Progress)
 
 	defer m.Close()
 
+	logger.Info("Running migrations")
+
 	for {
 		currentSchemaVersion := m.CurrentSchemaVersion()
 		targetSchemaVersion := m.RequiredSchemaVersion()
@@ -144,21 +159,15 @@ func (s *MigrateJob) runMigrations(ctx context.Context, progress *job.Progress)
 		progress.Increment()
 	}
 
-	// reinitialise the database
+	// perform post-migrate analyze using the migrator connection
-	if err := database.ReInitialise(); err != nil {
-		return fmt.Errorf("error reinitialising database: %s", err)
-	}
-
-	// optimise the database
 	progress.ExecuteTask("Optimising database", func() {
-		err = database.Optimise(ctx)
+		err = m.PostMigrate(ctx)
+		progress.Increment()
 	})
 
 	if err != nil {
 		return fmt.Errorf("error optimising database: %s", err)
 	}
 
-	progress.Increment()
-
 	return nil
 }

pkg/sqlite/database.go

@@ -430,7 +430,19 @@ func (db *Database) Vacuum(ctx context.Context) error {
 
 // Analyze runs an ANALYZE on the database to improve query performance.
 func (db *Database) Analyze(ctx context.Context) error {
-	_, err := db.writeDB.ExecContext(ctx, "ANALYZE")
+	return analyze(ctx, db.writeDB)
+}
+
+// analyze runs an ANALYZE on the database to improve query performance.
+func analyze(ctx context.Context, db *sqlx.DB) error {
+	_, err := db.ExecContext(ctx, "ANALYZE")
+	return err
+}
+
+// flushWAL flushes the Write-Ahead Log (WAL) to the main database file.
+// It also truncates the WAL file to 0 bytes.
+func flushWAL(ctx context.Context, db *sqlx.DB) error {
+	_, err := db.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
 	return err
 }
 

pkg/sqlite/migrate.go

@@ -39,6 +39,12 @@ func NewMigrator(db *Database) (*Migrator, error) {
 	m.conn.SetConnMaxIdleTime(dbConnTimeout)
 
 	m.m, err = m.getMigrate()
+
+	// if error encountered, close the connection
+	if err != nil {
+		m.Close()
+	}
+
 	return m, err
 }
 
@@ -124,6 +130,27 @@ func (m *Migrator) runCustomMigration(ctx context.Context, fn customMigrationFun
 	return nil
 }
 
+func (m *Migrator) PostMigrate(ctx context.Context) error {
+	// optimise the database
+	var err error
+	logger.Info("Running database analyze")
+
+	// don't use Optimize/vacuum as this adds a significant amount of time
+	// to the migration
+	err = analyze(ctx, m.conn)
+
+	if err == nil {
+		logger.Debug("Flushing WAL")
+		err = flushWAL(ctx, m.conn)
+	}
+
+	if err != nil {
+		return fmt.Errorf("error optimising database: %s", err)
+	}
+
+	return nil
+}
+
 func (db *Database) getDatabaseSchemaVersion() (uint, error) {
 	m, err := NewMigrator(db)
 	if err != nil {