Fix incorrectly formatted timestamps (#2918)

* Update updated_at when adding file to object
* Use models.SQLTimestamp for timestamps
* Add data massage to fix incorrect timestamps

pkg/gallery/scan.go

@@ -16,6 +16,7 @@ import (
 type FinderCreatorUpdater interface {
 	Finder
 	Create(ctx context.Context, newGallery *models.Gallery, fileIDs []file.ID) error
+	UpdatePartial(ctx context.Context, id int, updatedGallery models.GalleryPartial) (*models.Gallery, error)
 	AddFileID(ctx context.Context, id int, fileID file.ID) error
 	models.FileLoader
 }
@@ -100,6 +101,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
 			if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.Base().ID); err != nil {
 				return fmt.Errorf("adding file to gallery: %w", err)
 			}
+			// update updated_at time
+			if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewGalleryPartial()); err != nil {
+				return fmt.Errorf("updating gallery: %w", err)
+			}
 		}
 
 	}

pkg/image/scan.go

@@ -22,6 +22,7 @@ type FinderCreatorUpdater interface {
 	FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error)
 	FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error)
 	Create(ctx context.Context, newImage *models.ImageCreateInput) error
+	UpdatePartial(ctx context.Context, id int, updatedImage models.ImagePartial) (*models.Image, error)
 	AddFileID(ctx context.Context, id int, fileID file.ID) error
 	models.GalleryIDLoader
 	models.ImageFileLoader
@@ -169,6 +170,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
 			if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.ID); err != nil {
 				return fmt.Errorf("adding file to image: %w", err)
 			}
+			// update updated_at time
+			if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewImagePartial()); err != nil {
+				return fmt.Errorf("updating image: %w", err)
+			}
 		}
 	}
 

pkg/scene/scan.go

@@ -133,6 +133,11 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
 			if err := h.CreatorUpdater.AddFileID(ctx, s.ID, f.ID); err != nil {
 				return fmt.Errorf("adding file to scene: %w", err)
 			}
+
+			// update updated_at time
+			if _, err := h.CreatorUpdater.UpdatePartial(ctx, s.ID, models.NewScenePartial()); err != nil {
+				return fmt.Errorf("updating scene: %w", err)
+			}
 		}
 	}
 

pkg/sqlite/database.go

@@ -21,7 +21,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 )
 
-var appSchemaVersion uint = 32
+var appSchemaVersion uint = 33
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS

pkg/sqlite/gallery.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"path/filepath"
 	"regexp"
-	"time"
 
 	"github.com/doug-martin/goqu/v9"
 	"github.com/doug-martin/goqu/v9/exp"
@@ -40,8 +39,8 @@ type galleryRow struct {
 	Organized bool                   `db:"organized"`
 	StudioID  null.Int               `db:"studio_id,omitempty"`
 	FolderID  null.Int               `db:"folder_id,omitempty"`
-	CreatedAt time.Time         `db:"created_at"`
+	CreatedAt models.SQLiteTimestamp `db:"created_at"`
-	UpdatedAt time.Time         `db:"updated_at"`
+	UpdatedAt models.SQLiteTimestamp `db:"updated_at"`
 }
 
 func (r *galleryRow) fromGallery(o models.Gallery) {
@@ -56,8 +55,8 @@ func (r *galleryRow) fromGallery(o models.Gallery) {
 	r.Organized = o.Organized
 	r.StudioID = intFromPtr(o.StudioID)
 	r.FolderID = nullIntFromFolderIDPtr(o.FolderID)
-	r.CreatedAt = o.CreatedAt
+	r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt}
-	r.UpdatedAt = o.UpdatedAt
+	r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt}
 }
 
 type galleryQueryRow struct {
@@ -81,8 +80,8 @@ func (r *galleryQueryRow) resolve() *models.Gallery {
 		StudioID:      nullIntPtr(r.StudioID),
 		FolderID:      nullIntFolderIDPtr(r.FolderID),
 		PrimaryFileID: nullIntFileIDPtr(r.PrimaryFileID),
-		CreatedAt:     r.CreatedAt,
+		CreatedAt:     r.CreatedAt.Timestamp,
-		UpdatedAt:     r.UpdatedAt,
+		UpdatedAt:     r.UpdatedAt.Timestamp,
 	}
 
 	if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid {

pkg/sqlite/image.go

@@ -5,7 +5,6 @@ import (
 	"database/sql"
 	"fmt"
 	"path/filepath"
-	"time"
 
 	"github.com/jmoiron/sqlx"
 	"github.com/stashapp/stash/pkg/file"
@@ -34,8 +33,8 @@ type imageRow struct {
 	Organized bool                   `db:"organized"`
 	OCounter  int                    `db:"o_counter"`
 	StudioID  null.Int               `db:"studio_id,omitempty"`
-	CreatedAt time.Time   `db:"created_at"`
+	CreatedAt models.SQLiteTimestamp `db:"created_at"`
-	UpdatedAt time.Time   `db:"updated_at"`
+	UpdatedAt models.SQLiteTimestamp `db:"updated_at"`
 }
 
 func (r *imageRow) fromImage(i models.Image) {
@@ -45,8 +44,8 @@ func (r *imageRow) fromImage(i models.Image) {
 	r.Organized = i.Organized
 	r.OCounter = i.OCounter
 	r.StudioID = intFromPtr(i.StudioID)
-	r.CreatedAt = i.CreatedAt
+	r.CreatedAt = models.SQLiteTimestamp{Timestamp: i.CreatedAt}
-	r.UpdatedAt = i.UpdatedAt
+	r.UpdatedAt = models.SQLiteTimestamp{Timestamp: i.UpdatedAt}
 }
 
 type imageQueryRow struct {
@@ -69,8 +68,8 @@ func (r *imageQueryRow) resolve() *models.Image {
 		PrimaryFileID: nullIntFileIDPtr(r.PrimaryFileID),
 		Checksum:      r.PrimaryFileChecksum.String,
 
-		CreatedAt: r.CreatedAt,
+		CreatedAt: r.CreatedAt.Timestamp,
-		UpdatedAt: r.UpdatedAt,
+		UpdatedAt: r.UpdatedAt.Timestamp,
 	}
 
 	if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid {

pkg/sqlite/migrations/33_postmigrate.go

@@ -0,0 +1,116 @@
+package migrations
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/sqlite"
+)
+
+type schema33Migrator struct {
+	migrator
+}
+
+func post33(ctx context.Context, db *sqlx.DB) error {
+	logger.Info("Running post-migration for schema version 33")
+
+	m := schema33Migrator{
+		migrator: migrator{
+			db: db,
+		},
+	}
+
+	if err := m.migrateObjects(ctx, "scenes"); err != nil {
+		return fmt.Errorf("migrating scenes: %w", err)
+	}
+	if err := m.migrateObjects(ctx, "images"); err != nil {
+		return fmt.Errorf("migrating images: %w", err)
+	}
+	if err := m.migrateObjects(ctx, "galleries"); err != nil {
+		return fmt.Errorf("migrating galleries: %w", err)
+	}
+
+	return nil
+}
+
+func (m *schema33Migrator) migrateObjects(ctx context.Context, table string) error {
+	logger.Infof("Migrating %s table", table)
+
+	const (
+		limit    = 1000
+		logEvery = 10000
+	)
+
+	lastID := 0
+	count := 0
+
+	for {
+		gotSome := false
+
+		if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
+			query := fmt.Sprintf("SELECT `id`, `created_at`, `updated_at` FROM `%s` WHERE `created_at` like '%% %%' OR `updated_at` like '%% %%'", table)
+
+			if lastID != 0 {
+				query += fmt.Sprintf("AND `id` > %d ", lastID)
+			}
+
+			query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit)
+
+			rows, err := m.db.Query(query)
+			if err != nil {
+				return err
+			}
+			defer rows.Close()
+
+			for rows.Next() {
+				var (
+					id        int
+					createdAt time.Time
+					updatedAt time.Time
+				)
+
+				err := rows.Scan(&id, &createdAt, &updatedAt)
+				if err != nil {
+					return err
+				}
+
+				lastID = id
+				gotSome = true
+				count++
+
+				// convert incorrect timestamp string to correct one
+				// based on models.SQLTimestamp
+				fixedCreated := createdAt.Format(time.RFC3339)
+				fixedUpdated := updatedAt.Format(time.RFC3339)
+
+				updateSQL := fmt.Sprintf("UPDATE `%s` SET `created_at` = ?, `updated_at` = ? WHERE `id` = ?", table)
+
+				_, err = m.db.Exec(updateSQL, fixedCreated, fixedUpdated, id)
+				if err != nil {
+					return err
+				}
+			}
+
+			return rows.Err()
+		}); err != nil {
+			return err
+		}
+
+		if !gotSome {
+			break
+		}
+
+		if count%logEvery == 0 {
+			logger.Infof("Migrated %d rows", count, table)
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	sqlite.RegisterPostMigration(33, post33)
+}

pkg/sqlite/migrations/33_time_fix.up.sql

@@ -0,0 +1 @@
+-- no schema changes

pkg/sqlite/scene.go

@@ -8,7 +8,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/doug-martin/goqu/v9"
 	"github.com/doug-martin/goqu/v9/exp"
@@ -62,8 +61,8 @@ type sceneRow struct {
 	Organized bool                   `db:"organized"`
 	OCounter  int                    `db:"o_counter"`
 	StudioID  null.Int               `db:"studio_id,omitempty"`
-	CreatedAt time.Time         `db:"created_at"`
+	CreatedAt models.SQLiteTimestamp `db:"created_at"`
-	UpdatedAt time.Time         `db:"updated_at"`
+	UpdatedAt models.SQLiteTimestamp `db:"updated_at"`
 }
 
 func (r *sceneRow) fromScene(o models.Scene) {
@@ -78,8 +77,8 @@ func (r *sceneRow) fromScene(o models.Scene) {
 	r.Organized = o.Organized
 	r.OCounter = o.OCounter
 	r.StudioID = intFromPtr(o.StudioID)
-	r.CreatedAt = o.CreatedAt
+	r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt}
-	r.UpdatedAt = o.UpdatedAt
+	r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt}
 }
 
 type sceneQueryRow struct {
@@ -107,8 +106,8 @@ func (r *sceneQueryRow) resolve() *models.Scene {
 		OSHash:        r.PrimaryFileOshash.String,
 		Checksum:      r.PrimaryFileChecksum.String,
 
-		CreatedAt: r.CreatedAt,
+		CreatedAt: r.CreatedAt.Timestamp,
-		UpdatedAt: r.UpdatedAt,
+		UpdatedAt: r.UpdatedAt.Timestamp,
 	}
 
 	if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid {