Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-06 08:26:00 +01:00
Code Issues Projects Releases Wiki Activity

Fix incorrectly formatted timestamps (#2918)

Browse source 
* Update updated_at when adding file to object
* Use models.SQLTimestamp for timestamps
* Add data massage to fix incorrect timestamps
This commit is contained in:
WithoutPants 2022-09-19 14:53:46 +10:00 committed by GitHub
parent 2564351265
commit 1207629a76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 175 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
pkg/gallery/scan.go
View file

@ -16,6 +16,7 @@ import (
type FinderCreatorUpdater interface {
Finder
Create(ctx context.Context, newGallery *models.Gallery, fileIDs []file.ID) error
UpdatePartial(ctx context.Context, id int, updatedGallery models.GalleryPartial) (*models.Gallery, error)
AddFileID(ctx context.Context, id int, fileID file.ID) error
models.FileLoader
}
@ -100,6 +101,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.Base().ID); err != nil {
return fmt.Errorf("adding file to gallery: %w", err)
}
// update updated_at time
if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewGalleryPartial()); err != nil {
return fmt.Errorf("updating gallery: %w", err)
}
}
}

5
pkg/image/scan.go
View file

@ -22,6 +22,7 @@ type FinderCreatorUpdater interface {
FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error)
FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error)
Create(ctx context.Context, newImage *models.ImageCreateInput) error
UpdatePartial(ctx context.Context, id int, updatedImage models.ImagePartial) (*models.Image, error)
AddFileID(ctx context.Context, id int, fileID file.ID) error
models.GalleryIDLoader
models.ImageFileLoader
@ -169,6 +170,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.ID); err != nil {
return fmt.Errorf("adding file to image: %w", err)
}
// update updated_at time
if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewImagePartial()); err != nil {
return fmt.Errorf("updating image: %w", err)
}
}
}

5
pkg/scene/scan.go
View file

@ -133,6 +133,11 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
if err := h.CreatorUpdater.AddFileID(ctx, s.ID, f.ID); err != nil {
return fmt.Errorf("adding file to scene: %w", err)
}
// update updated_at time
if _, err := h.CreatorUpdater.UpdatePartial(ctx, s.ID, models.NewScenePartial()); err != nil {
return fmt.Errorf("updating scene: %w", err)
}
}
}

2
pkg/sqlite/database.go
View file

@ -21,7 +21,7 @@ import (
"github.com/stashapp/stash/pkg/logger"
)
var appSchemaVersion uint = 32
var appSchemaVersion uint = 33
//go:embed migrations/*.sql
var migrationsBox embed.FS

31
pkg/sqlite/gallery.go
View file

@ -7,7 +7,6 @@ import (
"fmt"
"path/filepath"
"regexp"
"time"
"github.com/doug-martin/goqu/v9"
"github.com/doug-martin/goqu/v9/exp"
@ -31,17 +30,17 @@ const (
)
type galleryRow struct {
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
URL zero.String `db:"url"`
Date models.SQLiteDate `db:"date"`
Details zero.String `db:"details"`
Rating null.Int `db:"rating"`
Organized bool `db:"organized"`
StudioID null.Int `db:"studio_id,omitempty"`
FolderID null.Int `db:"folder_id,omitempty"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
URL zero.String `db:"url"`
Date models.SQLiteDate `db:"date"`
Details zero.String `db:"details"`
Rating null.Int `db:"rating"`
Organized bool `db:"organized"`
StudioID null.Int `db:"studio_id,omitempty"`
FolderID null.Int `db:"folder_id,omitempty"`
CreatedAt models.SQLiteTimestamp `db:"created_at"`
UpdatedAt models.SQLiteTimestamp `db:"updated_at"`
}
func (r *galleryRow) fromGallery(o models.Gallery) {
@ -56,8 +55,8 @@ func (r *galleryRow) fromGallery(o models.Gallery) {
r.Organized = o.Organized
r.StudioID = intFromPtr(o.StudioID)
r.FolderID = nullIntFromFolderIDPtr(o.FolderID)
r.CreatedAt = o.CreatedAt
r.UpdatedAt = o.UpdatedAt
r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt}
r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt}
}
type galleryQueryRow struct {
@ -81,8 +80,8 @@ func (r *galleryQueryRow) resolve() *models.Gallery {
StudioID: nullIntPtr(r.StudioID),
FolderID: nullIntFolderIDPtr(r.FolderID),
PrimaryFileID: nullIntFileIDPtr(r.PrimaryFileID),
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
CreatedAt: r.CreatedAt.Timestamp,
UpdatedAt: r.UpdatedAt.Timestamp,
}
if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid {

25
pkg/sqlite/image.go
View file

@ -5,7 +5,6 @@ import (
"database/sql"
"fmt"
"path/filepath"
"time"
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/file"
@ -28,14 +27,14 @@ const (
)
type imageRow struct {
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
Rating null.Int `db:"rating"`
Organized bool `db:"organized"`
OCounter int `db:"o_counter"`
StudioID null.Int `db:"studio_id,omitempty"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
Rating null.Int `db:"rating"`
Organized bool `db:"organized"`
OCounter int `db:"o_counter"`
StudioID null.Int `db:"studio_id,omitempty"`
CreatedAt models.SQLiteTimestamp `db:"created_at"`
UpdatedAt models.SQLiteTimestamp `db:"updated_at"`
}
func (r *imageRow) fromImage(i models.Image) {
@ -45,8 +44,8 @@ func (r *imageRow) fromImage(i models.Image) {
r.Organized = i.Organized
r.OCounter = i.OCounter
r.StudioID = intFromPtr(i.StudioID)
r.CreatedAt = i.CreatedAt
r.UpdatedAt = i.UpdatedAt
r.CreatedAt = models.SQLiteTimestamp{Timestamp: i.CreatedAt}
r.UpdatedAt = models.SQLiteTimestamp{Timestamp: i.UpdatedAt}
}
type imageQueryRow struct {
@ -69,8 +68,8 @@ func (r *imageQueryRow) resolve() *models.Image {
PrimaryFileID: nullIntFileIDPtr(r.PrimaryFileID),
Checksum: r.PrimaryFileChecksum.String,
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
CreatedAt: r.CreatedAt.Timestamp,
UpdatedAt: r.UpdatedAt.Timestamp,
}
if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid {

116
pkg/sqlite/migrations/33_postmigrate.go Normal file
View file

@ -0,0 +1,116 @@
package migrations
import (
"context"
"fmt"
"time"
"github.com/jmoiron/sqlx"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/sqlite"
)
type schema33Migrator struct {
migrator
}
func post33(ctx context.Context, db *sqlx.DB) error {
logger.Info("Running post-migration for schema version 33")
m := schema33Migrator{
migrator: migrator{
db: db,
},
}
if err := m.migrateObjects(ctx, "scenes"); err != nil {
return fmt.Errorf("migrating scenes: %w", err)
}
if err := m.migrateObjects(ctx, "images"); err != nil {
return fmt.Errorf("migrating images: %w", err)
}
if err := m.migrateObjects(ctx, "galleries"); err != nil {
return fmt.Errorf("migrating galleries: %w", err)
}
return nil
}
func (m *schema33Migrator) migrateObjects(ctx context.Context, table string) error {
logger.Infof("Migrating %s table", table)
const (
limit = 1000
logEvery = 10000
)
lastID := 0
count := 0
for {
gotSome := false
if err := m.withTxn(ctx, func(tx *sqlx.Tx) error {
query := fmt.Sprintf("SELECT `id`, `created_at`, `updated_at` FROM `%s` WHERE `created_at` like '%% %%' OR `updated_at` like '%% %%'", table)
if lastID != 0 {
query += fmt.Sprintf("AND `id` > %d ", lastID)
}
query += fmt.Sprintf("ORDER BY `id` LIMIT %d", limit)
rows, err := m.db.Query(query)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var (
id int
createdAt time.Time
updatedAt time.Time
)
err := rows.Scan(&id, &createdAt, &updatedAt)
if err != nil {
return err
}
lastID = id
gotSome = true
count++
// convert incorrect timestamp string to correct one
// based on models.SQLTimestamp
fixedCreated := createdAt.Format(time.RFC3339)
fixedUpdated := updatedAt.Format(time.RFC3339)
updateSQL := fmt.Sprintf("UPDATE `%s` SET `created_at` = ?, `updated_at` = ? WHERE `id` = ?", table)
_, err = m.db.Exec(updateSQL, fixedCreated, fixedUpdated, id)
if err != nil {
return err
}
}
return rows.Err()
}); err != nil {
return err
}
if !gotSome {
break
}
if count%logEvery == 0 {
logger.Infof("Migrated %d rows", count, table)
}
}
return nil
}
func init() {
sqlite.RegisterPostMigration(33, post33)
}

1
pkg/sqlite/migrations/33_time_fix.up.sql Normal file
View file

@ -0,0 +1 @@
-- no schema changes

31
pkg/sqlite/scene.go
View file

@ -8,7 +8,6 @@ import (
"path/filepath"
"strconv"
"strings"
"time"
"github.com/doug-martin/goqu/v9"
"github.com/doug-martin/goqu/v9/exp"
@ -53,17 +52,17 @@ ORDER BY files.size DESC
`
type sceneRow struct {
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
Details zero.String `db:"details"`
URL zero.String `db:"url"`
Date models.SQLiteDate `db:"date"`
Rating null.Int `db:"rating"`
Organized bool `db:"organized"`
OCounter int `db:"o_counter"`
StudioID null.Int `db:"studio_id,omitempty"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
ID int `db:"id" goqu:"skipinsert"`
Title zero.String `db:"title"`
Details zero.String `db:"details"`
URL zero.String `db:"url"`
Date models.SQLiteDate `db:"date"`
Rating null.Int `db:"rating"`
Organized bool `db:"organized"`
OCounter int `db:"o_counter"`
StudioID null.Int `db:"studio_id,omitempty"`
CreatedAt models.SQLiteTimestamp `db:"created_at"`
UpdatedAt models.SQLiteTimestamp `db:"updated_at"`
}
func (r *sceneRow) fromScene(o models.Scene) {
@ -78,8 +77,8 @@ func (r *sceneRow) fromScene(o models.Scene) {
r.Organized = o.Organized
r.OCounter = o.OCounter
r.StudioID = intFromPtr(o.StudioID)
r.CreatedAt = o.CreatedAt
r.UpdatedAt = o.UpdatedAt
r.CreatedAt = models.SQLiteTimestamp{Timestamp: o.CreatedAt}
r.UpdatedAt = models.SQLiteTimestamp{Timestamp: o.UpdatedAt}
}
type sceneQueryRow struct {
@ -107,8 +106,8 @@ func (r *sceneQueryRow) resolve() *models.Scene {
OSHash: r.PrimaryFileOshash.String,
Checksum: r.PrimaryFileChecksum.String,
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
CreatedAt: r.CreatedAt.Timestamp,
UpdatedAt: r.UpdatedAt.Timestamp,
}
if r.PrimaryFileFolderPath.Valid && r.PrimaryFileBasename.Valid {