Handle file rescan (#2951)

* Fire handlers when file updated or moved
* Create galleries as needed
* Clean empty galleries
* Handle cleaning zip folders when path changed
* Fix gallery association on duplicate images
* Re-create missing folder-based galleries

internal/manager/task_clean.go

@@ -16,6 +16,7 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/plugin"
 	"github.com/stashapp/stash/pkg/scene"
+	"github.com/stashapp/stash/pkg/txn"
 )
 
 type cleaner interface {
@@ -49,11 +50,95 @@ func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) {
 		return
 	}
 
+	j.cleanEmptyGalleries(ctx)
+
 	j.scanSubs.notify()
 	elapsed := time.Since(start)
 	logger.Info(fmt.Sprintf("Finished Cleaning (%s)", elapsed))
 }
 
+func (j *cleanJob) cleanEmptyGalleries(ctx context.Context) {
+	const batchSize = 1000
+	var toClean []int
+	findFilter := models.BatchFindFilter(batchSize)
+	if err := txn.WithTxn(ctx, j.txnManager, func(ctx context.Context) error {
+		found := true
+		for found {
+			emptyGalleries, _, err := j.txnManager.Gallery.Query(ctx, &models.GalleryFilterType{
+				ImageCount: &models.IntCriterionInput{
+					Value:    0,
+					Modifier: models.CriterionModifierEquals,
+				},
+			}, findFilter)
+
+			if err != nil {
+				return err
+			}
+
+			found = len(emptyGalleries) > 0
+
+			for _, g := range emptyGalleries {
+				if g.Path == "" {
+					continue
+				}
+
+				if len(j.input.Paths) > 0 && !fsutil.IsPathInDirs(j.input.Paths, g.Path) {
+					continue
+				}
+
+				logger.Infof("Gallery has 0 images. Marking to clean: %s", g.DisplayName())
+				toClean = append(toClean, g.ID)
+			}
+
+			*findFilter.Page++
+		}
+
+		return nil
+	}); err != nil {
+		logger.Errorf("Error finding empty galleries: %v", err)
+		return
+	}
+
+	if !j.input.DryRun {
+		for _, id := range toClean {
+			j.deleteGallery(ctx, id)
+		}
+	}
+}
+
+func (j *cleanJob) deleteGallery(ctx context.Context, id int) {
+	pluginCache := GetInstance().PluginCache
+	qb := j.txnManager.Gallery
+
+	if err := txn.WithTxn(ctx, j.txnManager, func(ctx context.Context) error {
+		g, err := qb.Find(ctx, id)
+		if err != nil {
+			return err
+		}
+
+		if g == nil {
+			return fmt.Errorf("gallery not found: %d", id)
+		}
+
+		if err := g.LoadPrimaryFile(ctx, j.txnManager.File); err != nil {
+			return err
+		}
+
+		if err := qb.Destroy(ctx, id); err != nil {
+			return err
+		}
+
+		pluginCache.RegisterPostHooks(ctx, id, plugin.GalleryDestroyPost, plugin.GalleryDestroyInput{
+			Checksum: g.PrimaryChecksum(),
+			Path:     g.Path,
+		}, nil)
+
+		return nil
+	}); err != nil {
+		logger.Errorf("Error deleting gallery from database: %s", err.Error())
+	}
+}
+
 type cleanFilter struct {
 	scanFilter
 }

internal/manager/task_scan.go

@@ -9,6 +9,7 @@ import (
 	"regexp"
 	"time"
 
+	"github.com/99designs/gqlgen/graphql/handler/lru"
 	"github.com/stashapp/stash/internal/manager/config"
 	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/file/video"
@@ -61,7 +62,9 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) {
 		ScanFilters:       []file.PathFilter{newScanFilter(instance.Config, minModTime)},
 		ZipFileExtensions: instance.Config.GetGalleryExtensions(),
 		ParallelTasks:     instance.Config.GetParallelTasksWithAutoDetection(),
-		HandlerRequiredFilters: []file.Filter{newHandlerRequiredFilter(instance.Config)},
+		HandlerRequiredFilters: []file.Filter{
+			newHandlerRequiredFilter(instance.Config),
+		},
 	}, progress)
 
 	taskQueue.Close()
@@ -95,22 +98,31 @@ type fileCounter interface {
 	CountByFileID(ctx context.Context, fileID file.ID) (int, error)
 }
 
+type galleryFinder interface {
+	fileCounter
+	FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Gallery, error)
+}
+
 // handlerRequiredFilter returns true if a File's handler needs to be executed despite the file not being updated.
 type handlerRequiredFilter struct {
 	extensionConfig
 	SceneFinder   fileCounter
 	ImageFinder   fileCounter
-	GalleryFinder fileCounter
+	GalleryFinder galleryFinder
+
+	FolderCache *lru.LRU
 }
 
 func newHandlerRequiredFilter(c *config.Instance) *handlerRequiredFilter {
 	db := instance.Database
+	processes := c.GetParallelTasksWithAutoDetection()
 
 	return &handlerRequiredFilter{
 		extensionConfig: newExtensionConfig(c),
 		SceneFinder:     db.Scene,
 		ImageFinder:     db.Image,
 		GalleryFinder:   db.Gallery,
+		FolderCache:     lru.New(processes * 2),
 	}
 }
 
@@ -143,7 +155,32 @@ func (f *handlerRequiredFilter) Accept(ctx context.Context, ff file.File) bool {
 	}
 
 	// execute handler if there are no related objects
-	return n == 0
+	if n == 0 {
+		return true
+	}
+
+	// if create galleries from folder is enabled and the file is not in a zip
+	// file, then check if there is a folder-based gallery for the file's
+	// directory
+	if isImageFile && instance.Config.GetCreateGalleriesFromFolders() && ff.Base().ZipFileID == nil {
+		// only do this for the first time it encounters the folder
+		// the first instance should create the gallery
+		_, found := f.FolderCache.Get(ctx, ff.Base().ParentFolderID.String())
+		if found {
+			// should already be handled
+			return false
+		}
+
+		g, _ := f.GalleryFinder.FindByFolderID(ctx, ff.Base().ParentFolderID)
+		f.FolderCache.Add(ctx, ff.Base().ParentFolderID.String(), true)
+
+		if len(g) == 0 {
+			// no folder gallery. Return true so that it creates one.
+			return true
+		}
+	}
+
+	return false
 }
 
 type scanFilter struct {
@@ -248,6 +285,7 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre
 					isGenerateThumbnails: options.ScanGenerateThumbnails,
 				},
 				PluginCache: pluginCache,
+				Paths:       instance.Paths,
 			},
 		},
 		&file.FilteredHandler{
@@ -255,6 +293,7 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre
 			Handler: &gallery.ScanHandler{
 				CreatorUpdater:     db.Gallery,
 				SceneFinderUpdater: db.Scene,
+				ImageFinderUpdater: db.Image,
 				PluginCache:        pluginCache,
 			},
 		},
@@ -269,6 +308,8 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre
 					taskQueue: taskQueue,
 					progress:  progress,
 				},
+				FileNamingAlgorithm: instance.Config.GetVideoFileNamingAlgorithm(),
+				Paths:               instance.Paths,
 			},
 		},
 	}

pkg/file/clean.go

@@ -338,7 +338,9 @@ func (j *cleanJob) shouldCleanFolder(ctx context.Context, f *Folder) bool {
 	path := f.Path
 
 	info, err := f.Info(j.FS)
-	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+	// ErrInvalid can occur in zip files where the zip file path changed
+	// and the underlying folder did not
+	if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, fs.ErrInvalid) {
 		logger.Errorf("error getting folder info for %q, not cleaning: %v", path, err)
 		return false
 	}

pkg/file/handler.go

@@ -29,7 +29,7 @@ func (ff FilterFunc) Accept(ctx context.Context, f File) bool {
 
 // Handler provides a handler for Files.
 type Handler interface {
-	Handle(ctx context.Context, f File) error
+	Handle(ctx context.Context, f File, oldFile File) error
 }
 
 // FilteredHandler is a Handler runs only if the filter accepts the file.
@@ -39,9 +39,9 @@ type FilteredHandler struct {
 }
 
 // Handle runs the handler if the filter accepts the file.
-func (h *FilteredHandler) Handle(ctx context.Context, f File) error {
+func (h *FilteredHandler) Handle(ctx context.Context, f File, oldFile File) error {
 	if h.Accept(ctx, f) {
-		return h.Handler.Handle(ctx, f)
+		return h.Handler.Handle(ctx, f, oldFile)
 	}
 	return nil
 }

pkg/file/scan.go

@@ -638,7 +638,7 @@ func (s *scanJob) onNewFile(ctx context.Context, f scanFile) (File, error) {
 			return fmt.Errorf("creating file %q: %w", path, err)
 		}
 
-		if err := s.fireHandlers(ctx, file); err != nil {
+		if err := s.fireHandlers(ctx, file, nil); err != nil {
 			return err
 		}
 
@@ -662,9 +662,9 @@ func (s *scanJob) fireDecorators(ctx context.Context, fs FS, f File) (File, erro
 	return f, nil
 }
 
-func (s *scanJob) fireHandlers(ctx context.Context, f File) error {
+func (s *scanJob) fireHandlers(ctx context.Context, f File, oldFile File) error {
 	for _, h := range s.handlers {
-		if err := h.Handle(ctx, f); err != nil {
+		if err := h.Handle(ctx, f, oldFile); err != nil {
 			return err
 		}
 	}
@@ -774,6 +774,10 @@ func (s *scanJob) handleRename(ctx context.Context, f File, fp []Fingerprint) (F
 			return fmt.Errorf("updating file for rename %q: %w", fBase.Path, err)
 		}
 
+		if err := s.fireHandlers(ctx, f, other); err != nil {
+			return err
+		}
+
 		return nil
 	}); err != nil {
 		return nil, err
@@ -888,6 +892,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
 		return s.onUnchangedFile(ctx, f, existing)
 	}
 
+	oldBase := *base
+
 	logger.Infof("%s has been updated: rescanning", path)
 	base.ModTime = fileModTime
 	base.Size = f.Size
@@ -914,7 +920,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
 			return fmt.Errorf("updating file %q: %w", path, err)
 		}
 
-		if err := s.fireHandlers(ctx, existing); err != nil {
+		if err := s.fireHandlers(ctx, existing, &oldBase); err != nil {
 			return err
 		}
 
@@ -991,7 +997,7 @@ func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File
 	}
 
 	if err := s.withTxn(ctx, func(ctx context.Context) error {
-		if err := s.fireHandlers(ctx, existing); err != nil {
+		if err := s.fireHandlers(ctx, existing, nil); err != nil {
 			return err
 		}
 
@@ -1002,9 +1008,5 @@ func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File
 
 	// if this file is a zip file, then we need to rescan the contents
 	// as well. We do this by returning the file, instead of nil.
-	if isMissingMetdata {
 	return existing, nil
-	}
-
-	return nil, nil
 }

pkg/fsutil/dir.go

@@ -34,6 +34,17 @@ func IsPathInDir(dir, pathToCheck string) bool {
 	return false
 }
 
+// IsPathInDirs returns true if pathToCheck is within anys of the paths in dirs.
+func IsPathInDirs(dirs []string, pathToCheck string) bool {
+	for _, dir := range dirs {
+		if IsPathInDir(dir, pathToCheck) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // GetHomeDirectory returns the path of the user's home directory.  ~ on Unix and C:\Users\UserName on Windows
 func GetHomeDirectory() string {
 	currentUser, err := user.Current()

pkg/gallery/scan.go

@@ -27,14 +27,19 @@ type SceneFinderUpdater interface {
 	AddGalleryIDs(ctx context.Context, sceneID int, galleryIDs []int) error
 }
 
+type ImageFinderUpdater interface {
+	FindByZipFileID(ctx context.Context, zipFileID file.ID) ([]*models.Image, error)
+	UpdatePartial(ctx context.Context, id int, partial models.ImagePartial) (*models.Image, error)
+}
+
 type ScanHandler struct {
 	CreatorUpdater     FullCreatorUpdater
 	SceneFinderUpdater SceneFinderUpdater
-
+	ImageFinderUpdater ImageFinderUpdater
 	PluginCache        *plugin.Cache
 }
 
-func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
+func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error {
 	baseFile := f.Base()
 
 	// try to match the file to a gallery
@@ -52,10 +57,23 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 	}
 
 	if len(existing) > 0 {
-		if err := h.associateExisting(ctx, existing, f); err != nil {
+		updateExisting := oldFile != nil
+		if err := h.associateExisting(ctx, existing, f, updateExisting); err != nil {
 			return err
 		}
 	} else {
+		// only create galleries if there is something to put in them
+		// otherwise, they will be created on the fly when an image is created
+		images, err := h.ImageFinderUpdater.FindByZipFileID(ctx, f.Base().ID)
+		if err != nil {
+			return err
+		}
+
+		if len(images) == 0 {
+			// don't create an empty gallery
+			return nil
+		}
+
 		// create a new gallery
 		now := time.Now()
 		newGallery := &models.Gallery{
@@ -71,6 +89,19 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 
 		h.PluginCache.RegisterPostHooks(ctx, newGallery.ID, plugin.GalleryCreatePost, nil, nil)
 
+		// associate all the images in the zip file with the gallery
+		for _, i := range images {
+			if _, err := h.ImageFinderUpdater.UpdatePartial(ctx, i.ID, models.ImagePartial{
+				GalleryIDs: &models.UpdateIDs{
+					IDs:  []int{newGallery.ID},
+					Mode: models.RelationshipUpdateModeAdd,
+				},
+				UpdatedAt: models.NewOptionalTime(now),
+			}); err != nil {
+				return fmt.Errorf("adding image %s to gallery: %w", i.Path, err)
+			}
+		}
+
 		existing = []*models.Gallery{newGallery}
 	}
 
@@ -81,7 +112,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 	return nil
 }
 
-func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Gallery, f file.File) error {
+func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Gallery, f file.File, updateExisting bool) error {
 	for _, i := range existing {
 		if err := i.LoadFiles(ctx, h.CreatorUpdater); err != nil {
 			return err
@@ -107,6 +138,9 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
 			}
 		}
 
+		if !found || updateExisting {
+			h.PluginCache.RegisterPostHooks(ctx, i.ID, plugin.GalleryUpdatePost, nil, nil)
+		}
 	}
 
 	return nil

pkg/image/scan.go

@@ -4,14 +4,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/paths"
 	"github.com/stashapp/stash/pkg/plugin"
 	"github.com/stashapp/stash/pkg/sliceutil/intslice"
+	"github.com/stashapp/stash/pkg/txn"
 )
 
 var (
@@ -20,6 +23,7 @@ var (
 
 type FinderCreatorUpdater interface {
 	FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error)
+	FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Image, error)
 	FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error)
 	Create(ctx context.Context, newImage *models.ImageCreateInput) error
 	UpdatePartial(ctx context.Context, id int, updatedImage models.ImagePartial) (*models.Image, error)
@@ -48,6 +52,8 @@ type ScanHandler struct {
 	ScanConfig ScanConfig
 
 	PluginCache *plugin.Cache
+
+	Paths *paths.Paths
 }
 
 func (h *ScanHandler) validate() error {
@@ -60,11 +66,34 @@ func (h *ScanHandler) validate() error {
 	if h.ScanConfig == nil {
 		return errors.New("ScanConfig is required")
 	}
+	if h.Paths == nil {
+		return errors.New("Paths is required")
+	}
 
 	return nil
 }
 
-func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
+func (h *ScanHandler) logInfo(ctx context.Context, format string, args ...interface{}) {
+	// log at the end so that if anything fails above due to a locked database
+	// error and the transaction must be retried, then we shouldn't get multiple
+	// logs of the same thing.
+	txn.AddPostCompleteHook(ctx, func(ctx context.Context) error {
+		logger.Infof(format, args...)
+		return nil
+	})
+}
+
+func (h *ScanHandler) logError(ctx context.Context, format string, args ...interface{}) {
+	// log at the end so that if anything fails above due to a locked database
+	// error and the transaction must be retried, then we shouldn't get multiple
+	// logs of the same thing.
+	txn.AddPostCompleteHook(ctx, func(ctx context.Context) error {
+		logger.Errorf(format, args...)
+		return nil
+	})
+}
+
+func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error {
 	if err := h.validate(); err != nil {
 		return err
 	}
@@ -89,7 +118,9 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 	}
 
 	if len(existing) > 0 {
-		if err := h.associateExisting(ctx, existing, imageFile); err != nil {
+		updateExisting := oldFile != nil
+
+		if err := h.associateExisting(ctx, existing, imageFile, updateExisting); err != nil {
 			return err
 		}
 	} else {
@@ -101,23 +132,11 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 			GalleryIDs: models.NewRelatedIDs([]int{}),
 		}
 
-		// if the file is in a zip, then associate it with the gallery
+		h.logInfo(ctx, "%s doesn't exist. Creating new image...", f.Base().Path)
-		if imageFile.ZipFileID != nil {
-			g, err := h.GalleryFinder.FindByFileID(ctx, *imageFile.ZipFileID)
-			if err != nil {
-				return fmt.Errorf("finding gallery for zip file id %d: %w", *imageFile.ZipFileID, err)
-			}
 
-			for _, gg := range g {
+		if _, err := h.associateGallery(ctx, newImage, imageFile); err != nil {
-				newImage.GalleryIDs.Add(gg.ID)
-			}
-		} else if h.ScanConfig.GetCreateGalleriesFromFolders() {
-			if err := h.associateFolderBasedGallery(ctx, newImage, imageFile); err != nil {
 			return err
 		}
-		}
-
-		logger.Infof("%s doesn't exist. Creating new image...", f.Base().Path)
 
 		if err := h.CreatorUpdater.Create(ctx, &models.ImageCreateInput{
 			Image:   newImage,
@@ -131,11 +150,22 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 		existing = []*models.Image{newImage}
 	}
 
+	// remove the old thumbnail if the checksum changed - we'll regenerate it
+	if oldFile != nil {
+		oldHash := oldFile.Base().Fingerprints.GetString(file.FingerprintTypeMD5)
+		newHash := f.Base().Fingerprints.GetString(file.FingerprintTypeMD5)
+
+		if oldHash != "" && newHash != "" && oldHash != newHash {
+			// remove cache dir of gallery
+			_ = os.Remove(h.Paths.Generated.GetThumbnailPath(oldHash, models.DefaultGthumbWidth))
+		}
+	}
+
 	if h.ScanConfig.IsGenerateThumbnails() {
 		for _, s := range existing {
 			if err := h.ThumbnailGenerator.GenerateThumbnail(ctx, s, imageFile); err != nil {
 				// just log if cover generation fails. We can try again on rescan
-				logger.Errorf("Error generating thumbnail for %s: %v", imageFile.Path, err)
+				h.logError(ctx, "Error generating thumbnail for %s: %v", imageFile.Path, err)
 			}
 		}
 	}
@@ -143,7 +173,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 	return nil
 }
 
-func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Image, f *file.ImageFile) error {
+func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Image, f *file.ImageFile, updateExisting bool) error {
 	for _, i := range existing {
 		if err := i.LoadFiles(ctx, h.CreatorUpdater); err != nil {
 			return err
@@ -157,35 +187,49 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
 			}
 		}
 
-		if !found {
+		// associate with gallery if applicable
-			logger.Infof("Adding %s to image %s", f.Path, i.DisplayName())
+		changed, err := h.associateGallery(ctx, i, f)
-
+		if err != nil {
-			// associate with folder-based gallery if applicable
-			if h.ScanConfig.GetCreateGalleriesFromFolders() {
-				if err := h.associateFolderBasedGallery(ctx, i, f); err != nil {
 			return err
 		}
+
+		var galleryIDs *models.UpdateIDs
+		if changed {
+			galleryIDs = &models.UpdateIDs{
+				IDs:  i.GalleryIDs.List(),
+				Mode: models.RelationshipUpdateModeSet,
 			}
+		}
+
+		if !found {
+			h.logInfo(ctx, "Adding %s to image %s", f.Path, i.DisplayName())
 
 			if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.ID); err != nil {
 				return fmt.Errorf("adding file to image: %w", err)
 			}
-			// update updated_at time
+
-			if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewImagePartial()); err != nil {
+			changed = true
+		}
+
+		if changed {
+			// always update updated_at time
+			if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.ImagePartial{
+				GalleryIDs: galleryIDs,
+				UpdatedAt:  models.NewOptionalTime(time.Now()),
+			}); err != nil {
 				return fmt.Errorf("updating image: %w", err)
 			}
 		}
+
+		if changed || updateExisting {
+			h.PluginCache.RegisterPostHooks(ctx, i.ID, plugin.ImageUpdatePost, nil, nil)
+		}
 	}
 
 	return nil
 }
 
 func (h *ScanHandler) getOrCreateFolderBasedGallery(ctx context.Context, f file.File) (*models.Gallery, error) {
-	// don't create folder-based galleries for files in zip file
-	if f.Base().ZipFileID != nil {
-		return nil, nil
-	}
-
 	folderID := f.Base().ParentFolderID
 	g, err := h.GalleryFinder.FindByFolderID(ctx, folderID)
 	if err != nil {
@@ -205,28 +249,100 @@ func (h *ScanHandler) getOrCreateFolderBasedGallery(ctx context.Context, f file.
 		UpdatedAt: now,
 	}
 
-	logger.Infof("Creating folder-based gallery for %s", filepath.Dir(f.Base().Path))
+	h.logInfo(ctx, "Creating folder-based gallery for %s", filepath.Dir(f.Base().Path))
+
 	if err := h.GalleryFinder.Create(ctx, newGallery, nil); err != nil {
 		return nil, fmt.Errorf("creating folder based gallery: %w", err)
 	}
 
+	// it's possible that there are other images in the folder that
+	// need to be added to the new gallery. Find and add them now.
+	if err := h.associateFolderImages(ctx, newGallery); err != nil {
+		return nil, fmt.Errorf("associating existing folder images: %w", err)
+	}
+
 	return newGallery, nil
 }
 
-func (h *ScanHandler) associateFolderBasedGallery(ctx context.Context, newImage *models.Image, f file.File) error {
+func (h *ScanHandler) associateFolderImages(ctx context.Context, g *models.Gallery) error {
-	g, err := h.getOrCreateFolderBasedGallery(ctx, f)
+	i, err := h.CreatorUpdater.FindByFolderID(ctx, *g.FolderID)
 	if err != nil {
-		return err
+		return fmt.Errorf("finding images in folder: %w", err)
 	}
 
-	if err := newImage.LoadGalleryIDs(ctx, h.CreatorUpdater); err != nil {
+	for _, ii := range i {
-		return err
+		h.logInfo(ctx, "Adding %s to gallery %s", ii.Path, g.Path)
-	}
 
-	if g != nil && !intslice.IntInclude(newImage.GalleryIDs.List(), g.ID) {
+		if _, err := h.CreatorUpdater.UpdatePartial(ctx, ii.ID, models.ImagePartial{
-		newImage.GalleryIDs.Add(g.ID)
+			GalleryIDs: &models.UpdateIDs{
-		logger.Infof("Adding %s to folder-based gallery %s", f.Base().Path, g.Path)
+				IDs:  []int{g.ID},
+				Mode: models.RelationshipUpdateModeAdd,
+			},
+			UpdatedAt: models.NewOptionalTime(time.Now()),
+		}); err != nil {
+			return fmt.Errorf("updating image: %w", err)
+		}
 	}
 
 	return nil
 }
+
+func (h *ScanHandler) getOrCreateZipBasedGallery(ctx context.Context, zipFile file.File) (*models.Gallery, error) {
+	g, err := h.GalleryFinder.FindByFileID(ctx, zipFile.Base().ID)
+	if err != nil {
+		return nil, fmt.Errorf("finding zip based gallery: %w", err)
+	}
+
+	if len(g) > 0 {
+		gg := g[0]
+		return gg, nil
+	}
+
+	// create a new zip-based gallery
+	now := time.Now()
+	newGallery := &models.Gallery{
+		CreatedAt: now,
+		UpdatedAt: now,
+	}
+
+	h.logInfo(ctx, "%s doesn't exist. Creating new gallery...", zipFile.Base().Path)
+
+	if err := h.GalleryFinder.Create(ctx, newGallery, []file.ID{zipFile.Base().ID}); err != nil {
+		return nil, fmt.Errorf("creating zip-based gallery: %w", err)
+	}
+
+	return newGallery, nil
+}
+
+func (h *ScanHandler) getOrCreateGallery(ctx context.Context, f file.File) (*models.Gallery, error) {
+	// don't create folder-based galleries for files in zip file
+	if f.Base().ZipFile != nil {
+		return h.getOrCreateZipBasedGallery(ctx, f.Base().ZipFile)
+	}
+
+	if h.ScanConfig.GetCreateGalleriesFromFolders() {
+		return h.getOrCreateFolderBasedGallery(ctx, f)
+	}
+
+	return nil, nil
+}
+
+func (h *ScanHandler) associateGallery(ctx context.Context, newImage *models.Image, f file.File) (bool, error) {
+	g, err := h.getOrCreateGallery(ctx, f)
+	if err != nil {
+		return false, err
+	}
+
+	if err := newImage.LoadGalleryIDs(ctx, h.CreatorUpdater); err != nil {
+		return false, err
+	}
+
+	ret := false
+	if g != nil && !intslice.IntInclude(newImage.GalleryIDs.List(), g.ID) {
+		ret = true
+		newImage.GalleryIDs.Add(g.ID)
+		h.logInfo(ctx, "Adding %s to gallery %s", f.Base().Path, g.Path)
+	}
+
+	return ret, nil
+}

pkg/models/model_file.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"io"
 	"strconv"
-	"time"
 )
 
 type HashAlgorithm string
@@ -48,28 +47,3 @@ func (e *HashAlgorithm) UnmarshalGQL(v interface{}) error {
 func (e HashAlgorithm) MarshalGQL(w io.Writer) {
 	fmt.Fprint(w, strconv.Quote(e.String()))
 }
-
-type File struct {
-	Checksum    string    `db:"checksum" json:"checksum"`
-	OSHash      string    `db:"oshash" json:"oshash"`
-	Path        string    `db:"path" json:"path"`
-	Size        string    `db:"size" json:"size"`
-	FileModTime time.Time `db:"file_mod_time" json:"file_mod_time"`
-}
-
-// GetHash returns the hash of the scene, based on the hash algorithm provided. If
-// hash algorithm is MD5, then Checksum is returned. Otherwise, OSHash is returned.
-func (s File) GetHash(hashAlgorithm HashAlgorithm) string {
-	switch hashAlgorithm {
-	case HashAlgorithmMd5:
-		return s.Checksum
-	case HashAlgorithmOshash:
-		return s.OSHash
-	default:
-		panic("unknown hash algorithm")
-	}
-}
-
-func (s File) Equal(o File) bool {
-	return s.Path == o.Path && s.Checksum == o.Checksum && s.OSHash == o.OSHash && s.Size == o.Size && s.FileModTime.Equal(o.FileModTime)
-}

pkg/scene/hash.go

@@ -0,0 +1,19 @@
+package scene
+
+import (
+	"github.com/stashapp/stash/pkg/file"
+	"github.com/stashapp/stash/pkg/models"
+)
+
+// GetHash returns the hash of the file, based on the hash algorithm provided. If
+// hash algorithm is MD5, then Checksum is returned. Otherwise, OSHash is returned.
+func GetHash(f file.File, hashAlgorithm models.HashAlgorithm) string {
+	switch hashAlgorithm {
+	case models.HashAlgorithmMd5:
+		return f.Base().Fingerprints.GetString(file.FingerprintTypeMD5)
+	case models.HashAlgorithmOshash:
+		return f.Base().Fingerprints.GetString(file.FingerprintTypeOshash)
+	default:
+		panic("unknown hash algorithm")
+	}
+}

pkg/scene/scan.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/paths"
 	"github.com/stashapp/stash/pkg/plugin"
 )
 
@@ -35,6 +36,9 @@ type ScanHandler struct {
 	CoverGenerator CoverGenerator
 	ScanGenerator  ScanGenerator
 	PluginCache    *plugin.Cache
+
+	FileNamingAlgorithm models.HashAlgorithm
+	Paths               *paths.Paths
 }
 
 func (h *ScanHandler) validate() error {
@@ -47,11 +51,17 @@ func (h *ScanHandler) validate() error {
 	if h.ScanGenerator == nil {
 		return errors.New("ScanGenerator is required")
 	}
+	if !h.FileNamingAlgorithm.IsValid() {
+		return errors.New("FileNamingAlgorithm is required")
+	}
+	if h.Paths == nil {
+		return errors.New("Paths is required")
+	}
 
 	return nil
 }
 
-func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
+func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error {
 	if err := h.validate(); err != nil {
 		return err
 	}
@@ -76,7 +86,8 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 	}
 
 	if len(existing) > 0 {
-		if err := h.associateExisting(ctx, existing, videoFile); err != nil {
+		updateExisting := oldFile != nil
+		if err := h.associateExisting(ctx, existing, videoFile, updateExisting); err != nil {
 			return err
 		}
 	} else {
@@ -98,6 +109,16 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 		existing = []*models.Scene{newScene}
 	}
 
+	if oldFile != nil {
+		// migrate hashes from the old file to the new
+		oldHash := GetHash(oldFile, h.FileNamingAlgorithm)
+		newHash := GetHash(f, h.FileNamingAlgorithm)
+
+		if oldHash != "" && newHash != "" && oldHash != newHash {
+			MigrateHash(h.Paths, oldHash, newHash)
+		}
+	}
+
 	for _, s := range existing {
 		if err := h.CoverGenerator.GenerateCover(ctx, s, videoFile); err != nil {
 			// just log if cover generation fails. We can try again on rescan
@@ -113,7 +134,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
 	return nil
 }
 
-func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Scene, f *file.VideoFile) error {
+func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Scene, f *file.VideoFile, updateExisting bool) error {
 	for _, s := range existing {
 		if err := s.LoadFiles(ctx, h.CreatorUpdater); err != nil {
 			return err
@@ -139,6 +160,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
 				return fmt.Errorf("updating scene: %w", err)
 			}
 		}
+
+		if !found || updateExisting {
+			h.PluginCache.RegisterPostHooks(ctx, s.ID, plugin.SceneUpdatePost, nil, nil)
+		}
 	}
 
 	return nil

pkg/txn/hooks.go

@@ -13,6 +13,7 @@ const (
 type hookManager struct {
 	postCommitHooks   []TxnFunc
 	postRollbackHooks []TxnFunc
+	postCompleteHooks []TxnFunc
 }
 
 func (m *hookManager) register(ctx context.Context) context.Context {
@@ -27,20 +28,26 @@ func hookManagerCtx(ctx context.Context) *hookManager {
 	return m
 }
 
-func executePostCommitHooks(ctx context.Context) {
+func executeHooks(ctx context.Context, hooks []TxnFunc) {
-	m := hookManagerCtx(ctx)
+	for _, h := range hooks {
-	for _, h := range m.postCommitHooks {
 		// ignore errors
 		_ = h(ctx)
 	}
 }
 
+func executePostCommitHooks(ctx context.Context) {
+	m := hookManagerCtx(ctx)
+	executeHooks(ctx, m.postCommitHooks)
+}
+
 func executePostRollbackHooks(ctx context.Context) {
 	m := hookManagerCtx(ctx)
-	for _, h := range m.postRollbackHooks {
+	executeHooks(ctx, m.postRollbackHooks)
-		// ignore errors
+}
-		_ = h(ctx)
+
-	}
+func executePostCompleteHooks(ctx context.Context) {
+	m := hookManagerCtx(ctx)
+	executeHooks(ctx, m.postCompleteHooks)
 }
 
 func AddPostCommitHook(ctx context.Context, hook TxnFunc) {
@@ -52,3 +59,8 @@ func AddPostRollbackHook(ctx context.Context, hook TxnFunc) {
 	m := hookManagerCtx(ctx)
 	m.postRollbackHooks = append(m.postRollbackHooks, hook)
 }
+
+func AddPostCompleteHook(ctx context.Context, hook TxnFunc) {
+	m := hookManagerCtx(ctx)
+	m.postCompleteHooks = append(m.postCompleteHooks, hook)
+}

pkg/txn/transaction.go

@@ -22,6 +22,11 @@ type TxnFunc func(ctx context.Context) error
 // WithTxn executes fn in a transaction. If fn returns an error then
 // the transaction is rolled back. Otherwise it is committed.
 func WithTxn(ctx context.Context, m Manager, fn TxnFunc) error {
+	const execComplete = true
+	return withTxn(ctx, m, fn, execComplete)
+}
+
+func withTxn(ctx context.Context, m Manager, fn TxnFunc, execCompleteOnLocked bool) error {
 	var err error
 	ctx, err = begin(ctx, m)
 	if err != nil {
@@ -38,10 +43,16 @@ func WithTxn(ctx context.Context, m Manager, fn TxnFunc) error {
 		if err != nil {
 			// something went wrong, rollback
 			rollback(ctx, m)
+
+			if execCompleteOnLocked || !m.IsLocked(err) {
+				executePostCompleteHooks(ctx)
+			}
 		} else {
 			// all good, commit
 			err = commit(ctx, m)
+			executePostCompleteHooks(ctx)
 		}
+
 	}()
 
 	err = fn(ctx)
@@ -102,7 +113,8 @@ func (r Retryer) WithTxn(ctx context.Context, fn TxnFunc) error {
 	var attempt int
 	var err error
 	for attempt = 1; attempt <= r.Retries || r.Retries < 0; attempt++ {
-		err = WithTxn(ctx, r.Manager, fn)
+		const execComplete = false
+		err = withTxn(ctx, r.Manager, fn, execComplete)
 
 		if err == nil {
 			return nil