mirror of
https://github.com/stashapp/stash.git
synced 2025-12-06 08:26:00 +01:00
Handle file rescan (#2951)
* Fire handlers when file updated or moved * Create galleries as needed * Clean empty galleries * Handle cleaning zip folders when path changed * Fix gallery association on duplicate images * Re-create missing folder-based galleries
This commit is contained in:
WithoutPants
GitHub
parent
00820a8789
commit
dce90a3ed9
13 changed files with 439 additions and 106 deletions
|
|
@ -16,6 +16,7 @@ import (
|
|||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/plugin"
|
||||
"github.com/stashapp/stash/pkg/scene"
|
||||
"github.com/stashapp/stash/pkg/txn"
|
||||
)
|
||||
|
||||
type cleaner interface {
|
||||
|
|
@ -49,11 +50,95 @@ func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) {
|
|||
return
|
||||
}
|
||||
|
||||
j.cleanEmptyGalleries(ctx)
|
||||
|
||||
j.scanSubs.notify()
|
||||
elapsed := time.Since(start)
|
||||
logger.Info(fmt.Sprintf("Finished Cleaning (%s)", elapsed))
|
||||
}
|
||||
|
||||
func (j *cleanJob) cleanEmptyGalleries(ctx context.Context) {
|
||||
const batchSize = 1000
|
||||
var toClean []int
|
||||
findFilter := models.BatchFindFilter(batchSize)
|
||||
if err := txn.WithTxn(ctx, j.txnManager, func(ctx context.Context) error {
|
||||
found := true
|
||||
for found {
|
||||
emptyGalleries, _, err := j.txnManager.Gallery.Query(ctx, &models.GalleryFilterType{
|
||||
ImageCount: &models.IntCriterionInput{
|
||||
Value: 0,
|
||||
Modifier: models.CriterionModifierEquals,
|
||||
},
|
||||
}, findFilter)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
found = len(emptyGalleries) > 0
|
||||
|
||||
for _, g := range emptyGalleries {
|
||||
if g.Path == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(j.input.Paths) > 0 && !fsutil.IsPathInDirs(j.input.Paths, g.Path) {
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Infof("Gallery has 0 images. Marking to clean: %s", g.DisplayName())
|
||||
toClean = append(toClean, g.ID)
|
||||
}
|
||||
|
||||
*findFilter.Page++
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
logger.Errorf("Error finding empty galleries: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !j.input.DryRun {
|
||||
for _, id := range toClean {
|
||||
j.deleteGallery(ctx, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (j *cleanJob) deleteGallery(ctx context.Context, id int) {
|
||||
pluginCache := GetInstance().PluginCache
|
||||
qb := j.txnManager.Gallery
|
||||
|
||||
if err := txn.WithTxn(ctx, j.txnManager, func(ctx context.Context) error {
|
||||
g, err := qb.Find(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if g == nil {
|
||||
return fmt.Errorf("gallery not found: %d", id)
|
||||
}
|
||||
|
||||
if err := g.LoadPrimaryFile(ctx, j.txnManager.File); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := qb.Destroy(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pluginCache.RegisterPostHooks(ctx, id, plugin.GalleryDestroyPost, plugin.GalleryDestroyInput{
|
||||
Checksum: g.PrimaryChecksum(),
|
||||
Path: g.Path,
|
||||
}, nil)
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
logger.Errorf("Error deleting gallery from database: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
type cleanFilter struct {
|
||||
scanFilter
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/99designs/gqlgen/graphql/handler/lru"
|
||||
"github.com/stashapp/stash/internal/manager/config"
|
||||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/file/video"
|
||||
|
|
@ -61,7 +62,9 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) {
|
|||
ScanFilters: []file.PathFilter{newScanFilter(instance.Config, minModTime)},
|
||||
ZipFileExtensions: instance.Config.GetGalleryExtensions(),
|
||||
ParallelTasks: instance.Config.GetParallelTasksWithAutoDetection(),
|
||||
HandlerRequiredFilters: []file.Filter{newHandlerRequiredFilter(instance.Config)},
|
||||
HandlerRequiredFilters: []file.Filter{
|
||||
newHandlerRequiredFilter(instance.Config),
|
||||
},
|
||||
}, progress)
|
||||
|
||||
taskQueue.Close()
|
||||
|
|
@ -95,22 +98,31 @@ type fileCounter interface {
|
|||
CountByFileID(ctx context.Context, fileID file.ID) (int, error)
|
||||
}
|
||||
|
||||
type galleryFinder interface {
|
||||
fileCounter
|
||||
FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Gallery, error)
|
||||
}
|
||||
|
||||
// handlerRequiredFilter returns true if a File's handler needs to be executed despite the file not being updated.
|
||||
type handlerRequiredFilter struct {
|
||||
extensionConfig
|
||||
SceneFinder fileCounter
|
||||
ImageFinder fileCounter
|
||||
GalleryFinder fileCounter
|
||||
GalleryFinder galleryFinder
|
||||
|
||||
FolderCache *lru.LRU
|
||||
}
|
||||
|
||||
func newHandlerRequiredFilter(c *config.Instance) *handlerRequiredFilter {
|
||||
db := instance.Database
|
||||
processes := c.GetParallelTasksWithAutoDetection()
|
||||
|
||||
return &handlerRequiredFilter{
|
||||
extensionConfig: newExtensionConfig(c),
|
||||
SceneFinder: db.Scene,
|
||||
ImageFinder: db.Image,
|
||||
GalleryFinder: db.Gallery,
|
||||
FolderCache: lru.New(processes * 2),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -143,7 +155,32 @@ func (f *handlerRequiredFilter) Accept(ctx context.Context, ff file.File) bool {
|
|||
}
|
||||
|
||||
// execute handler if there are no related objects
|
||||
return n == 0
|
||||
if n == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// if create galleries from folder is enabled and the file is not in a zip
|
||||
// file, then check if there is a folder-based gallery for the file's
|
||||
// directory
|
||||
if isImageFile && instance.Config.GetCreateGalleriesFromFolders() && ff.Base().ZipFileID == nil {
|
||||
// only do this for the first time it encounters the folder
|
||||
// the first instance should create the gallery
|
||||
_, found := f.FolderCache.Get(ctx, ff.Base().ParentFolderID.String())
|
||||
if found {
|
||||
// should already be handled
|
||||
return false
|
||||
}
|
||||
|
||||
g, _ := f.GalleryFinder.FindByFolderID(ctx, ff.Base().ParentFolderID)
|
||||
f.FolderCache.Add(ctx, ff.Base().ParentFolderID.String(), true)
|
||||
|
||||
if len(g) == 0 {
|
||||
// no folder gallery. Return true so that it creates one.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
type scanFilter struct {
|
||||
|
|
@ -248,6 +285,7 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre
|
|||
isGenerateThumbnails: options.ScanGenerateThumbnails,
|
||||
},
|
||||
PluginCache: pluginCache,
|
||||
Paths: instance.Paths,
|
||||
},
|
||||
},
|
||||
&file.FilteredHandler{
|
||||
|
|
@ -255,6 +293,7 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre
|
|||
Handler: &gallery.ScanHandler{
|
||||
CreatorUpdater: db.Gallery,
|
||||
SceneFinderUpdater: db.Scene,
|
||||
ImageFinderUpdater: db.Image,
|
||||
PluginCache: pluginCache,
|
||||
},
|
||||
},
|
||||
|
|
@ -269,6 +308,8 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre
|
|||
taskQueue: taskQueue,
|
||||
progress: progress,
|
||||
},
|
||||
FileNamingAlgorithm: instance.Config.GetVideoFileNamingAlgorithm(),
|
||||
Paths: instance.Paths,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -338,7 +338,9 @@ func (j *cleanJob) shouldCleanFolder(ctx context.Context, f *Folder) bool {
|
|||
path := f.Path
|
||||
|
||||
info, err := f.Info(j.FS)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
// ErrInvalid can occur in zip files where the zip file path changed
|
||||
// and the underlying folder did not
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, fs.ErrInvalid) {
|
||||
logger.Errorf("error getting folder info for %q, not cleaning: %v", path, err)
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ func (ff FilterFunc) Accept(ctx context.Context, f File) bool {
|
|||
|
||||
// Handler provides a handler for Files.
|
||||
type Handler interface {
|
||||
Handle(ctx context.Context, f File) error
|
||||
Handle(ctx context.Context, f File, oldFile File) error
|
||||
}
|
||||
|
||||
// FilteredHandler is a Handler runs only if the filter accepts the file.
|
||||
|
|
@ -39,9 +39,9 @@ type FilteredHandler struct {
|
|||
}
|
||||
|
||||
// Handle runs the handler if the filter accepts the file.
|
||||
func (h *FilteredHandler) Handle(ctx context.Context, f File) error {
|
||||
func (h *FilteredHandler) Handle(ctx context.Context, f File, oldFile File) error {
|
||||
if h.Accept(ctx, f) {
|
||||
return h.Handler.Handle(ctx, f)
|
||||
return h.Handler.Handle(ctx, f, oldFile)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -638,7 +638,7 @@ func (s *scanJob) onNewFile(ctx context.Context, f scanFile) (File, error) {
|
|||
return fmt.Errorf("creating file %q: %w", path, err)
|
||||
}
|
||||
|
||||
if err := s.fireHandlers(ctx, file); err != nil {
|
||||
if err := s.fireHandlers(ctx, file, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -662,9 +662,9 @@ func (s *scanJob) fireDecorators(ctx context.Context, fs FS, f File) (File, erro
|
|||
return f, nil
|
||||
}
|
||||
|
||||
func (s *scanJob) fireHandlers(ctx context.Context, f File) error {
|
||||
func (s *scanJob) fireHandlers(ctx context.Context, f File, oldFile File) error {
|
||||
for _, h := range s.handlers {
|
||||
if err := h.Handle(ctx, f); err != nil {
|
||||
if err := h.Handle(ctx, f, oldFile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -774,6 +774,10 @@ func (s *scanJob) handleRename(ctx context.Context, f File, fp []Fingerprint) (F
|
|||
return fmt.Errorf("updating file for rename %q: %w", fBase.Path, err)
|
||||
}
|
||||
|
||||
if err := s.fireHandlers(ctx, f, other); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -888,6 +892,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
|
|||
return s.onUnchangedFile(ctx, f, existing)
|
||||
}
|
||||
|
||||
oldBase := *base
|
||||
|
||||
logger.Infof("%s has been updated: rescanning", path)
|
||||
base.ModTime = fileModTime
|
||||
base.Size = f.Size
|
||||
|
|
@ -914,7 +920,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
|
|||
return fmt.Errorf("updating file %q: %w", path, err)
|
||||
}
|
||||
|
||||
if err := s.fireHandlers(ctx, existing); err != nil {
|
||||
if err := s.fireHandlers(ctx, existing, &oldBase); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -991,7 +997,7 @@ func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File
|
|||
}
|
||||
|
||||
if err := s.withTxn(ctx, func(ctx context.Context) error {
|
||||
if err := s.fireHandlers(ctx, existing); err != nil {
|
||||
if err := s.fireHandlers(ctx, existing, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -1002,9 +1008,5 @@ func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File
|
|||
|
||||
// if this file is a zip file, then we need to rescan the contents
|
||||
// as well. We do this by returning the file, instead of nil.
|
||||
if isMissingMetdata {
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,17 @@ func IsPathInDir(dir, pathToCheck string) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
// IsPathInDirs returns true if pathToCheck is within anys of the paths in dirs.
|
||||
func IsPathInDirs(dirs []string, pathToCheck string) bool {
|
||||
for _, dir := range dirs {
|
||||
if IsPathInDir(dir, pathToCheck) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// GetHomeDirectory returns the path of the user's home directory. ~ on Unix and C:\Users\UserName on Windows
|
||||
func GetHomeDirectory() string {
|
||||
currentUser, err := user.Current()
|
||||
|
|
|
|||
|
|
@ -27,14 +27,19 @@ type SceneFinderUpdater interface {
|
|||
AddGalleryIDs(ctx context.Context, sceneID int, galleryIDs []int) error
|
||||
}
|
||||
|
||||
type ImageFinderUpdater interface {
|
||||
FindByZipFileID(ctx context.Context, zipFileID file.ID) ([]*models.Image, error)
|
||||
UpdatePartial(ctx context.Context, id int, partial models.ImagePartial) (*models.Image, error)
|
||||
}
|
||||
|
||||
type ScanHandler struct {
|
||||
CreatorUpdater FullCreatorUpdater
|
||||
SceneFinderUpdater SceneFinderUpdater
|
||||
|
||||
ImageFinderUpdater ImageFinderUpdater
|
||||
PluginCache *plugin.Cache
|
||||
}
|
||||
|
||||
func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
||||
func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error {
|
||||
baseFile := f.Base()
|
||||
|
||||
// try to match the file to a gallery
|
||||
|
|
@ -52,10 +57,23 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
}
|
||||
|
||||
if len(existing) > 0 {
|
||||
if err := h.associateExisting(ctx, existing, f); err != nil {
|
||||
updateExisting := oldFile != nil
|
||||
if err := h.associateExisting(ctx, existing, f, updateExisting); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// only create galleries if there is something to put in them
|
||||
// otherwise, they will be created on the fly when an image is created
|
||||
images, err := h.ImageFinderUpdater.FindByZipFileID(ctx, f.Base().ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(images) == 0 {
|
||||
// don't create an empty gallery
|
||||
return nil
|
||||
}
|
||||
|
||||
// create a new gallery
|
||||
now := time.Now()
|
||||
newGallery := &models.Gallery{
|
||||
|
|
@ -71,6 +89,19 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
|
||||
h.PluginCache.RegisterPostHooks(ctx, newGallery.ID, plugin.GalleryCreatePost, nil, nil)
|
||||
|
||||
// associate all the images in the zip file with the gallery
|
||||
for _, i := range images {
|
||||
if _, err := h.ImageFinderUpdater.UpdatePartial(ctx, i.ID, models.ImagePartial{
|
||||
GalleryIDs: &models.UpdateIDs{
|
||||
IDs: []int{newGallery.ID},
|
||||
Mode: models.RelationshipUpdateModeAdd,
|
||||
},
|
||||
UpdatedAt: models.NewOptionalTime(now),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("adding image %s to gallery: %w", i.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
existing = []*models.Gallery{newGallery}
|
||||
}
|
||||
|
||||
|
|
@ -81,7 +112,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Gallery, f file.File) error {
|
||||
func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Gallery, f file.File, updateExisting bool) error {
|
||||
for _, i := range existing {
|
||||
if err := i.LoadFiles(ctx, h.CreatorUpdater); err != nil {
|
||||
return err
|
||||
|
|
@ -107,6 +138,9 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
|
|||
}
|
||||
}
|
||||
|
||||
if !found || updateExisting {
|
||||
h.PluginCache.RegisterPostHooks(ctx, i.ID, plugin.GalleryUpdatePost, nil, nil)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -4,14 +4,17 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/models/paths"
|
||||
"github.com/stashapp/stash/pkg/plugin"
|
||||
"github.com/stashapp/stash/pkg/sliceutil/intslice"
|
||||
"github.com/stashapp/stash/pkg/txn"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -20,6 +23,7 @@ var (
|
|||
|
||||
type FinderCreatorUpdater interface {
|
||||
FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error)
|
||||
FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Image, error)
|
||||
FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error)
|
||||
Create(ctx context.Context, newImage *models.ImageCreateInput) error
|
||||
UpdatePartial(ctx context.Context, id int, updatedImage models.ImagePartial) (*models.Image, error)
|
||||
|
|
@ -48,6 +52,8 @@ type ScanHandler struct {
|
|||
ScanConfig ScanConfig
|
||||
|
||||
PluginCache *plugin.Cache
|
||||
|
||||
Paths *paths.Paths
|
||||
}
|
||||
|
||||
func (h *ScanHandler) validate() error {
|
||||
|
|
@ -60,11 +66,34 @@ func (h *ScanHandler) validate() error {
|
|||
if h.ScanConfig == nil {
|
||||
return errors.New("ScanConfig is required")
|
||||
}
|
||||
if h.Paths == nil {
|
||||
return errors.New("Paths is required")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
||||
func (h *ScanHandler) logInfo(ctx context.Context, format string, args ...interface{}) {
|
||||
// log at the end so that if anything fails above due to a locked database
|
||||
// error and the transaction must be retried, then we shouldn't get multiple
|
||||
// logs of the same thing.
|
||||
txn.AddPostCompleteHook(ctx, func(ctx context.Context) error {
|
||||
logger.Infof(format, args...)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (h *ScanHandler) logError(ctx context.Context, format string, args ...interface{}) {
|
||||
// log at the end so that if anything fails above due to a locked database
|
||||
// error and the transaction must be retried, then we shouldn't get multiple
|
||||
// logs of the same thing.
|
||||
txn.AddPostCompleteHook(ctx, func(ctx context.Context) error {
|
||||
logger.Errorf(format, args...)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error {
|
||||
if err := h.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -89,7 +118,9 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
}
|
||||
|
||||
if len(existing) > 0 {
|
||||
if err := h.associateExisting(ctx, existing, imageFile); err != nil {
|
||||
updateExisting := oldFile != nil
|
||||
|
||||
if err := h.associateExisting(ctx, existing, imageFile, updateExisting); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
|
|
@ -101,23 +132,11 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
GalleryIDs: models.NewRelatedIDs([]int{}),
|
||||
}
|
||||
|
||||
// if the file is in a zip, then associate it with the gallery
|
||||
if imageFile.ZipFileID != nil {
|
||||
g, err := h.GalleryFinder.FindByFileID(ctx, *imageFile.ZipFileID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finding gallery for zip file id %d: %w", *imageFile.ZipFileID, err)
|
||||
}
|
||||
h.logInfo(ctx, "%s doesn't exist. Creating new image...", f.Base().Path)
|
||||
|
||||
for _, gg := range g {
|
||||
newImage.GalleryIDs.Add(gg.ID)
|
||||
}
|
||||
} else if h.ScanConfig.GetCreateGalleriesFromFolders() {
|
||||
if err := h.associateFolderBasedGallery(ctx, newImage, imageFile); err != nil {
|
||||
if _, err := h.associateGallery(ctx, newImage, imageFile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("%s doesn't exist. Creating new image...", f.Base().Path)
|
||||
|
||||
if err := h.CreatorUpdater.Create(ctx, &models.ImageCreateInput{
|
||||
Image: newImage,
|
||||
|
|
@ -131,11 +150,22 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
existing = []*models.Image{newImage}
|
||||
}
|
||||
|
||||
// remove the old thumbnail if the checksum changed - we'll regenerate it
|
||||
if oldFile != nil {
|
||||
oldHash := oldFile.Base().Fingerprints.GetString(file.FingerprintTypeMD5)
|
||||
newHash := f.Base().Fingerprints.GetString(file.FingerprintTypeMD5)
|
||||
|
||||
if oldHash != "" && newHash != "" && oldHash != newHash {
|
||||
// remove cache dir of gallery
|
||||
_ = os.Remove(h.Paths.Generated.GetThumbnailPath(oldHash, models.DefaultGthumbWidth))
|
||||
}
|
||||
}
|
||||
|
||||
if h.ScanConfig.IsGenerateThumbnails() {
|
||||
for _, s := range existing {
|
||||
if err := h.ThumbnailGenerator.GenerateThumbnail(ctx, s, imageFile); err != nil {
|
||||
// just log if cover generation fails. We can try again on rescan
|
||||
logger.Errorf("Error generating thumbnail for %s: %v", imageFile.Path, err)
|
||||
h.logError(ctx, "Error generating thumbnail for %s: %v", imageFile.Path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -143,7 +173,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Image, f *file.ImageFile) error {
|
||||
func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Image, f *file.ImageFile, updateExisting bool) error {
|
||||
for _, i := range existing {
|
||||
if err := i.LoadFiles(ctx, h.CreatorUpdater); err != nil {
|
||||
return err
|
||||
|
|
@ -157,35 +187,49 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
|
|||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
logger.Infof("Adding %s to image %s", f.Path, i.DisplayName())
|
||||
|
||||
// associate with folder-based gallery if applicable
|
||||
if h.ScanConfig.GetCreateGalleriesFromFolders() {
|
||||
if err := h.associateFolderBasedGallery(ctx, i, f); err != nil {
|
||||
// associate with gallery if applicable
|
||||
changed, err := h.associateGallery(ctx, i, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var galleryIDs *models.UpdateIDs
|
||||
if changed {
|
||||
galleryIDs = &models.UpdateIDs{
|
||||
IDs: i.GalleryIDs.List(),
|
||||
Mode: models.RelationshipUpdateModeSet,
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
h.logInfo(ctx, "Adding %s to image %s", f.Path, i.DisplayName())
|
||||
|
||||
if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.ID); err != nil {
|
||||
return fmt.Errorf("adding file to image: %w", err)
|
||||
}
|
||||
// update updated_at time
|
||||
if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewImagePartial()); err != nil {
|
||||
|
||||
changed = true
|
||||
}
|
||||
|
||||
if changed {
|
||||
// always update updated_at time
|
||||
if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.ImagePartial{
|
||||
GalleryIDs: galleryIDs,
|
||||
UpdatedAt: models.NewOptionalTime(time.Now()),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("updating image: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if changed || updateExisting {
|
||||
h.PluginCache.RegisterPostHooks(ctx, i.ID, plugin.ImageUpdatePost, nil, nil)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) getOrCreateFolderBasedGallery(ctx context.Context, f file.File) (*models.Gallery, error) {
|
||||
// don't create folder-based galleries for files in zip file
|
||||
if f.Base().ZipFileID != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
folderID := f.Base().ParentFolderID
|
||||
g, err := h.GalleryFinder.FindByFolderID(ctx, folderID)
|
||||
if err != nil {
|
||||
|
|
@ -205,28 +249,100 @@ func (h *ScanHandler) getOrCreateFolderBasedGallery(ctx context.Context, f file.
|
|||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
logger.Infof("Creating folder-based gallery for %s", filepath.Dir(f.Base().Path))
|
||||
h.logInfo(ctx, "Creating folder-based gallery for %s", filepath.Dir(f.Base().Path))
|
||||
|
||||
if err := h.GalleryFinder.Create(ctx, newGallery, nil); err != nil {
|
||||
return nil, fmt.Errorf("creating folder based gallery: %w", err)
|
||||
}
|
||||
|
||||
// it's possible that there are other images in the folder that
|
||||
// need to be added to the new gallery. Find and add them now.
|
||||
if err := h.associateFolderImages(ctx, newGallery); err != nil {
|
||||
return nil, fmt.Errorf("associating existing folder images: %w", err)
|
||||
}
|
||||
|
||||
return newGallery, nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) associateFolderBasedGallery(ctx context.Context, newImage *models.Image, f file.File) error {
|
||||
g, err := h.getOrCreateFolderBasedGallery(ctx, f)
|
||||
func (h *ScanHandler) associateFolderImages(ctx context.Context, g *models.Gallery) error {
|
||||
i, err := h.CreatorUpdater.FindByFolderID(ctx, *g.FolderID)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("finding images in folder: %w", err)
|
||||
}
|
||||
|
||||
if err := newImage.LoadGalleryIDs(ctx, h.CreatorUpdater); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ii := range i {
|
||||
h.logInfo(ctx, "Adding %s to gallery %s", ii.Path, g.Path)
|
||||
|
||||
if g != nil && !intslice.IntInclude(newImage.GalleryIDs.List(), g.ID) {
|
||||
newImage.GalleryIDs.Add(g.ID)
|
||||
logger.Infof("Adding %s to folder-based gallery %s", f.Base().Path, g.Path)
|
||||
if _, err := h.CreatorUpdater.UpdatePartial(ctx, ii.ID, models.ImagePartial{
|
||||
GalleryIDs: &models.UpdateIDs{
|
||||
IDs: []int{g.ID},
|
||||
Mode: models.RelationshipUpdateModeAdd,
|
||||
},
|
||||
UpdatedAt: models.NewOptionalTime(time.Now()),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("updating image: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) getOrCreateZipBasedGallery(ctx context.Context, zipFile file.File) (*models.Gallery, error) {
|
||||
g, err := h.GalleryFinder.FindByFileID(ctx, zipFile.Base().ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("finding zip based gallery: %w", err)
|
||||
}
|
||||
|
||||
if len(g) > 0 {
|
||||
gg := g[0]
|
||||
return gg, nil
|
||||
}
|
||||
|
||||
// create a new zip-based gallery
|
||||
now := time.Now()
|
||||
newGallery := &models.Gallery{
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
h.logInfo(ctx, "%s doesn't exist. Creating new gallery...", zipFile.Base().Path)
|
||||
|
||||
if err := h.GalleryFinder.Create(ctx, newGallery, []file.ID{zipFile.Base().ID}); err != nil {
|
||||
return nil, fmt.Errorf("creating zip-based gallery: %w", err)
|
||||
}
|
||||
|
||||
return newGallery, nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) getOrCreateGallery(ctx context.Context, f file.File) (*models.Gallery, error) {
|
||||
// don't create folder-based galleries for files in zip file
|
||||
if f.Base().ZipFile != nil {
|
||||
return h.getOrCreateZipBasedGallery(ctx, f.Base().ZipFile)
|
||||
}
|
||||
|
||||
if h.ScanConfig.GetCreateGalleriesFromFolders() {
|
||||
return h.getOrCreateFolderBasedGallery(ctx, f)
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) associateGallery(ctx context.Context, newImage *models.Image, f file.File) (bool, error) {
|
||||
g, err := h.getOrCreateGallery(ctx, f)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err := newImage.LoadGalleryIDs(ctx, h.CreatorUpdater); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
ret := false
|
||||
if g != nil && !intslice.IntInclude(newImage.GalleryIDs.List(), g.ID) {
|
||||
ret = true
|
||||
newImage.GalleryIDs.Add(g.ID)
|
||||
h.logInfo(ctx, "Adding %s to gallery %s", f.Base().Path, g.Path)
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type HashAlgorithm string
|
||||
|
|
@ -48,28 +47,3 @@ func (e *HashAlgorithm) UnmarshalGQL(v interface{}) error {
|
|||
func (e HashAlgorithm) MarshalGQL(w io.Writer) {
|
||||
fmt.Fprint(w, strconv.Quote(e.String()))
|
||||
}
|
||||
|
||||
type File struct {
|
||||
Checksum string `db:"checksum" json:"checksum"`
|
||||
OSHash string `db:"oshash" json:"oshash"`
|
||||
Path string `db:"path" json:"path"`
|
||||
Size string `db:"size" json:"size"`
|
||||
FileModTime time.Time `db:"file_mod_time" json:"file_mod_time"`
|
||||
}
|
||||
|
||||
// GetHash returns the hash of the scene, based on the hash algorithm provided. If
|
||||
// hash algorithm is MD5, then Checksum is returned. Otherwise, OSHash is returned.
|
||||
func (s File) GetHash(hashAlgorithm HashAlgorithm) string {
|
||||
switch hashAlgorithm {
|
||||
case HashAlgorithmMd5:
|
||||
return s.Checksum
|
||||
case HashAlgorithmOshash:
|
||||
return s.OSHash
|
||||
default:
|
||||
panic("unknown hash algorithm")
|
||||
}
|
||||
}
|
||||
|
||||
func (s File) Equal(o File) bool {
|
||||
return s.Path == o.Path && s.Checksum == o.Checksum && s.OSHash == o.OSHash && s.Size == o.Size && s.FileModTime.Equal(o.FileModTime)
|
||||
}
|
||||
|
|
|
|||
19
pkg/scene/hash.go
Normal file
19
pkg/scene/hash.go
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
package scene
|
||||
|
||||
import (
|
||||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
)
|
||||
|
||||
// GetHash returns the hash of the file, based on the hash algorithm provided. If
|
||||
// hash algorithm is MD5, then Checksum is returned. Otherwise, OSHash is returned.
|
||||
func GetHash(f file.File, hashAlgorithm models.HashAlgorithm) string {
|
||||
switch hashAlgorithm {
|
||||
case models.HashAlgorithmMd5:
|
||||
return f.Base().Fingerprints.GetString(file.FingerprintTypeMD5)
|
||||
case models.HashAlgorithmOshash:
|
||||
return f.Base().Fingerprints.GetString(file.FingerprintTypeOshash)
|
||||
default:
|
||||
panic("unknown hash algorithm")
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/models/paths"
|
||||
"github.com/stashapp/stash/pkg/plugin"
|
||||
)
|
||||
|
||||
|
|
@ -35,6 +36,9 @@ type ScanHandler struct {
|
|||
CoverGenerator CoverGenerator
|
||||
ScanGenerator ScanGenerator
|
||||
PluginCache *plugin.Cache
|
||||
|
||||
FileNamingAlgorithm models.HashAlgorithm
|
||||
Paths *paths.Paths
|
||||
}
|
||||
|
||||
func (h *ScanHandler) validate() error {
|
||||
|
|
@ -47,11 +51,17 @@ func (h *ScanHandler) validate() error {
|
|||
if h.ScanGenerator == nil {
|
||||
return errors.New("ScanGenerator is required")
|
||||
}
|
||||
if !h.FileNamingAlgorithm.IsValid() {
|
||||
return errors.New("FileNamingAlgorithm is required")
|
||||
}
|
||||
if h.Paths == nil {
|
||||
return errors.New("Paths is required")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
||||
func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error {
|
||||
if err := h.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -76,7 +86,8 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
}
|
||||
|
||||
if len(existing) > 0 {
|
||||
if err := h.associateExisting(ctx, existing, videoFile); err != nil {
|
||||
updateExisting := oldFile != nil
|
||||
if err := h.associateExisting(ctx, existing, videoFile, updateExisting); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
|
|
@ -98,6 +109,16 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
existing = []*models.Scene{newScene}
|
||||
}
|
||||
|
||||
if oldFile != nil {
|
||||
// migrate hashes from the old file to the new
|
||||
oldHash := GetHash(oldFile, h.FileNamingAlgorithm)
|
||||
newHash := GetHash(f, h.FileNamingAlgorithm)
|
||||
|
||||
if oldHash != "" && newHash != "" && oldHash != newHash {
|
||||
MigrateHash(h.Paths, oldHash, newHash)
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range existing {
|
||||
if err := h.CoverGenerator.GenerateCover(ctx, s, videoFile); err != nil {
|
||||
// just log if cover generation fails. We can try again on rescan
|
||||
|
|
@ -113,7 +134,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Scene, f *file.VideoFile) error {
|
||||
func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Scene, f *file.VideoFile, updateExisting bool) error {
|
||||
for _, s := range existing {
|
||||
if err := s.LoadFiles(ctx, h.CreatorUpdater); err != nil {
|
||||
return err
|
||||
|
|
@ -139,6 +160,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.
|
|||
return fmt.Errorf("updating scene: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if !found || updateExisting {
|
||||
h.PluginCache.RegisterPostHooks(ctx, s.ID, plugin.SceneUpdatePost, nil, nil)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const (
|
|||
type hookManager struct {
|
||||
postCommitHooks []TxnFunc
|
||||
postRollbackHooks []TxnFunc
|
||||
postCompleteHooks []TxnFunc
|
||||
}
|
||||
|
||||
func (m *hookManager) register(ctx context.Context) context.Context {
|
||||
|
|
@ -27,20 +28,26 @@ func hookManagerCtx(ctx context.Context) *hookManager {
|
|||
return m
|
||||
}
|
||||
|
||||
func executePostCommitHooks(ctx context.Context) {
|
||||
m := hookManagerCtx(ctx)
|
||||
for _, h := range m.postCommitHooks {
|
||||
func executeHooks(ctx context.Context, hooks []TxnFunc) {
|
||||
for _, h := range hooks {
|
||||
// ignore errors
|
||||
_ = h(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func executePostCommitHooks(ctx context.Context) {
|
||||
m := hookManagerCtx(ctx)
|
||||
executeHooks(ctx, m.postCommitHooks)
|
||||
}
|
||||
|
||||
func executePostRollbackHooks(ctx context.Context) {
|
||||
m := hookManagerCtx(ctx)
|
||||
for _, h := range m.postRollbackHooks {
|
||||
// ignore errors
|
||||
_ = h(ctx)
|
||||
}
|
||||
executeHooks(ctx, m.postRollbackHooks)
|
||||
}
|
||||
|
||||
func executePostCompleteHooks(ctx context.Context) {
|
||||
m := hookManagerCtx(ctx)
|
||||
executeHooks(ctx, m.postCompleteHooks)
|
||||
}
|
||||
|
||||
func AddPostCommitHook(ctx context.Context, hook TxnFunc) {
|
||||
|
|
@ -52,3 +59,8 @@ func AddPostRollbackHook(ctx context.Context, hook TxnFunc) {
|
|||
m := hookManagerCtx(ctx)
|
||||
m.postRollbackHooks = append(m.postRollbackHooks, hook)
|
||||
}
|
||||
|
||||
func AddPostCompleteHook(ctx context.Context, hook TxnFunc) {
|
||||
m := hookManagerCtx(ctx)
|
||||
m.postCompleteHooks = append(m.postCompleteHooks, hook)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,11 @@ type TxnFunc func(ctx context.Context) error
|
|||
// WithTxn executes fn in a transaction. If fn returns an error then
|
||||
// the transaction is rolled back. Otherwise it is committed.
|
||||
func WithTxn(ctx context.Context, m Manager, fn TxnFunc) error {
|
||||
const execComplete = true
|
||||
return withTxn(ctx, m, fn, execComplete)
|
||||
}
|
||||
|
||||
func withTxn(ctx context.Context, m Manager, fn TxnFunc, execCompleteOnLocked bool) error {
|
||||
var err error
|
||||
ctx, err = begin(ctx, m)
|
||||
if err != nil {
|
||||
|
|
@ -38,10 +43,16 @@ func WithTxn(ctx context.Context, m Manager, fn TxnFunc) error {
|
|||
if err != nil {
|
||||
// something went wrong, rollback
|
||||
rollback(ctx, m)
|
||||
|
||||
if execCompleteOnLocked || !m.IsLocked(err) {
|
||||
executePostCompleteHooks(ctx)
|
||||
}
|
||||
} else {
|
||||
// all good, commit
|
||||
err = commit(ctx, m)
|
||||
executePostCompleteHooks(ctx)
|
||||
}
|
||||
|
||||
}()
|
||||
|
||||
err = fn(ctx)
|
||||
|
|
@ -102,7 +113,8 @@ func (r Retryer) WithTxn(ctx context.Context, fn TxnFunc) error {
|
|||
var attempt int
|
||||
var err error
|
||||
for attempt = 1; attempt <= r.Retries || r.Retries < 0; attempt++ {
|
||||
err = WithTxn(ctx, r.Manager, fn)
|
||||
const execComplete = false
|
||||
err = withTxn(ctx, r.Manager, fn, execComplete)
|
||||
|
||||
if err == nil {
|
||||
return nil
|
||||
|
|
|
|||
Loading…
Reference in a new issue