package manager import ( "context" "fmt" "path/filepath" "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scene" ) type cleanJob struct { txnManager models.TransactionManager input CleanMetadataInput scanSubs *subscriptionManager } func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) { logger.Infof("Starting cleaning of tracked files") if j.input.DryRun { logger.Infof("Running in Dry Mode") } if err := j.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { total, err := j.getCount(r) if err != nil { return fmt.Errorf("error getting count: %w", err) } progress.SetTotal(total) if job.IsCancelled(ctx) { return nil } if err := j.processScenes(ctx, progress, r.Scene()); err != nil { return fmt.Errorf("error cleaning scenes: %w", err) } if err := j.processImages(ctx, progress, r.Image()); err != nil { return fmt.Errorf("error cleaning images: %w", err) } if err := j.processGalleries(ctx, progress, r.Gallery(), r.Image()); err != nil { return fmt.Errorf("error cleaning galleries: %w", err) } return nil }); err != nil { logger.Error(err.Error()) return } if job.IsCancelled(ctx) { logger.Info("Stopping due to user request") return } j.scanSubs.notify() logger.Info("Finished Cleaning") } func (j *cleanJob) getCount(r models.ReaderRepository) (int, error) { sceneFilter := scene.PathsFilter(j.input.Paths) sceneResult, err := r.Scene().Query(models.SceneQueryOptions{ QueryOptions: models.QueryOptions{ Count: true, }, SceneFilter: sceneFilter, }) if err != nil { return 0, err } imageCount, err := r.Image().QueryCount(image.PathsFilter(j.input.Paths), nil) if err != nil { return 0, err } galleryCount, err := r.Gallery().QueryCount(gallery.PathsFilter(j.input.Paths), nil) if err != nil { return 0, err } return sceneResult.Count + imageCount + galleryCount, nil } func (j *cleanJob) processScenes(ctx context.Context, progress *job.Progress, qb models.SceneReader) error { batchSize := 1000 findFilter := models.BatchFindFilter(batchSize) sceneFilter := scene.PathsFilter(j.input.Paths) sort := "path" findFilter.Sort = &sort var toDelete []int more := true for more { if job.IsCancelled(ctx) { return nil } scenes, err := scene.Query(qb, sceneFilter, findFilter) if err != nil { return fmt.Errorf("error querying for scenes: %w", err) } for _, scene := range scenes { progress.ExecuteTask(fmt.Sprintf("Assessing scene %s for clean", scene.Path), func() { if j.shouldCleanScene(scene) { toDelete = append(toDelete, scene.ID) } else { // increment progress, no further processing progress.Increment() } }) } if len(scenes) != batchSize { more = false } else { *findFilter.Page++ } } if j.input.DryRun && len(toDelete) > 0 { // add progress for scenes that would've been deleted progress.AddProcessed(len(toDelete)) } fileNamingAlgorithm := instance.Config.GetVideoFileNamingAlgorithm() if !j.input.DryRun && len(toDelete) > 0 { progress.ExecuteTask(fmt.Sprintf("Cleaning %d scenes", len(toDelete)), func() { for _, sceneID := range toDelete { if job.IsCancelled(ctx) { return } j.deleteScene(ctx, fileNamingAlgorithm, sceneID) progress.Increment() } }) } return nil } func (j *cleanJob) processGalleries(ctx context.Context, progress *job.Progress, qb models.GalleryReader, iqb models.ImageReader) error { batchSize := 1000 findFilter := models.BatchFindFilter(batchSize) galleryFilter := gallery.PathsFilter(j.input.Paths) sort := "path" findFilter.Sort = &sort var toDelete []int more := true for more { if job.IsCancelled(ctx) { return nil } galleries, _, err := qb.Query(galleryFilter, findFilter) if err != nil { return fmt.Errorf("error querying for galleries: %w", err) } for _, gallery := range galleries { progress.ExecuteTask(fmt.Sprintf("Assessing gallery %s for clean", gallery.GetTitle()), func() { if j.shouldCleanGallery(gallery, iqb) { toDelete = append(toDelete, gallery.ID) } else { // increment progress, no further processing progress.Increment() } }) } if len(galleries) != batchSize { more = false } else { *findFilter.Page++ } } if j.input.DryRun && len(toDelete) > 0 { // add progress for galleries that would've been deleted progress.AddProcessed(len(toDelete)) } if !j.input.DryRun && len(toDelete) > 0 { progress.ExecuteTask(fmt.Sprintf("Cleaning %d galleries", len(toDelete)), func() { for _, galleryID := range toDelete { if job.IsCancelled(ctx) { return } j.deleteGallery(ctx, galleryID) progress.Increment() } }) } return nil } func (j *cleanJob) processImages(ctx context.Context, progress *job.Progress, qb models.ImageReader) error { batchSize := 1000 findFilter := models.BatchFindFilter(batchSize) imageFilter := image.PathsFilter(j.input.Paths) // performance consideration: order by path since default ordering by // title is slow sortBy := "path" findFilter.Sort = &sortBy var toDelete []int more := true for more { if job.IsCancelled(ctx) { return nil } images, err := image.Query(qb, imageFilter, findFilter) if err != nil { return fmt.Errorf("error querying for images: %w", err) } for _, image := range images { progress.ExecuteTask(fmt.Sprintf("Assessing image %s for clean", image.Path), func() { if j.shouldCleanImage(image) { toDelete = append(toDelete, image.ID) } else { // increment progress, no further processing progress.Increment() } }) } if len(images) != batchSize { more = false } else { *findFilter.Page++ } } if j.input.DryRun && len(toDelete) > 0 { // add progress for images that would've been deleted progress.AddProcessed(len(toDelete)) } if !j.input.DryRun && len(toDelete) > 0 { progress.ExecuteTask(fmt.Sprintf("Cleaning %d images", len(toDelete)), func() { for _, imageID := range toDelete { if job.IsCancelled(ctx) { return } j.deleteImage(ctx, imageID) progress.Increment() } }) } return nil } func (j *cleanJob) shouldClean(path string) bool { // use image.FileExists for zip file checking fileExists := image.FileExists(path) // #1102 - clean anything in generated path generatedPath := config.GetInstance().GetGeneratedPath() if !fileExists || getStashFromPath(path) == nil || fsutil.IsPathInDir(generatedPath, path) { logger.Infof("File not found. Marking to clean: \"%s\"", path) return true } return false } func (j *cleanJob) shouldCleanScene(s *models.Scene) bool { if j.shouldClean(s.Path) { return true } stash := getStashFromPath(s.Path) if stash.ExcludeVideo { logger.Infof("File in stash library that excludes video. Marking to clean: \"%s\"", s.Path) return true } config := config.GetInstance() if !fsutil.MatchExtension(s.Path, config.GetVideoExtensions()) { logger.Infof("File extension does not match video extensions. Marking to clean: \"%s\"", s.Path) return true } if matchFile(s.Path, config.GetExcludes()) { logger.Infof("File matched regex. Marking to clean: \"%s\"", s.Path) return true } return false } func (j *cleanJob) shouldCleanGallery(g *models.Gallery, qb models.ImageReader) bool { // never clean manually created galleries if !g.Path.Valid { return false } path := g.Path.String if j.shouldClean(path) { return true } stash := getStashFromPath(path) if stash.ExcludeImage { logger.Infof("File in stash library that excludes images. Marking to clean: \"%s\"", path) return true } config := config.GetInstance() if g.Zip { if !fsutil.MatchExtension(path, config.GetGalleryExtensions()) { logger.Infof("File extension does not match gallery extensions. Marking to clean: \"%s\"", path) return true } if countImagesInZip(path) == 0 { logger.Infof("Gallery has 0 images. Marking to clean: \"%s\"", path) return true } } else { // folder-based - delete if it has no images count, err := qb.CountByGalleryID(g.ID) if err != nil { logger.Warnf("Error trying to count gallery images for %q: %v", path, err) return false } if count == 0 { return true } } if matchFile(path, config.GetImageExcludes()) { logger.Infof("File matched regex. Marking to clean: \"%s\"", path) return true } return false } func (j *cleanJob) shouldCleanImage(s *models.Image) bool { if j.shouldClean(s.Path) { return true } stash := getStashFromPath(s.Path) if stash.ExcludeImage { logger.Infof("File in stash library that excludes images. Marking to clean: \"%s\"", s.Path) return true } config := config.GetInstance() if !fsutil.MatchExtension(s.Path, config.GetImageExtensions()) { logger.Infof("File extension does not match image extensions. Marking to clean: \"%s\"", s.Path) return true } if matchFile(s.Path, config.GetImageExcludes()) { logger.Infof("File matched regex. Marking to clean: \"%s\"", s.Path) return true } return false } func (j *cleanJob) deleteScene(ctx context.Context, fileNamingAlgorithm models.HashAlgorithm, sceneID int) { fileNamingAlgo := GetInstance().Config.GetVideoFileNamingAlgorithm() fileDeleter := &scene.FileDeleter{ Deleter: *file.NewDeleter(), FileNamingAlgo: fileNamingAlgo, Paths: GetInstance().Paths, } var s *models.Scene if err := j.txnManager.WithTxn(ctx, func(repo models.Repository) error { qb := repo.Scene() var err error s, err = qb.Find(sceneID) if err != nil { return err } return scene.Destroy(s, repo, fileDeleter, true, false) }); err != nil { fileDeleter.Rollback() logger.Errorf("Error deleting scene from database: %s", err.Error()) return } // perform the post-commit actions fileDeleter.Commit() GetInstance().PluginCache.ExecutePostHooks(ctx, sceneID, plugin.SceneDestroyPost, plugin.SceneDestroyInput{ Checksum: s.Checksum.String, OSHash: s.OSHash.String, Path: s.Path, }, nil) } func (j *cleanJob) deleteGallery(ctx context.Context, galleryID int) { var g *models.Gallery if err := j.txnManager.WithTxn(ctx, func(repo models.Repository) error { qb := repo.Gallery() var err error g, err = qb.Find(galleryID) if err != nil { return err } return qb.Destroy(galleryID) }); err != nil { logger.Errorf("Error deleting gallery from database: %s", err.Error()) return } GetInstance().PluginCache.ExecutePostHooks(ctx, galleryID, plugin.GalleryDestroyPost, plugin.GalleryDestroyInput{ Checksum: g.Checksum, Path: g.Path.String, }, nil) } func (j *cleanJob) deleteImage(ctx context.Context, imageID int) { fileDeleter := &image.FileDeleter{ Deleter: *file.NewDeleter(), Paths: GetInstance().Paths, } var i *models.Image if err := j.txnManager.WithTxn(ctx, func(repo models.Repository) error { qb := repo.Image() var err error i, err = qb.Find(imageID) if err != nil { return err } if i == nil { return fmt.Errorf("image not found: %d", imageID) } return image.Destroy(i, qb, fileDeleter, true, false) }); err != nil { fileDeleter.Rollback() logger.Errorf("Error deleting image from database: %s", err.Error()) return } // perform the post-commit actions fileDeleter.Commit() GetInstance().PluginCache.ExecutePostHooks(ctx, imageID, plugin.ImageDestroyPost, plugin.ImageDestroyInput{ Checksum: i.Checksum, Path: i.Path, }, nil) } func getStashFromPath(pathToCheck string) *config.StashConfig { for _, s := range config.GetInstance().GetStashPaths() { if fsutil.IsPathInDir(s.Path, filepath.Dir(pathToCheck)) { return s } } return nil } func getStashFromDirPath(pathToCheck string) *config.StashConfig { for _, s := range config.GetInstance().GetStashPaths() { if fsutil.IsPathInDir(s.Path, pathToCheck) { return s } } return nil }