Add Clean generated files task (#4607)

* Add clean generate task
* Add to library tasks
* Save and read defaults
* Stop handling and logging
* Make filename parsing more robust

gqlgen.yml

@@ -70,6 +70,8 @@ models:
     model: github.com/stashapp/stash/internal/manager/config.ConfigDisableDropdownCreate
   ScanMetadataOptions:
     model:  github.com/stashapp/stash/internal/manager/config.ScanMetadataOptions
+  CleanGeneratedInput:
+    model: github.com/stashapp/stash/internal/manager/task.CleanGeneratedOptions
   AutoTagMetadataOptions:
     model: github.com/stashapp/stash/internal/manager/config.AutoTagMetadataOptions
   SystemStatus:

graphql/schema/schema.graphql

@@ -378,6 +378,8 @@ type Mutation {
   metadataAutoTag(input: AutoTagMetadataInput!): ID!
   "Clean metadata. Returns the job ID"
   metadataClean(input: CleanMetadataInput!): ID!
+  "Clean generated files. Returns the job ID"
+  metadataCleanGenerated(input: CleanGeneratedInput!): ID!
   "Identifies scenes using scrapers. Returns the job ID"
   metadataIdentify(input: IdentifyMetadataInput!): ID!
 

graphql/schema/types/metadata.graphql

@@ -118,6 +118,26 @@ input CleanMetadataInput {
   dryRun: Boolean!
 }
 
+input CleanGeneratedInput {
+  "Clean blob files without blob entries"
+  blobFiles: Boolean
+  "Clean sprite and vtt files without scene entries"
+  sprites: Boolean
+  "Clean preview files without scene entries"
+  screenshots: Boolean
+  "Clean scene transcodes without scene entries"
+  transcodes: Boolean
+
+  "Clean marker files without marker entries"
+  markers: Boolean
+
+  "Clean image thumbnails/clips without image entries"
+  imageThumbnails: Boolean
+
+  "Do a dry run. Don't delete any files"
+  dryRun: Boolean
+}
+
 input AutoTagMetadataInput {
   "Paths to tag, null for all files"
   paths: [String!]

internal/api/resolver_mutation_metadata.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stashapp/stash/internal/identify"
 	"github.com/stashapp/stash/internal/manager"
 	"github.com/stashapp/stash/internal/manager/config"
+	"github.com/stashapp/stash/internal/manager/task"
 	"github.com/stashapp/stash/pkg/logger"
 )
 
@@ -98,6 +99,21 @@ func (r *mutationResolver) MetadataClean(ctx context.Context, input manager.Clea
 	return strconv.Itoa(jobID), nil
 }
 
+func (r *mutationResolver) MetadataCleanGenerated(ctx context.Context, input task.CleanGeneratedOptions) (string, error) {
+	mgr := manager.GetInstance()
+	t := &task.CleanGeneratedJob{
+		Options:                  input,
+		Paths:                    mgr.Paths,
+		BlobsStorageType:         mgr.Config.GetBlobsStorage(),
+		VideoFileNamingAlgorithm: mgr.Config.GetVideoFileNamingAlgorithm(),
+		Repository:               mgr.Repository,
+		BlobCleaner:              mgr.Repository.Blob,
+	}
+	jobID := mgr.JobManager.Add(ctx, "Cleaning generated files...", t)
+
+	return strconv.Itoa(jobID), nil
+}
+
 func (r *mutationResolver) MigrateHashNaming(ctx context.Context) (string, error) {
 	jobID := manager.GetInstance().MigrateHash(ctx)
 	return strconv.Itoa(jobID), nil

internal/manager/task/clean_generated.go

@@ -0,0 +1,727 @@
+package task
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/stashapp/stash/internal/manager/config"
+	"github.com/stashapp/stash/pkg/job"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/models/paths"
+)
+
+type CleanGeneratedOptions struct {
+	BlobFiles bool `json:"blobs"`
+
+	Sprites     bool `json:"sprites"`
+	Screenshots bool `json:"screenshots"`
+	Transcodes  bool `json:"transcodes"`
+
+	Markers bool `json:"markers"`
+
+	ImageThumbnails bool `json:"imageThumbnails"`
+
+	DryRun bool `json:"dryRun"`
+}
+
+type BlobCleaner interface {
+	EntryExists(ctx context.Context, checksum string) (bool, error)
+}
+
+type CleanGeneratedJob struct {
+	Options CleanGeneratedOptions
+
+	Paths                    *paths.Paths
+	BlobsStorageType         config.BlobsStorageType
+	VideoFileNamingAlgorithm models.HashAlgorithm
+
+	BlobCleaner BlobCleaner
+	Repository  models.Repository
+
+	dryRunPrefix  string
+	totalTasks    int
+	tasksComplete int
+}
+
+func (j *CleanGeneratedJob) deleteFile(path string) {
+	if j.Options.DryRun {
+		logger.Debugf("would delete file: %s", path)
+		return
+	}
+
+	if err := os.Remove(path); err != nil {
+		logger.Errorf("error deleting file %s: %v", path, err)
+	}
+}
+
+func (j *CleanGeneratedJob) deleteDir(path string) {
+	if j.Options.DryRun {
+		logger.Debugf("would delete file: %s", path)
+		return
+	}
+
+	if err := os.RemoveAll(path); err != nil {
+		logger.Errorf("error deleting directory %s: %v", path, err)
+	}
+}
+
+func (j *CleanGeneratedJob) countTasks() int {
+	tasks := 0
+
+	if j.Options.BlobFiles {
+		tasks++
+	}
+	if j.Options.Sprites {
+		tasks++
+	}
+	if j.Options.Screenshots {
+		tasks++
+	}
+	if j.Options.Transcodes {
+		tasks++
+	}
+	if j.Options.Markers {
+		tasks++
+	}
+	if j.Options.ImageThumbnails {
+		tasks++
+	}
+	return tasks
+}
+
+func (j *CleanGeneratedJob) taskComplete(progress *job.Progress) {
+	j.tasksComplete++
+	progress.SetPercent(float64(j.tasksComplete) / float64(j.totalTasks))
+}
+
+func (j *CleanGeneratedJob) logError(err error) {
+	if !errors.Is(err, context.Canceled) {
+		logger.Error(err)
+	}
+}
+
+func (j *CleanGeneratedJob) Execute(ctx context.Context, progress *job.Progress) {
+	j.tasksComplete = 0
+
+	if !j.BlobsStorageType.IsValid() {
+		logger.Errorf("invalid blobs storage type: %s", j.BlobsStorageType)
+		return
+	}
+
+	if !j.VideoFileNamingAlgorithm.IsValid() {
+		logger.Errorf("invalid video file naming algorithm: %s", j.VideoFileNamingAlgorithm)
+		return
+	}
+
+	if j.Options.DryRun {
+		j.dryRunPrefix = "[dry run] "
+	}
+
+	logger.Infof("Cleaning generated files %s", j.dryRunPrefix)
+
+	j.totalTasks = j.countTasks()
+
+	if j.Options.BlobFiles {
+		progress.ExecuteTask("Cleaning blob files", func() {
+			if err := j.cleanBlobFiles(ctx, progress); err != nil {
+				j.logError(fmt.Errorf("error cleaning blob files: %w", err))
+			}
+		})
+		j.taskComplete(progress)
+	}
+
+	if j.Options.Sprites {
+		progress.ExecuteTask("Cleaning sprite files", func() {
+			if err := j.cleanSpriteFiles(ctx, progress); err != nil {
+				j.logError(fmt.Errorf("error cleaning sprite files: %w", err))
+			}
+		})
+		j.taskComplete(progress)
+	}
+
+	if j.Options.Screenshots {
+		progress.ExecuteTask("Cleaning screenshot files", func() {
+			if err := j.cleanScreenshotFiles(ctx, progress); err != nil {
+				j.logError(fmt.Errorf("error cleaning screenshot files: %w", err))
+			}
+		})
+		j.taskComplete(progress)
+	}
+
+	if j.Options.Transcodes {
+		progress.ExecuteTask("Cleaning transcode files", func() {
+			if err := j.cleanTranscodeFiles(ctx, progress); err != nil {
+				j.logError(fmt.Errorf("error cleaning transcode files: %w", err))
+			}
+		})
+		j.taskComplete(progress)
+	}
+
+	if j.Options.Markers {
+		progress.ExecuteTask("Cleaning marker files", func() {
+			if err := j.cleanMarkerFiles(ctx, progress); err != nil {
+				j.logError(fmt.Errorf("error cleaning marker files: %w", err))
+			}
+		})
+		j.taskComplete(progress)
+	}
+
+	if j.Options.ImageThumbnails {
+		progress.ExecuteTask("Cleaning thumbnail files", func() {
+			if err := j.cleanThumbnailFiles(ctx, progress); err != nil {
+				j.logError(fmt.Errorf("error cleaning thumbnail files: %w", err))
+			}
+		})
+		j.taskComplete(progress)
+	}
+
+	if job.IsCancelled(ctx) {
+		logger.Info("Stopping due to user request")
+		return
+	}
+
+	logger.Infof("Finished cleaning generated files")
+}
+
+func (j *CleanGeneratedJob) setTaskProgress(taskProgress float64, progress *job.Progress) {
+	progress.SetPercent((float64(j.tasksComplete) + taskProgress) / float64(j.totalTasks))
+}
+
+func (j *CleanGeneratedJob) logDelete(format string, args ...interface{}) {
+	logger.Infof(j.dryRunPrefix+format, args...)
+}
+
+// estimates the progress by the hash prefix - first two characters
+// this is a rough estimate, but it's better than nothing
+// the prefix ranges from 00 to ff
+func (j *CleanGeneratedJob) estimateProgress(hashPrefix string) (float64, error) {
+	toInt, err := strconv.ParseInt(hashPrefix, 16, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	const total = 256 // ff
+	return float64(toInt) / total, nil
+}
+
+func (j *CleanGeneratedJob) setProgressFromFilename(prefix string, progress *job.Progress) {
+	p, err := j.estimateProgress(prefix)
+	if err != nil {
+		logger.Errorf("error estimating progress: %v", err)
+		return
+	}
+	j.setTaskProgress(p, progress)
+}
+
+func (j *CleanGeneratedJob) getIntraFolderPrefix(basename string) (string, error) {
+	var hash string
+	_, err := fmt.Sscanf(basename, "%2x", &hash)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", hash), nil
+}
+
+func (j *CleanGeneratedJob) getBlobFileHash(basename string) (string, error) {
+	var hash string
+	_, err := fmt.Sscanf(basename, "%32x", &hash)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", hash), nil
+}
+
+func (j *CleanGeneratedJob) cleanBlobFiles(ctx context.Context, progress *job.Progress) error {
+	if job.IsCancelled(ctx) {
+		return nil
+	}
+
+	if j.BlobsStorageType != config.BlobStorageTypeFilesystem {
+		logger.Debugf("skipping blob file cleanup, storage type is not filesystem")
+		return nil
+	}
+
+	logger.Infof("Cleaning blob files")
+
+	// walk through the blob directory
+	if err := filepath.Walk(j.Paths.Blobs, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if err = ctx.Err(); err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			if path == j.Paths.Blobs {
+				return nil
+			}
+
+			// ignore any directory that isn't a two character hash prefix
+			_, err := j.getIntraFolderPrefix(info.Name())
+			if err != nil {
+				logger.Warnf("Ignoring unknown directory: %s", path)
+				return fs.SkipDir
+			}
+
+			// estimate progress by the hash prefix
+			if filepath.Dir(path) == j.Paths.Blobs {
+				hashPrefix := filepath.Base(path)
+				j.setProgressFromFilename(hashPrefix, progress)
+			}
+
+			return nil
+		}
+
+		blobname := info.Name()
+
+		// ignore any files that aren't a 32 character hash
+		_, err = j.getBlobFileHash(blobname)
+		if err != nil {
+			logger.Warnf("ignoring unknown blob file: %s", blobname)
+			return nil
+		}
+
+		// if blob entry does not exist, delete the file
+		if err := j.Repository.WithReadTxn(ctx, func(ctx context.Context) error {
+			exists, err := j.BlobCleaner.EntryExists(ctx, blobname)
+			if err != nil {
+				return err
+			}
+
+			if !exists {
+				j.logDelete("deleting unused blob file: %s", blobname)
+				j.deleteFile(path)
+			}
+
+			return nil
+		}); err != nil {
+			logger.Errorf("error checking blob entry: %v", err)
+			return nil
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (j *CleanGeneratedJob) getScenesWithHash(ctx context.Context, hash string) ([]*models.Scene, error) {
+	fp := models.Fingerprint{
+		Fingerprint: hash,
+	}
+
+	if j.VideoFileNamingAlgorithm == models.HashAlgorithmMd5 {
+		fp.Type = models.FingerprintTypeMD5
+	} else {
+		fp.Type = models.FingerprintTypeOshash
+	}
+
+	return j.Repository.Scene.FindByFingerprints(ctx, []models.Fingerprint{fp})
+}
+
+const (
+	md5Length    = 32
+	oshashLength = 16
+)
+
+func (j *CleanGeneratedJob) hashPatternPrefix() string {
+	hashLen := oshashLength
+	if j.VideoFileNamingAlgorithm == models.HashAlgorithmMd5 {
+		hashLen = md5Length
+	}
+
+	return fmt.Sprintf("%%%dx", hashLen)
+}
+
+func (j *CleanGeneratedJob) getSpriteFileHash(basename string) (string, error) {
+	patternPrefix := j.hashPatternPrefix()
+	spritePattern := patternPrefix + "_sprite.jpg"
+
+	var hash string
+	_, err := fmt.Sscanf(basename, spritePattern, &hash)
+	if err != nil {
+		// also try thumbs
+		thumbPattern := patternPrefix + "_thumbs.vtt"
+		_, err = fmt.Sscanf(basename, thumbPattern, &hash)
+
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return fmt.Sprintf("%x", hash), nil
+}
+
+func (j *CleanGeneratedJob) cleanSpriteFiles(ctx context.Context, progress *job.Progress) error {
+	if job.IsCancelled(ctx) {
+		return nil
+	}
+
+	logger.Infof("Cleaning sprite files")
+
+	// walk through the sprite directory
+	if err := filepath.Walk(j.Paths.Generated.Vtt, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if err = ctx.Err(); err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		filename := info.Name()
+
+		hash, err := j.getSpriteFileHash(filename)
+		if err != nil {
+			logger.Warnf("Ignoring unknown sprite file: %s", filename)
+			return nil
+		}
+
+		j.setProgressFromFilename(hash[0:2], progress)
+
+		var exists []*models.Scene
+
+		if err := j.Repository.WithReadTxn(ctx, func(ctx context.Context) error {
+			exists, err = j.getScenesWithHash(ctx, hash)
+			return err
+		}); err != nil {
+			logger.Errorf("error checking scene entry for sprite: %v", err)
+			return nil
+		}
+
+		if len(exists) == 0 {
+			j.logDelete("deleting unused sprite file: %s", filename)
+			j.deleteFile(path)
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (j *CleanGeneratedJob) cleanSceneFiles(ctx context.Context, path string, typ string, getSceneFileHash func(filename string) (string, error), progress *job.Progress) error {
+	if job.IsCancelled(ctx) {
+		return nil
+	}
+
+	logger.Infof("Cleaning %s files", typ)
+
+	// walk through the sprite directory
+	if err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		if err = ctx.Err(); err != nil {
+			return err
+		}
+
+		filename := info.Name()
+		hash, err := getSceneFileHash(filename)
+		if err != nil {
+			logger.Warnf("Ignoring unknown %s file: %s", typ, filename)
+			return nil
+		}
+
+		j.setProgressFromFilename(hash[0:2], progress)
+
+		var exists []*models.Scene
+
+		if err := j.Repository.WithReadTxn(ctx, func(ctx context.Context) error {
+			exists, err = j.getScenesWithHash(ctx, hash)
+			return err
+		}); err != nil {
+			logger.Errorf("error checking scene entry: %v", err)
+			return nil
+		}
+
+		if len(exists) == 0 {
+			j.logDelete("deleting unused %s file: %s", typ, filename)
+			j.deleteFile(path)
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (j *CleanGeneratedJob) getScreenshotFileHash(basename string) (string, error) {
+	var hash string
+	var ext string
+	// include the extension - which could be mp4/jpg/webp
+	_, err := fmt.Sscanf(basename, j.hashPatternPrefix()+".%s", &hash, &ext)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", hash), nil
+}
+
+func (j *CleanGeneratedJob) cleanScreenshotFiles(ctx context.Context, progress *job.Progress) error {
+	return j.cleanSceneFiles(ctx, j.Paths.Generated.Screenshots, "screenshot", j.getScreenshotFileHash, progress)
+}
+
+func (j *CleanGeneratedJob) getTranscodeFileHash(basename string) (string, error) {
+	var hash string
+	_, err := fmt.Sscanf(basename, j.hashPatternPrefix()+".mp4", &hash)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", hash), nil
+}
+
+func (j *CleanGeneratedJob) cleanTranscodeFiles(ctx context.Context, progress *job.Progress) error {
+	return j.cleanSceneFiles(ctx, j.Paths.Generated.Transcodes, "transcode", j.getTranscodeFileHash, progress)
+}
+
+func (j *CleanGeneratedJob) getMarkerSceneFileHash(basename string) (string, error) {
+	var hash string
+	_, err := fmt.Sscanf(basename, j.hashPatternPrefix(), &hash)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", hash), nil
+}
+
+func (j *CleanGeneratedJob) getMarkerFileSeconds(basename string) (int, error) {
+	var ret int
+	var ext string
+	// include the extension - which could be mp4/jpg/webp
+	_, err := fmt.Sscanf(basename, "%d.%s", &ret, &ext)
+	if err != nil {
+		return 0, err
+	}
+
+	return ret, nil
+}
+
+func (j *CleanGeneratedJob) cleanMarkerFiles(ctx context.Context, progress *job.Progress) error {
+	if job.IsCancelled(ctx) {
+		return nil
+	}
+
+	logger.Infof("Cleaning marker files")
+
+	var scenes []*models.Scene
+	var sceneHash string
+	var markers []*models.SceneMarker
+
+	// walk through the markers directory
+	if err := filepath.Walk(j.Paths.Generated.Markers, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if err = ctx.Err(); err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			// ignore markers directory
+			if path == j.Paths.Generated.Markers {
+				return nil
+			}
+
+			markers = nil
+
+			if filepath.Dir(path) != j.Paths.Generated.Markers {
+				logger.Warnf("Ignoring unknown marker directory: %s", path)
+				return nil
+			}
+
+			sceneHash, err = j.getMarkerSceneFileHash(info.Name())
+			if err != nil {
+				logger.Warnf("Ignoring unknown marker directory: %s", path)
+				return nil
+			}
+
+			j.setProgressFromFilename(sceneHash[0:2], progress)
+
+			// check if the scene exists
+			if err := j.Repository.WithReadTxn(ctx, func(ctx context.Context) error {
+				var err error
+				scenes, err = j.getScenesWithHash(ctx, sceneHash)
+				if err != nil {
+					return fmt.Errorf("error checking scene entry: %v", err)
+				}
+
+				if len(scenes) == 0 {
+					j.logDelete("deleting unused marker directory: %s", sceneHash)
+					j.deleteDir(path)
+				} else {
+					// get the markers now
+					for _, scene := range scenes {
+						thisMarkers, err := j.Repository.SceneMarker.FindBySceneID(ctx, scene.ID)
+						if err != nil {
+							return fmt.Errorf("error getting markers for scene: %v", err)
+						}
+						markers = append(markers, thisMarkers...)
+					}
+				}
+
+				return nil
+			}); err != nil {
+				logger.Error(err.Error())
+			}
+
+			return nil
+		}
+
+		filename := info.Name()
+		seconds, err := j.getMarkerFileSeconds(filename)
+		if err != nil {
+			logger.Warnf("Ignoring unknown marker file: %s", filename)
+			return nil
+		}
+
+		// scenes should be set by the directory walk
+		hash := filepath.Base(filepath.Dir(path))
+		if hash != sceneHash {
+			logger.Errorf("internal error: scene hash mismatch: %s != %s", hash, sceneHash)
+			return nil
+		}
+
+		if len(scenes) == 0 {
+			logger.Errorf("no scenes found for marker file: %s", filename)
+			return nil
+		}
+
+		// find the marker
+		var marker *models.SceneMarker
+		for _, m := range markers {
+			if int(m.Seconds) == seconds {
+				marker = m
+				break
+			}
+		}
+
+		if marker == nil {
+			// not found, delete the file
+			j.logDelete("deleting unused marker file: %s", filename)
+			j.deleteFile(path)
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (j *CleanGeneratedJob) getImagesWithHash(ctx context.Context, checksum string) ([]*models.Image, error) {
+	var exists []*models.Image
+	if err := j.Repository.WithReadTxn(ctx, func(ctx context.Context) error {
+		// if scene entry does not exist, delete the file
+		var err error
+		exists, err = j.Repository.Image.FindByChecksum(ctx, checksum)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+
+	return exists, nil
+}
+
+func (j *CleanGeneratedJob) getThumbnailFileHash(basename string) (string, error) {
+	var hash string
+	var width int
+	_, err := fmt.Sscanf(basename, "%32x_%d.jpg", &hash, &width)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", hash), nil
+}
+
+func (j *CleanGeneratedJob) cleanThumbnailFiles(ctx context.Context, progress *job.Progress) error {
+	if job.IsCancelled(ctx) {
+		return nil
+	}
+
+	logger.Infof("Cleaning image thumbnail files")
+
+	// walk through the sprite directory
+	if err := filepath.Walk(j.Paths.Generated.Thumbnails, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if err = ctx.Err(); err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			if path == j.Paths.Generated.Thumbnails {
+				return nil
+			}
+
+			// ensure the directory is a hash prefix
+			_, err := j.getIntraFolderPrefix(info.Name())
+			if err != nil {
+				logger.Warnf("Ignoring unknown thumbnail directory: %s", path)
+				return nil
+			}
+
+			// estimate progress by the hash prefix
+			if filepath.Dir(path) == j.Paths.Generated.Thumbnails {
+				hashPrefix := filepath.Base(path)
+				j.setProgressFromFilename(hashPrefix, progress)
+			}
+
+			return nil
+		}
+
+		filename := info.Name()
+		checksum, err := j.getThumbnailFileHash(filename)
+		if err != nil {
+			logger.Warnf("Ignoring unknown thumbnail file: %s", filename)
+			return nil
+		}
+
+		exists, err := j.getImagesWithHash(ctx, checksum)
+		if err != nil {
+			logger.Errorf("error checking image entry: %v", err)
+			return nil
+		}
+
+		if len(exists) == 0 {
+			j.logDelete("deleting unused thumbnail file: %s", filename)
+			j.deleteFile(path)
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/models/repository.go

@@ -14,6 +14,7 @@ type TxnManager interface {
 type Repository struct {
 	TxnManager TxnManager
 
+	Blob           BlobReader
 	File           FileReaderWriter
 	Folder         FolderReaderWriter
 	Gallery        GalleryReaderWriter

pkg/models/repository_blob.go

@@ -0,0 +1,8 @@
+package models
+
+import "context"
+
+// FileGetter provides methods to get files by ID.
+type BlobReader interface {
+	EntryExists(ctx context.Context, checksum string) (bool, error)
+}

pkg/sqlite/blob.go

@@ -235,6 +235,17 @@ func (qb *BlobStore) readFromFilesystem(ctx context.Context, checksum string) ([
 	}
 }
 
+func (qb *BlobStore) EntryExists(ctx context.Context, checksum string) (bool, error) {
+	q := dialect.From(qb.table()).Select(goqu.COUNT("*")).Where(qb.tableMgr.byID(checksum))
+
+	var found int
+	if err := querySimple(ctx, q, &found); err != nil {
+		return false, fmt.Errorf("querying %s: %w", qb.table(), err)
+	}
+
+	return found != 0, nil
+}
+
 // Read reads the data from the database or filesystem, depending on which is enabled.
 func (qb *BlobStore) Read(ctx context.Context, checksum string) ([]byte, error) {
 	if !qb.options.UseDatabase && !qb.options.UseFilesystem {

pkg/sqlite/transaction.go

@@ -126,6 +126,7 @@ func (db *Database) IsLocked(err error) bool {
 func (db *Database) Repository() models.Repository {
 	return models.Repository{
 		TxnManager:     db,
+		Blob:           db.Blobs,
 		File:           db.File,
 		Folder:         db.Folder,
 		Gallery:        db.Gallery,

ui/v2.5/graphql/mutations/metadata.graphql

@@ -34,6 +34,10 @@ mutation MetadataClean($input: CleanMetadataInput!) {
   metadataClean(input: $input)
 }
 
+mutation MetadataCleanGenerated($input: CleanGeneratedInput!) {
+  metadataCleanGenerated(input: $input)
+}
+
 mutation MigrateHashNaming {
   migrateHashNaming
 }

ui/v2.5/src/components/Settings/Tasks/CleanGeneratedDialog.tsx

@@ -0,0 +1,132 @@
+import React, { useEffect, useState } from "react";
+import { FormattedMessage, useIntl } from "react-intl";
+import { ModalComponent } from "src/components/Shared/Modal";
+import * as GQL from "src/core/generated-graphql";
+import { BooleanSetting } from "../Inputs";
+import { faTrashAlt } from "@fortawesome/free-solid-svg-icons";
+import { SettingSection } from "../SettingSection";
+import { useSettings } from "../context";
+import { LoadingIndicator } from "src/components/Shared/LoadingIndicator";
+
+const CleanGeneratedOptions: React.FC<{
+  options: GQL.CleanGeneratedInput;
+  setOptions: (s: GQL.CleanGeneratedInput) => void;
+}> = ({ options, setOptions: setOptionsState }) => {
+  function setOptions(input: Partial<GQL.CleanGeneratedInput>) {
+    setOptionsState({ ...options, ...input });
+  }
+
+  return (
+    <>
+      <BooleanSetting
+        id="clean-generated-blob-files"
+        checked={options.blobFiles ?? false}
+        headingID="config.tasks.clean_generated.blob_files"
+        onChange={(v) => setOptions({ blobFiles: v })}
+      />
+      <BooleanSetting
+        id="clean-generated-screenshots"
+        checked={options.screenshots ?? false}
+        headingID="config.tasks.clean_generated.previews"
+        subHeadingID="config.tasks.clean_generated.previews_desc"
+        onChange={(v) => setOptions({ screenshots: v })}
+      />
+      <BooleanSetting
+        id="clean-generated-sprites"
+        checked={options.sprites ?? false}
+        headingID="config.tasks.clean_generated.sprites"
+        onChange={(v) => setOptions({ sprites: v })}
+      />
+      <BooleanSetting
+        id="clean-generated-transcodes"
+        checked={options.transcodes ?? false}
+        headingID="config.tasks.clean_generated.transcodes"
+        onChange={(v) => setOptions({ transcodes: v })}
+      />
+      <BooleanSetting
+        id="clean-generated-markers"
+        checked={options.markers ?? false}
+        headingID="config.tasks.clean_generated.markers"
+        onChange={(v) => setOptions({ markers: v })}
+      />
+      <BooleanSetting
+        id="clean-generated-image-thumbnails"
+        checked={options.imageThumbnails ?? false}
+        headingID="config.tasks.clean_generated.image_thumbnails"
+        subHeadingID="config.tasks.clean_generated.image_thumbnails_desc"
+        onChange={(v) => setOptions({ imageThumbnails: v })}
+      />
+      <BooleanSetting
+        id="clean-generated-dryrun"
+        checked={options.dryRun ?? false}
+        headingID="config.tasks.only_dry_run"
+        onChange={(v) => setOptions({ dryRun: v })}
+      />
+    </>
+  );
+};
+
+export const CleanGeneratedDialog: React.FC<{
+  onClose: (input?: GQL.CleanGeneratedInput) => void;
+}> = ({ onClose }) => {
+  const intl = useIntl();
+
+  const { ui, saveUI, loading } = useSettings();
+
+  const [options, setOptions] = useState<GQL.CleanGeneratedInput>({
+    blobFiles: true,
+    imageThumbnails: true,
+    markers: true,
+    screenshots: true,
+    sprites: true,
+    transcodes: true,
+    dryRun: false,
+  });
+
+  useEffect(() => {
+    const defaults = ui.taskDefaults?.cleanGenerated;
+    if (defaults) {
+      setOptions(defaults);
+    }
+  }, [ui?.taskDefaults?.cleanGenerated]);
+
+  function confirm() {
+    saveUI({
+      taskDefaults: {
+        ...ui.taskDefaults,
+        cleanGenerated: options,
+      },
+    });
+    onClose(options);
+  }
+
+  if (loading) return <LoadingIndicator />;
+
+  return (
+    <ModalComponent
+      show
+      header={<FormattedMessage id="actions.clean_generated" />}
+      icon={faTrashAlt}
+      accept={{
+        text: intl.formatMessage({ id: "actions.clean_generated" }),
+        variant: "danger",
+        onClick: () => confirm(),
+      }}
+      cancel={{ onClick: () => onClose() }}
+    >
+      <div className="dialog-container">
+        <p>
+          <FormattedMessage id="config.tasks.clean_generated.description" />
+        </p>
+        <SettingSection>
+          <CleanGeneratedOptions options={options} setOptions={setOptions} />
+        </SettingSection>
+        {options.dryRun && (
+          <p>
+            <FormattedMessage id="actions.tasks.dry_mode_selected" />
+          </p>
+        )}
+      </div>
+    </ModalComponent>
+  );
+};

ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx

@@ -11,6 +11,7 @@ import {
   mutateMigrateSceneScreenshots,
   mutateMigrateBlobs,
   mutateOptimiseDatabase,
+  mutateCleanGenerated,
 } from "src/core/StashService";
 import { useToast } from "src/hooks/Toast";
 import downloadFile from "src/utils/download";
@@ -29,6 +30,7 @@ import {
   faQuestionCircle,
   faTrashAlt,
 } from "@fortawesome/free-solid-svg-icons";
+import { CleanGeneratedDialog } from "./CleanGeneratedDialog";
 
 interface ICleanDialog {
   pathSelection?: boolean;
@@ -167,6 +169,7 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
     import: false,
     clean: false,
     cleanAlert: false,
+    cleanGenerated: false,
   });
 
   const [cleanOptions, setCleanOptions] = useState<GQL.CleanMetadataInput>({
@@ -252,6 +255,27 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
     }
   }
 
+  async function onCleanGenerated(options: GQL.CleanGeneratedInput) {
+    try {
+      await mutateCleanGenerated({
+        ...options,
+      });
+
+      Toast.success(
+        intl.formatMessage(
+          { id: "config.tasks.added_job_to_queue" },
+          {
+            operation_name: intl.formatMessage({
+              id: "actions.clean_generated",
+            }),
+          }
+        )
+      );
+    } catch (e) {
+      Toast.error(e);
+    }
+  }
+
   async function onMigrateHashNaming() {
     try {
       await mutateMigrateHashNaming();
@@ -404,6 +428,17 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
       ) : (
         dialogOpen.clean
       )}
+      {dialogOpen.cleanGenerated && (
+        <CleanGeneratedDialog
+          onClose={(options) => {
+            if (options) {
+              onCleanGenerated(options);
+            }
+
+            setDialogOpen({ cleanGenerated: false });
+          }}
+        />
+      )}
 
       <SettingSection headingID="config.tasks.maintenance">
         <div className="setting-group">
@@ -439,6 +474,21 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
           />
         </div>
 
+        <div className="setting-group">
+          <Setting
+            heading={<FormattedMessage id="actions.clean_generated" />}
+            subHeadingID="config.tasks.clean_generated.description"
+          >
+            <Button
+              variant="danger"
+              type="submit"
+              onClick={() => setDialogOpen({ cleanGenerated: true })}
+            >
+              <FormattedMessage id="actions.clean_generated" />…
+            </Button>
+          </Setting>
+        </div>
+
         <Setting
           headingID="actions.optimise_database"
           subHeading={

ui/v2.5/src/components/Settings/Tasks/LibraryTasks.tsx

@@ -74,7 +74,6 @@ export const LibraryTasks: React.FC = () => {
   const [configureDefaults] = useConfigureDefaults();
 
   const [dialogOpen, setDialogOpenState] = useState({
-    clean: false,
     scan: false,
     autoTag: false,
     identify: false,

ui/v2.5/src/core/StashService.ts

@@ -2506,6 +2506,12 @@ export const mutateMetadataClean = (input: GQL.CleanMetadataInput) =>
     variables: { input },
   });
 
+export const mutateCleanGenerated = (input: GQL.CleanGeneratedInput) =>
+  client.mutate<GQL.MetadataCleanGeneratedMutation>({
+    mutation: GQL.MetadataCleanGeneratedDocument,
+    variables: { input },
+  });
+
 export const mutateRunPluginTask = (
   pluginId: string,
   taskName: string,

ui/v2.5/src/core/config.ts

@@ -84,6 +84,8 @@ export interface IUIConfig {
   tableColumns?: Record<string, string[]>;
 
   advancedMode?: boolean;
+
+  taskDefaults?: Record<string, {}>;
 }
 
 interface ISavedFilterRowBroken extends ISavedFilterRow {

ui/v2.5/src/locales/en-GB.json

@@ -18,6 +18,7 @@
     "cancel": "Cancel",
     "choose_date": "Choose a date",
     "clean": "Clean",
+    "clean_generated": "Clean generated files",
     "clear": "Clear",
     "clear_back_image": "Clear back image",
     "clear_date_data": "Clear date data",
@@ -443,6 +444,17 @@
       "backup_and_download": "Performs a backup of the database and downloads the resulting file.",
       "backup_database": "Performs a backup of the database to the backups directory, with the filename format {filename_format}",
       "cleanup_desc": "Check for missing files and remove them from the database. This is a destructive action.",
+      "clean_generated": {
+        "blob_files": "Blob files",
+        "description": "Removes generated files without a corresponding database entry.",
+        "image_thumbnails": "Image Thumbnails",
+        "image_thumbnails_desc": "Image thumbnails and clips",
+        "markers": "Marker Previews",
+        "previews": "Scene Previews",
+        "previews_desc": "Scene previews and thumbnails",
+        "sprites": "Scene Sprites",
+        "transcodes": "Scene Transcodes"
+      },
       "data_management": "Data management",
       "defaults_set": "Defaults have been set and will be used when clicking the {action} button on the Tasks page.",
       "dont_include_file_extension_as_part_of_the_title": "Don't include file extension as part of the title",