Fix various generate issues (#1322)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

pkg/manager/generator_preview.go

@@ -58,11 +58,6 @@ func (g *PreviewGenerator) Generate() error {
 	}
 
 	encoder := ffmpeg.NewEncoder(instance.FFMPEGPath)
-
-	if err := g.generateConcatFile(); err != nil {
-		return err
-	}
-
 	if g.GenerateVideo {
 		if err := g.generateVideo(&encoder, false); err != nil {
 			logger.Warnf("[generator] failed generating scene preview, trying fallback")
@@ -101,18 +96,32 @@ func (g *PreviewGenerator) generateVideo(encoder *ffmpeg.Encoder, fallback bool)
 	if !g.Overwrite && outputExists {
 		return nil
 	}
+	err := g.generateConcatFile()
+	if err != nil {
+		return err
+	}
+
+	var tmpFiles []string                              // a list of tmp files used during the preview generation
+	tmpFiles = append(tmpFiles, g.getConcatFilePath()) // add concat filename to tmpFiles
+	defer func() { removeFiles(tmpFiles) }()           // remove tmpFiles when done
 
 	stepSize, offset := g.Info.getStepSizeAndOffset()
 
+	durationSegment := g.Info.ChunkDuration
+	if durationSegment < 0.75 { // a very short duration can create files without a video stream
+		durationSegment = 0.75 // use 0.75 in that case
+		logger.Warnf("[generator] Segment duration (%f) too short.Using 0.75 instead.", g.Info.ChunkDuration)
+	}
+
 	for i := 0; i < g.Info.ChunkCount; i++ {
 		time := offset + (float64(i) * stepSize)
 		num := fmt.Sprintf("%.3d", i)
 		filename := "preview_" + g.VideoChecksum + "_" + num + ".mp4"
 		chunkOutputPath := instance.Paths.Generated.GetTmpPath(filename)
-
+		tmpFiles = append(tmpFiles, chunkOutputPath) // add chunk filename to tmpFiles
 		options := ffmpeg.ScenePreviewChunkOptions{
 			StartTime:  time,
-			Duration:   g.Info.ChunkDuration,
+			Duration:   durationSegment,
 			Width:      640,
 			OutputPath: chunkOutputPath,
 		}
@@ -152,3 +161,11 @@ func (g *PreviewGenerator) generateImage(encoder *ffmpeg.Encoder) error {
 func (g *PreviewGenerator) getConcatFilePath() string {
 	return instance.Paths.Generated.GetTmpPath(fmt.Sprintf("files_%s.txt", g.VideoChecksum))
 }
+
+func removeFiles(list []string) {
+	for _, f := range list {
+		if err := os.Remove(f); err != nil {
+			logger.Warnf("[generator] Delete error: %s", err)
+		}
+	}
+}

pkg/manager/manager.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/stashapp/stash/pkg/database"
 	"github.com/stashapp/stash/pkg/ffmpeg"
@@ -142,8 +143,16 @@ func (s *singleton) PostInit() error {
 	// clear the downloads and tmp directories
 	// #1021 - only clear these directories if the generated folder is non-empty
 	if s.Config.GetGeneratedPath() != "" {
+		const deleteTimeout = 1 * time.Second
+
+		utils.Timeout(func() {
 			utils.EmptyDir(instance.Paths.Generated.Downloads)
 			utils.EmptyDir(instance.Paths.Generated.Tmp)
+		}, deleteTimeout, func(done chan struct{}) {
+			logger.Info("Please wait. Deleting temporary files...") // print
+			<-done                                                  // and wait for deletion
+			logger.Info("Temporary files deleted.")
+		})
 	}
 
 	if err := database.Initialize(s.Config.GetDatabasePath()); err != nil {

pkg/manager/manager_tasks.go

@@ -192,11 +192,12 @@ func (s *singleton) Scan(input models.ScanMetadataInput) {
 
 		i := 0
 		stoppingErr := errors.New("stopping")
+		var err error
 
 		var galleries []string
 
 		for _, sp := range paths {
-			err := walkFilesToScan(sp, func(path string, info os.FileInfo, err error) error {
+			err = walkFilesToScan(sp, func(path string, info os.FileInfo, err error) error {
 				if total != nil {
 					s.Status.setProgress(i, *total)
 					i++
@@ -231,26 +232,25 @@ func (s *singleton) Scan(input models.ScanMetadataInput) {
 			})
 
 			if err == stoppingErr {
+				logger.Info("Stopping due to user request")
 				break
 			}
 
 			if err != nil {
 				logger.Errorf("Error encountered scanning files: %s", err.Error())
-				return
+				break
 			}
 		}
 
-		if s.Status.stopping {
-			logger.Info("Stopping due to user request")
-			return
-		}
-
 		wg.Wait()
 		instance.Paths.Generated.EmptyTmpDir()
-
 		elapsed := time.Since(start)
 		logger.Info(fmt.Sprintf("Scan finished (%s)", elapsed))
 
+		if s.Status.stopping || err != nil {
+			return
+		}
+
 		for _, path := range galleries {
 			wg.Add()
 			task := ScanTask{
@@ -464,7 +464,7 @@ func (s *singleton) Generate(input models.GenerateMetadataInput) {
 		}
 		setGeneratePreviewOptionsInput(generatePreviewOptions)
 
-		// Start measuring how long the scan has taken. (consider moving this up)
+		// Start measuring how long the generate has taken. (consider moving this up)
 		start := time.Now()
 		instance.Paths.Generated.EnsureTmpDir()
 
@@ -472,6 +472,8 @@ func (s *singleton) Generate(input models.GenerateMetadataInput) {
 			s.Status.setProgress(i, total)
 			if s.Status.stopping {
 				logger.Info("Stopping due to user request")
+				wg.Wait()
+				instance.Paths.Generated.EmptyTmpDir()
 				return
 			}
 
@@ -540,6 +542,10 @@ func (s *singleton) Generate(input models.GenerateMetadataInput) {
 			s.Status.setProgress(lenScenes+i, total)
 			if s.Status.stopping {
 				logger.Info("Stopping due to user request")
+				wg.Wait()
+				instance.Paths.Generated.EmptyTmpDir()
+				elapsed := time.Since(start)
+				logger.Info(fmt.Sprintf("Generate finished (%s)", elapsed))
 				return
 			}
 
@@ -616,7 +622,7 @@ func (s *singleton) generateScreenshot(sceneId string, at *float64) {
 
 		wg.Wait()
 
-		logger.Infof("Generate finished")
+		logger.Infof("Generate screenshot finished")
 	}()
 }
 

pkg/utils/time.go

@@ -0,0 +1,22 @@
+package utils
+
+import "time"
+
+// Timeout executes the provided todo function, and waits for it to return. If
+// the function does not return before the waitTime duration is elapsed, then
+// onTimeout is executed, passing a channel that will be closed when the
+// function returns.
+func Timeout(todo func(), waitTime time.Duration, onTimeout func(done chan struct{})) {
+	done := make(chan struct{})
+
+	go func() {
+		todo()
+		close(done)
+	}()
+
+	select {
+	case <-done: // on time, just exit
+	case <-time.After(waitTime):
+		onTimeout(done)
+	}
+}

ui/v2.5/src/components/Changelog/versions/v070.md

@@ -11,6 +11,8 @@
 * Added scene queue.
 
 ### 🎨 Improvements
+* Clean generation artifacts after generating each scene.
+* Log message at startup when cleaning the `tmp` and `downloads` generated folders takes more than one second.
 * Sort movie scenes by scene number by default.
 * Support http request headers in scrapers.
 * Sort performers by gender in scene/image/gallery cards and details.

ui/v2.5/src/docs/en/Tagger.md

@@ -6,16 +6,16 @@ Stash can be integrated with stash-box which acts as a centralized metadata data
 
 The fingerprint search matches your current selection of files against the remote stash-box instance. Any scenes with a matching fingerprint will be returned, although there is currently no validation of fingerprints so it’s recommended to double-check the validity before saving.
 
-If no fingerprint match is found it’s possible to search by keywords. The search works by matching the query against a scene’s  title_, release date_, _studio name_, and _performer names_. By default the tagger uses metadata set on the file, or parses the filename, this can be changed in the config.
+If no fingerprint match is found it’s possible to search by keywords. The search works by matching the query against a scene’s _title_, _release date_, _studio name_, and _performer names_. By default the tagger uses metadata set on the file, or parses the filename, this can be changed in the config.
 
 An important thing to note is that it only returns a match *if all query terms are a match*. As an example, if a scene is titled `"A Trip to the Mall"` with the performer `"Jane Doe"`, a search for `"Trip to the Mall 1080p"` will *not* match, however `"trip mall doe"` would. Usually a few pieces of info is enough, for instance performer name + release date or studio name. To avoid common non-related keywords you can add them to the blacklist in the tagger config. Any items in the blacklist are stripped out of the query.
 
 #### Saving
 When a scene is matched stash will try to match the studio and performers against your local studios and performers. If you have previously matched them, they will automatically be selected. If not you either have to select the correct performer/studio from the dropdown, choose create to create a new entity, or skip to ignore it.
 
-Once a scene is saved the scene and the matched studio/performers will have the stash_id saved which will then be used for future tagging.
+Once a scene is saved the scene and the matched studio/performers will have the `stash_id` saved which will then be used for future tagging.
 
 By default male performers are not shown, this can be enabled in the tagger config. Likewise scene tags are by default not saved. They can be set to either merge with existing tags on the scene, or overwrite them. It is not recommended to set tags currently since they are hard to deduplicate and can litter your data.
 
 #### Submitting fingerprints
-After a scene is saved you will prompted to submit the fingerprint back to the stash-box instance. This is optional, but can be helpful for other users who have an identical copy who will then be able to match via the fingerprint search. No other information than the stash_id and file fingerprint is submitted.
+After a scene is saved you will prompted to submit the fingerprint back to the stash-box instance. This is optional, but can be helpful for other users who have an identical copy who will then be able to match via the fingerprint search. No other information than the `stash_id` and file fingerprint is submitted.