Generate screenshot images for markers (#1604)

* Generate screenshot images for markers

In some scenarios it might not be possible to use the preview video or
image of markers, i.e. when only static images are supported like in
Kodi. So generate a static screenshot as well.

* Make generating animated and static image optional for markers
* Use screenshot for markers when preview type is set to static image

graphql/documents/data/scene-marker.graphql

@@ -4,6 +4,7 @@ fragment SceneMarkerData on SceneMarker {
   seconds
   stream
   preview
+  screenshot
 
   scene {
     id

graphql/schema/types/metadata.graphql

@@ -6,6 +6,8 @@ input GenerateMetadataInput {
   imagePreviews: Boolean!
   previewOptions: GeneratePreviewOptionsInput
   markers: Boolean!
+  markerImagePreviews: Boolean!
+  markerScreenshots: Boolean!
   transcodes: Boolean!
   phashes: Boolean!
 

graphql/schema/types/scene-marker.graphql

@@ -12,6 +12,8 @@ type SceneMarker {
   stream: String! # Resolver
   """The path to the preview image for this marker"""
   preview: String! # Resolver
+  """The path to the screenshot image for this marker"""
+  screenshot: String! # Resolver
 }
 
 input SceneMarkerCreateInput {

pkg/api/resolver_model_scene_marker.go

@@ -58,6 +58,12 @@ func (r *sceneMarkerResolver) Preview(ctx context.Context, obj *models.SceneMark
 	return urlbuilders.NewSceneURLBuilder(baseURL, sceneID).GetSceneMarkerStreamPreviewURL(obj.ID), nil
 }
 
+func (r *sceneMarkerResolver) Screenshot(ctx context.Context, obj *models.SceneMarker) (string, error) {
+	baseURL, _ := ctx.Value(BaseURLCtxKey).(string)
+	sceneID := int(obj.SceneID.Int64)
+	return urlbuilders.NewSceneURLBuilder(baseURL, sceneID).GetSceneMarkerStreamScreenshotURL(obj.ID), nil
+}
+
 func (r *sceneMarkerResolver) CreatedAt(ctx context.Context, obj *models.SceneMarker) (*time.Time, error) {
 	return &obj.CreatedAt.Timestamp, nil
 }

pkg/api/routes_scene.go

@@ -41,6 +41,7 @@ func (rs sceneRoutes) Routes() chi.Router {
 
 		r.Get("/scene_marker/{sceneMarkerId}/stream", rs.SceneMarkerStream)
 		r.Get("/scene_marker/{sceneMarkerId}/preview", rs.SceneMarkerPreview)
+		r.Get("/scene_marker/{sceneMarkerId}/screenshot", rs.SceneMarkerScreenshot)
 	})
 	r.With(SceneCtx).Get("/{sceneId}_thumbs.vtt", rs.VttThumbs)
 	r.With(SceneCtx).Get("/{sceneId}_sprite.jpg", rs.VttSprite)
@@ -319,6 +320,33 @@ func (rs sceneRoutes) SceneMarkerPreview(w http.ResponseWriter, r *http.Request)
 	http.ServeFile(w, r, filepath)
 }
 
+func (rs sceneRoutes) SceneMarkerScreenshot(w http.ResponseWriter, r *http.Request) {
+	scene := r.Context().Value(sceneKey).(*models.Scene)
+	sceneMarkerID, _ := strconv.Atoi(chi.URLParam(r, "sceneMarkerId"))
+	var sceneMarker *models.SceneMarker
+	if err := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error {
+		var err error
+		sceneMarker, err = repo.SceneMarker().Find(sceneMarkerID)
+		return err
+	}); err != nil {
+		logger.Warnf("Error when getting scene marker for stream: %s", err.Error())
+		http.Error(w, http.StatusText(500), 500)
+		return
+	}
+	filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamScreenshotPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds))
+
+	// If the image doesn't exist, send the placeholder
+	exists, _ := utils.FileExists(filepath)
+	if !exists {
+		w.Header().Set("Content-Type", "image/png")
+		w.Header().Set("Cache-Control", "no-store")
+		_, _ = w.Write(utils.PendingGenerateResource)
+		return
+	}
+
+	http.ServeFile(w, r, filepath)
+}
+
 // endregion
 
 func SceneCtx(next http.Handler) http.Handler {

pkg/api/urlbuilders/scene.go

@@ -59,6 +59,10 @@ func (b SceneURLBuilder) GetSceneMarkerStreamPreviewURL(sceneMarkerID int) strin
 	return b.BaseURL + "/scene/" + b.SceneID + "/scene_marker/" + strconv.Itoa(sceneMarkerID) + "/preview"
 }
 
+func (b SceneURLBuilder) GetSceneMarkerStreamScreenshotURL(sceneMarkerID int) string {
+	return b.BaseURL + "/scene/" + b.SceneID + "/scene_marker/" + strconv.Itoa(sceneMarkerID) + "/screenshot"
+}
+
 func (b SceneURLBuilder) GetFunscriptURL() string {
 	return b.BaseURL + "/scene/" + b.SceneID + "/funscript"
 }

pkg/manager/manager_tasks.go

@@ -299,6 +299,8 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI
 					Scene:               scene,
 					Overwrite:           overwrite,
 					fileNamingAlgorithm: fileNamingAlgo,
+					ImagePreview:        input.MarkerImagePreviews,
+					Screenshot:          input.MarkerScreenshots,
 				}
 				go progress.ExecuteTask(fmt.Sprintf("Generating markers for %s", scene.Path), func() {
 					task.Start(&wg)

pkg/manager/paths/paths_scene_markers.go

@@ -22,3 +22,7 @@ func (sp *sceneMarkerPaths) GetStreamPath(checksum string, seconds int) string {
 func (sp *sceneMarkerPaths) GetStreamPreviewImagePath(checksum string, seconds int) string {
 	return filepath.Join(sp.generated.Markers, checksum, strconv.Itoa(seconds)+".webp")
 }
+
+func (sp *sceneMarkerPaths) GetStreamScreenshotPath(checksum string, seconds int) string {
+	return filepath.Join(sp.generated.Markers, checksum, strconv.Itoa(seconds)+".jpg")
+}

pkg/manager/scene.go

@@ -148,6 +148,7 @@ func DeleteGeneratedSceneFiles(scene *models.Scene, fileNamingAlgo models.HashAl
 func DeleteSceneMarkerFiles(scene *models.Scene, seconds int, fileNamingAlgo models.HashAlgorithm) {
 	videoPath := GetInstance().Paths.SceneMarkers.GetStreamPath(scene.GetHash(fileNamingAlgo), seconds)
 	imagePath := GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.GetHash(fileNamingAlgo), seconds)
+	screenshotPath := GetInstance().Paths.SceneMarkers.GetStreamScreenshotPath(scene.GetHash(fileNamingAlgo), seconds)
 
 	exists, _ := utils.FileExists(videoPath)
 	if exists {
@@ -161,7 +162,15 @@ func DeleteSceneMarkerFiles(scene *models.Scene, seconds int, fileNamingAlgo mod
 	if exists {
 		err := os.Remove(imagePath)
 		if err != nil {
-			logger.Warnf("Could not delete file %s: %s", videoPath, err.Error())
+			logger.Warnf("Could not delete file %s: %s", imagePath, err.Error())
+		}
+	}
+
+	exists, _ = utils.FileExists(screenshotPath)
+	if exists {
+		err := os.Remove(screenshotPath)
+		if err != nil {
+			logger.Warnf("Could not delete file %s: %s", screenshotPath, err.Error())
 		}
 	}
 }

pkg/manager/task_generate_markers.go

@@ -19,6 +19,9 @@ type GenerateMarkersTask struct {
 	Marker              *models.SceneMarker
 	Overwrite           bool
 	fileNamingAlgorithm models.HashAlgorithm
+
+	ImagePreview bool
+	Screenshot   bool
 }
 
 func (t *GenerateMarkersTask) Start(wg *sizedwaitgroup.SizedWaitGroup) {
@@ -94,7 +97,8 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene
 	seconds := int(sceneMarker.Seconds)
 
 	videoExists := t.videoExists(sceneHash, seconds)
-	imageExists := t.imageExists(sceneHash, seconds)
+	imageExists := !t.ImagePreview || t.imageExists(sceneHash, seconds)
+	screenshotExists := !t.Screenshot || t.screenshotExists(sceneHash, seconds)
 
 	baseFilename := strconv.Itoa(seconds)
 
@@ -119,7 +123,7 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene
 		}
 	}
 
-	if t.Overwrite || !imageExists {
+	if t.ImagePreview && (t.Overwrite || !imageExists) {
 		imageFilename := baseFilename + ".webp"
 		imagePath := instance.Paths.SceneMarkers.GetStreamPreviewImagePath(sceneHash, seconds)
 
@@ -131,6 +135,24 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene
 			logger.Debug("created marker image: ", imagePath)
 		}
 	}
+
+	if t.Screenshot && (t.Overwrite || !screenshotExists) {
+		screenshotFilename := baseFilename + ".jpg"
+		screenshotPath := instance.Paths.SceneMarkers.GetStreamScreenshotPath(sceneHash, seconds)
+
+		screenshotOptions := ffmpeg.ScreenshotOptions{
+			OutputPath: instance.Paths.Generated.GetTmpPath(screenshotFilename), // tmp output in case the process ends abruptly
+			Quality:    2,
+			Width:      videoFile.Width,
+			Time:       float64(seconds),
+		}
+		if err := encoder.Screenshot(*videoFile, screenshotOptions); err != nil {
+			logger.Errorf("[generator] failed to generate marker screenshot: %s", err)
+		} else {
+			_ = utils.SafeMove(screenshotOptions.OutputPath, screenshotPath)
+			logger.Debug("created marker screenshot: ", screenshotPath)
+		}
+	}
 }
 
 func (t *GenerateMarkersTask) isMarkerNeeded() int {
@@ -166,12 +188,11 @@ func (t *GenerateMarkersTask) markerExists(sceneChecksum string, seconds int) bo
 		return false
 	}
 
-	videoPath := instance.Paths.SceneMarkers.GetStreamPath(sceneChecksum, seconds)
+	videoExists := t.videoExists(sceneChecksum, seconds)
-	imagePath := instance.Paths.SceneMarkers.GetStreamPreviewImagePath(sceneChecksum, seconds)
+	imageExists := !t.ImagePreview || t.imageExists(sceneChecksum, seconds)
-	videoExists, _ := utils.FileExists(videoPath)
+	screenshotExists := !t.Screenshot || t.screenshotExists(sceneChecksum, seconds)
-	imageExists, _ := utils.FileExists(imagePath)
 
-	return videoExists && imageExists
+	return videoExists && imageExists && screenshotExists
 }
 
 func (t *GenerateMarkersTask) videoExists(sceneChecksum string, seconds int) bool {
@@ -195,3 +216,14 @@ func (t *GenerateMarkersTask) imageExists(sceneChecksum string, seconds int) boo
 
 	return imageExists
 }
+
+func (t *GenerateMarkersTask) screenshotExists(sceneChecksum string, seconds int) bool {
+	if sceneChecksum == "" {
+		return false
+	}
+
+	screenshotPath := instance.Paths.SceneMarkers.GetStreamScreenshotPath(sceneChecksum, seconds)
+	screenshotExists, _ := utils.FileExists(screenshotPath)
+
+	return screenshotExists
+}

ui/v2.5/src/components/Changelog/versions/v0100.md

@@ -1,4 +1,5 @@
 ### ✨ New Features
+* Added options to generate webp and static preview files for markers. ([#1604](https://github.com/stashapp/stash/pull/1604))
 * Added sort by option for gallery rating. ([#1720](https://github.com/stashapp/stash/pull/1720))
 * Added support for querying scene scrapers using keywords. ([#1712](https://github.com/stashapp/stash/pull/1712))
 * Added support for Studio aliases. ([#1660](https://github.com/stashapp/stash/pull/1660))

ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx

@@ -40,6 +40,8 @@ export const SceneGenerateDialog: React.FC<ISceneGenerateDialogProps> = (
   const [previewPreset, setPreviewPreset] = useState<string>(
     GQL.PreviewPreset.Slow
   );
+  const [markerImagePreviews, setMarkerImagePreviews] = useState(false);
+  const [markerScreenshots, setMarkerScreenshots] = useState(false);
 
   const [previewOptionsOpen, setPreviewOptionsOpen] = useState(false);
 
@@ -67,6 +69,8 @@ export const SceneGenerateDialog: React.FC<ISceneGenerateDialogProps> = (
         previews,
         imagePreviews: previews && imagePreviews,
         markers,
+        markerImagePreviews: markers && markerImagePreviews,
+        markerScreenshots: markers && markerScreenshots,
         transcodes,
         overwrite,
         sceneIDs: props.selectedIds,
@@ -276,6 +280,31 @@ export const SceneGenerateDialog: React.FC<ISceneGenerateDialogProps> = (
             label={intl.formatMessage({ id: "dialogs.scene_gen.markers" })}
             onChange={() => setMarkers(!markers)}
           />
+          <div className="d-flex flex-row">
+            <div>↳</div>
+            <Form.Group>
+              <Form.Check
+                id="marker-image-preview-task"
+                checked={markerImagePreviews}
+                disabled={!markers}
+                label={intl.formatMessage({
+                  id: "dialogs.scene_gen.marker_image_previews",
+                })}
+                onChange={() => setMarkerImagePreviews(!markerImagePreviews)}
+                className="ml-2 flex-grow"
+              />
+              <Form.Check
+                id="marker-screenshot-task"
+                checked={markerScreenshots}
+                disabled={!markers}
+                label={intl.formatMessage({
+                  id: "dialogs.scene_gen.marker_screenshots",
+                })}
+                onChange={() => setMarkerScreenshots(!markerScreenshots)}
+                className="ml-2 flex-grow"
+              />
+            </Form.Group>
+          </div>
           <Form.Check
             id="transcode-task"
             checked={transcodes}

ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx

@@ -13,6 +13,8 @@ export const GenerateButton: React.FC = () => {
   const [markers, setMarkers] = useState(true);
   const [transcodes, setTranscodes] = useState(false);
   const [imagePreviews, setImagePreviews] = useState(false);
+  const [markerImagePreviews, setMarkerImagePreviews] = useState(false);
+  const [markerScreenshots, setMarkerScreenshots] = useState(false);
 
   async function onGenerate() {
     try {
@@ -22,6 +24,8 @@ export const GenerateButton: React.FC = () => {
         previews,
         imagePreviews: previews && imagePreviews,
         markers,
+        markerImagePreviews: markers && markerImagePreviews,
+        markerScreenshots: markers && markerScreenshots,
         transcodes,
       });
       Toast.success({
@@ -68,6 +72,31 @@ export const GenerateButton: React.FC = () => {
           label={intl.formatMessage({ id: "dialogs.scene_gen.markers" })}
           onChange={() => setMarkers(!markers)}
         />
+        <div className="d-flex flex-row">
+          <div>↳</div>
+          <Form.Group>
+            <Form.Check
+              id="marker-image-preview-task"
+              checked={markerImagePreviews}
+              disabled={!markers}
+              label={intl.formatMessage({
+                id: "dialogs.scene_gen.marker_image_previews",
+              })}
+              onChange={() => setMarkerImagePreviews(!markerImagePreviews)}
+              className="ml-2 flex-grow"
+            />
+            <Form.Check
+              id="marker-screenshot-task"
+              checked={markerScreenshots}
+              disabled={!markers}
+              label={intl.formatMessage({
+                id: "dialogs.scene_gen.marker_screenshots",
+              })}
+              onChange={() => setMarkerScreenshots(!markerScreenshots)}
+              className="ml-2 flex-grow"
+            />
+          </Form.Group>
+        </div>
         <Form.Check
           id="transcode-task"
           checked={transcodes}

ui/v2.5/src/components/Wall/WallItem.tsx

@@ -114,6 +114,7 @@ export const WallItem: React.FC<IWallItemProps> = (props: IWallItemProps) => {
     ? {
         video: props.sceneMarker.stream,
         animation: props.sceneMarker.preview,
+        image: props.sceneMarker.screenshot,
       }
     : props.scene
     ? {

ui/v2.5/src/locales/en-GB.json

@@ -440,6 +440,8 @@
     "scene_gen": {
       "image_previews": "Image Previews (animated WebP previews, only required if Preview Type is set to Animated Image)",
       "markers": "Markers (20 second videos which begin at the given timecode)",
+      "marker_image_previews": "Marker Previews (animated WebP previews, only required if Preview Type is set to Animated Image)",
+      "marker_screenshots": "Marker Screenshots (static JPG image, only required if Preview Type is set to Static Image)",
       "overwrite": "Overwrite existing generated files",
       "phash": "Perceptual hashes (for deduplication)",
       "preview_exclude_end_time_desc": "Exclude the last x seconds from scene previews. This can be a value in seconds, or a percentage (eg 2%) of the total scene duration.",