From 2274db16b7c1cf6393b6c8d8974d8b331a19a6ec Mon Sep 17 00:00:00 2001 From: gitgiggety <79809426+gitgiggety@users.noreply.github.com> Date: Wed, 15 Sep 2021 04:27:05 +0200 Subject: [PATCH] Generate screenshot images for markers (#1604) * Generate screenshot images for markers In some scenarios it might not be possible to use the preview video or image of markers, i.e. when only static images are supported like in Kodi. So generate a static screenshot as well. * Make generating animated and static image optional for markers * Use screenshot for markers when preview type is set to static image --- graphql/documents/data/scene-marker.graphql | 1 + graphql/schema/types/metadata.graphql | 2 + graphql/schema/types/scene-marker.graphql | 2 + pkg/api/resolver_model_scene_marker.go | 6 +++ pkg/api/routes_scene.go | 28 +++++++++++ pkg/api/urlbuilders/scene.go | 4 ++ pkg/manager/manager_tasks.go | 2 + pkg/manager/paths/paths_scene_markers.go | 4 ++ pkg/manager/scene.go | 11 ++++- pkg/manager/task_generate_markers.go | 46 ++++++++++++++++--- .../components/Changelog/versions/v0100.md | 1 + .../components/Scenes/SceneGenerateDialog.tsx | 29 ++++++++++++ .../SettingsTasksPanel/GenerateButton.tsx | 29 ++++++++++++ ui/v2.5/src/components/Wall/WallItem.tsx | 1 + ui/v2.5/src/locales/en-GB.json | 2 + 15 files changed, 160 insertions(+), 8 deletions(-) diff --git a/graphql/documents/data/scene-marker.graphql b/graphql/documents/data/scene-marker.graphql index 30091d857..61439bd1e 100644 --- a/graphql/documents/data/scene-marker.graphql +++ b/graphql/documents/data/scene-marker.graphql @@ -4,6 +4,7 @@ fragment SceneMarkerData on SceneMarker { seconds stream preview + screenshot scene { id diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 567b77079..28f293fd8 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -6,6 +6,8 @@ input GenerateMetadataInput { imagePreviews: Boolean! previewOptions: GeneratePreviewOptionsInput markers: Boolean! + markerImagePreviews: Boolean! + markerScreenshots: Boolean! transcodes: Boolean! phashes: Boolean! diff --git a/graphql/schema/types/scene-marker.graphql b/graphql/schema/types/scene-marker.graphql index f29380f26..8e3e54c81 100644 --- a/graphql/schema/types/scene-marker.graphql +++ b/graphql/schema/types/scene-marker.graphql @@ -12,6 +12,8 @@ type SceneMarker { stream: String! # Resolver """The path to the preview image for this marker""" preview: String! # Resolver + """The path to the screenshot image for this marker""" + screenshot: String! # Resolver } input SceneMarkerCreateInput { diff --git a/pkg/api/resolver_model_scene_marker.go b/pkg/api/resolver_model_scene_marker.go index 84bd5c9f2..3a6ada5bc 100644 --- a/pkg/api/resolver_model_scene_marker.go +++ b/pkg/api/resolver_model_scene_marker.go @@ -58,6 +58,12 @@ func (r *sceneMarkerResolver) Preview(ctx context.Context, obj *models.SceneMark return urlbuilders.NewSceneURLBuilder(baseURL, sceneID).GetSceneMarkerStreamPreviewURL(obj.ID), nil } +func (r *sceneMarkerResolver) Screenshot(ctx context.Context, obj *models.SceneMarker) (string, error) { + baseURL, _ := ctx.Value(BaseURLCtxKey).(string) + sceneID := int(obj.SceneID.Int64) + return urlbuilders.NewSceneURLBuilder(baseURL, sceneID).GetSceneMarkerStreamScreenshotURL(obj.ID), nil +} + func (r *sceneMarkerResolver) CreatedAt(ctx context.Context, obj *models.SceneMarker) (*time.Time, error) { return &obj.CreatedAt.Timestamp, nil } diff --git a/pkg/api/routes_scene.go b/pkg/api/routes_scene.go index 0bc48f66f..fb038ce29 100644 --- a/pkg/api/routes_scene.go +++ b/pkg/api/routes_scene.go @@ -41,6 +41,7 @@ func (rs sceneRoutes) Routes() chi.Router { r.Get("/scene_marker/{sceneMarkerId}/stream", rs.SceneMarkerStream) r.Get("/scene_marker/{sceneMarkerId}/preview", rs.SceneMarkerPreview) + r.Get("/scene_marker/{sceneMarkerId}/screenshot", rs.SceneMarkerScreenshot) }) r.With(SceneCtx).Get("/{sceneId}_thumbs.vtt", rs.VttThumbs) r.With(SceneCtx).Get("/{sceneId}_sprite.jpg", rs.VttSprite) @@ -319,6 +320,33 @@ func (rs sceneRoutes) SceneMarkerPreview(w http.ResponseWriter, r *http.Request) http.ServeFile(w, r, filepath) } +func (rs sceneRoutes) SceneMarkerScreenshot(w http.ResponseWriter, r *http.Request) { + scene := r.Context().Value(sceneKey).(*models.Scene) + sceneMarkerID, _ := strconv.Atoi(chi.URLParam(r, "sceneMarkerId")) + var sceneMarker *models.SceneMarker + if err := rs.txnManager.WithReadTxn(r.Context(), func(repo models.ReaderRepository) error { + var err error + sceneMarker, err = repo.SceneMarker().Find(sceneMarkerID) + return err + }); err != nil { + logger.Warnf("Error when getting scene marker for stream: %s", err.Error()) + http.Error(w, http.StatusText(500), 500) + return + } + filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamScreenshotPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds)) + + // If the image doesn't exist, send the placeholder + exists, _ := utils.FileExists(filepath) + if !exists { + w.Header().Set("Content-Type", "image/png") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(utils.PendingGenerateResource) + return + } + + http.ServeFile(w, r, filepath) +} + // endregion func SceneCtx(next http.Handler) http.Handler { diff --git a/pkg/api/urlbuilders/scene.go b/pkg/api/urlbuilders/scene.go index e9766da9e..8e45feb6e 100644 --- a/pkg/api/urlbuilders/scene.go +++ b/pkg/api/urlbuilders/scene.go @@ -59,6 +59,10 @@ func (b SceneURLBuilder) GetSceneMarkerStreamPreviewURL(sceneMarkerID int) strin return b.BaseURL + "/scene/" + b.SceneID + "/scene_marker/" + strconv.Itoa(sceneMarkerID) + "/preview" } +func (b SceneURLBuilder) GetSceneMarkerStreamScreenshotURL(sceneMarkerID int) string { + return b.BaseURL + "/scene/" + b.SceneID + "/scene_marker/" + strconv.Itoa(sceneMarkerID) + "/screenshot" +} + func (b SceneURLBuilder) GetFunscriptURL() string { return b.BaseURL + "/scene/" + b.SceneID + "/funscript" } diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index 6173c30ed..101d3aa9f 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -299,6 +299,8 @@ func (s *singleton) Generate(ctx context.Context, input models.GenerateMetadataI Scene: scene, Overwrite: overwrite, fileNamingAlgorithm: fileNamingAlgo, + ImagePreview: input.MarkerImagePreviews, + Screenshot: input.MarkerScreenshots, } go progress.ExecuteTask(fmt.Sprintf("Generating markers for %s", scene.Path), func() { task.Start(&wg) diff --git a/pkg/manager/paths/paths_scene_markers.go b/pkg/manager/paths/paths_scene_markers.go index e792c649f..3d9dbd6a6 100644 --- a/pkg/manager/paths/paths_scene_markers.go +++ b/pkg/manager/paths/paths_scene_markers.go @@ -22,3 +22,7 @@ func (sp *sceneMarkerPaths) GetStreamPath(checksum string, seconds int) string { func (sp *sceneMarkerPaths) GetStreamPreviewImagePath(checksum string, seconds int) string { return filepath.Join(sp.generated.Markers, checksum, strconv.Itoa(seconds)+".webp") } + +func (sp *sceneMarkerPaths) GetStreamScreenshotPath(checksum string, seconds int) string { + return filepath.Join(sp.generated.Markers, checksum, strconv.Itoa(seconds)+".jpg") +} diff --git a/pkg/manager/scene.go b/pkg/manager/scene.go index e262dda18..58ce2e65d 100644 --- a/pkg/manager/scene.go +++ b/pkg/manager/scene.go @@ -148,6 +148,7 @@ func DeleteGeneratedSceneFiles(scene *models.Scene, fileNamingAlgo models.HashAl func DeleteSceneMarkerFiles(scene *models.Scene, seconds int, fileNamingAlgo models.HashAlgorithm) { videoPath := GetInstance().Paths.SceneMarkers.GetStreamPath(scene.GetHash(fileNamingAlgo), seconds) imagePath := GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.GetHash(fileNamingAlgo), seconds) + screenshotPath := GetInstance().Paths.SceneMarkers.GetStreamScreenshotPath(scene.GetHash(fileNamingAlgo), seconds) exists, _ := utils.FileExists(videoPath) if exists { @@ -161,7 +162,15 @@ func DeleteSceneMarkerFiles(scene *models.Scene, seconds int, fileNamingAlgo mod if exists { err := os.Remove(imagePath) if err != nil { - logger.Warnf("Could not delete file %s: %s", videoPath, err.Error()) + logger.Warnf("Could not delete file %s: %s", imagePath, err.Error()) + } + } + + exists, _ = utils.FileExists(screenshotPath) + if exists { + err := os.Remove(screenshotPath) + if err != nil { + logger.Warnf("Could not delete file %s: %s", screenshotPath, err.Error()) } } } diff --git a/pkg/manager/task_generate_markers.go b/pkg/manager/task_generate_markers.go index 0e11fb853..5c432bf3a 100644 --- a/pkg/manager/task_generate_markers.go +++ b/pkg/manager/task_generate_markers.go @@ -19,6 +19,9 @@ type GenerateMarkersTask struct { Marker *models.SceneMarker Overwrite bool fileNamingAlgorithm models.HashAlgorithm + + ImagePreview bool + Screenshot bool } func (t *GenerateMarkersTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { @@ -94,7 +97,8 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene seconds := int(sceneMarker.Seconds) videoExists := t.videoExists(sceneHash, seconds) - imageExists := t.imageExists(sceneHash, seconds) + imageExists := !t.ImagePreview || t.imageExists(sceneHash, seconds) + screenshotExists := !t.Screenshot || t.screenshotExists(sceneHash, seconds) baseFilename := strconv.Itoa(seconds) @@ -119,7 +123,7 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene } } - if t.Overwrite || !imageExists { + if t.ImagePreview && (t.Overwrite || !imageExists) { imageFilename := baseFilename + ".webp" imagePath := instance.Paths.SceneMarkers.GetStreamPreviewImagePath(sceneHash, seconds) @@ -131,6 +135,24 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene logger.Debug("created marker image: ", imagePath) } } + + if t.Screenshot && (t.Overwrite || !screenshotExists) { + screenshotFilename := baseFilename + ".jpg" + screenshotPath := instance.Paths.SceneMarkers.GetStreamScreenshotPath(sceneHash, seconds) + + screenshotOptions := ffmpeg.ScreenshotOptions{ + OutputPath: instance.Paths.Generated.GetTmpPath(screenshotFilename), // tmp output in case the process ends abruptly + Quality: 2, + Width: videoFile.Width, + Time: float64(seconds), + } + if err := encoder.Screenshot(*videoFile, screenshotOptions); err != nil { + logger.Errorf("[generator] failed to generate marker screenshot: %s", err) + } else { + _ = utils.SafeMove(screenshotOptions.OutputPath, screenshotPath) + logger.Debug("created marker screenshot: ", screenshotPath) + } + } } func (t *GenerateMarkersTask) isMarkerNeeded() int { @@ -166,12 +188,11 @@ func (t *GenerateMarkersTask) markerExists(sceneChecksum string, seconds int) bo return false } - videoPath := instance.Paths.SceneMarkers.GetStreamPath(sceneChecksum, seconds) - imagePath := instance.Paths.SceneMarkers.GetStreamPreviewImagePath(sceneChecksum, seconds) - videoExists, _ := utils.FileExists(videoPath) - imageExists, _ := utils.FileExists(imagePath) + videoExists := t.videoExists(sceneChecksum, seconds) + imageExists := !t.ImagePreview || t.imageExists(sceneChecksum, seconds) + screenshotExists := !t.Screenshot || t.screenshotExists(sceneChecksum, seconds) - return videoExists && imageExists + return videoExists && imageExists && screenshotExists } func (t *GenerateMarkersTask) videoExists(sceneChecksum string, seconds int) bool { @@ -195,3 +216,14 @@ func (t *GenerateMarkersTask) imageExists(sceneChecksum string, seconds int) boo return imageExists } + +func (t *GenerateMarkersTask) screenshotExists(sceneChecksum string, seconds int) bool { + if sceneChecksum == "" { + return false + } + + screenshotPath := instance.Paths.SceneMarkers.GetStreamScreenshotPath(sceneChecksum, seconds) + screenshotExists, _ := utils.FileExists(screenshotPath) + + return screenshotExists +} diff --git a/ui/v2.5/src/components/Changelog/versions/v0100.md b/ui/v2.5/src/components/Changelog/versions/v0100.md index 2381b4e12..d49dd5d05 100644 --- a/ui/v2.5/src/components/Changelog/versions/v0100.md +++ b/ui/v2.5/src/components/Changelog/versions/v0100.md @@ -1,4 +1,5 @@ ### ✨ New Features +* Added options to generate webp and static preview files for markers. ([#1604](https://github.com/stashapp/stash/pull/1604)) * Added sort by option for gallery rating. ([#1720](https://github.com/stashapp/stash/pull/1720)) * Added support for querying scene scrapers using keywords. ([#1712](https://github.com/stashapp/stash/pull/1712)) * Added support for Studio aliases. ([#1660](https://github.com/stashapp/stash/pull/1660)) diff --git a/ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx b/ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx index 776ea7992..4b7ddc547 100644 --- a/ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx +++ b/ui/v2.5/src/components/Scenes/SceneGenerateDialog.tsx @@ -40,6 +40,8 @@ export const SceneGenerateDialog: React.FC = ( const [previewPreset, setPreviewPreset] = useState( GQL.PreviewPreset.Slow ); + const [markerImagePreviews, setMarkerImagePreviews] = useState(false); + const [markerScreenshots, setMarkerScreenshots] = useState(false); const [previewOptionsOpen, setPreviewOptionsOpen] = useState(false); @@ -67,6 +69,8 @@ export const SceneGenerateDialog: React.FC = ( previews, imagePreviews: previews && imagePreviews, markers, + markerImagePreviews: markers && markerImagePreviews, + markerScreenshots: markers && markerScreenshots, transcodes, overwrite, sceneIDs: props.selectedIds, @@ -276,6 +280,31 @@ export const SceneGenerateDialog: React.FC = ( label={intl.formatMessage({ id: "dialogs.scene_gen.markers" })} onChange={() => setMarkers(!markers)} /> +
+
↳
+ + setMarkerImagePreviews(!markerImagePreviews)} + className="ml-2 flex-grow" + /> + setMarkerScreenshots(!markerScreenshots)} + className="ml-2 flex-grow" + /> + +
{ const [markers, setMarkers] = useState(true); const [transcodes, setTranscodes] = useState(false); const [imagePreviews, setImagePreviews] = useState(false); + const [markerImagePreviews, setMarkerImagePreviews] = useState(false); + const [markerScreenshots, setMarkerScreenshots] = useState(false); async function onGenerate() { try { @@ -22,6 +24,8 @@ export const GenerateButton: React.FC = () => { previews, imagePreviews: previews && imagePreviews, markers, + markerImagePreviews: markers && markerImagePreviews, + markerScreenshots: markers && markerScreenshots, transcodes, }); Toast.success({ @@ -68,6 +72,31 @@ export const GenerateButton: React.FC = () => { label={intl.formatMessage({ id: "dialogs.scene_gen.markers" })} onChange={() => setMarkers(!markers)} /> +
+
↳
+ + setMarkerImagePreviews(!markerImagePreviews)} + className="ml-2 flex-grow" + /> + setMarkerScreenshots(!markerScreenshots)} + className="ml-2 flex-grow" + /> + +
= (props: IWallItemProps) => { ? { video: props.sceneMarker.stream, animation: props.sceneMarker.preview, + image: props.sceneMarker.screenshot, } : props.scene ? { diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 64d57468c..7c0ece1e9 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -440,6 +440,8 @@ "scene_gen": { "image_previews": "Image Previews (animated WebP previews, only required if Preview Type is set to Animated Image)", "markers": "Markers (20 second videos which begin at the given timecode)", + "marker_image_previews": "Marker Previews (animated WebP previews, only required if Preview Type is set to Animated Image)", + "marker_screenshots": "Marker Screenshots (static JPG image, only required if Preview Type is set to Static Image)", "overwrite": "Overwrite existing generated files", "phash": "Perceptual hashes (for deduplication)", "preview_exclude_end_time_desc": "Exclude the last x seconds from scene previews. This can be a value in seconds, or a percentage (eg 2%) of the total scene duration.",