From 9cb1eccadbe9f4e2085e5cf7c816b4574ff265f5 Mon Sep 17 00:00:00 2001 From: InfiniteTF Date: Thu, 23 Sep 2021 07:22:14 +0200 Subject: [PATCH] Improve image scanning performance and thumbnail generation (#1655) * Improve image scanning performance and thumbnail generation * Add vips-tools to build image * Add option to write generated thumbnails to disk * Fallback to image if thumbnail generation fails Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com> --- docker/build/x86_64/Dockerfile | 2 +- docker/ci/x86_64/Dockerfile | 2 +- graphql/documents/data/config.graphql | 1 + graphql/schema/types/config.graphql | 4 + graphql/schema/types/metadata.graphql | 2 + pkg/api/resolver_mutation_configure.go | 4 + pkg/api/resolver_query_configuration.go | 1 + pkg/api/routes_image.go | 26 +++- pkg/image/image.go | 20 ++- pkg/image/thumbnail.go | 131 +++++++++++++++--- pkg/manager/config/config.go | 13 ++ pkg/manager/task_scan.go | 14 +- .../components/Changelog/versions/v0100.md | 1 + .../Settings/SettingsConfigurationPanel.tsx | 27 ++++ .../SettingsTasksPanel/SettingsTasksPanel.tsx | 12 ++ ui/v2.5/src/locales/en-GB.json | 9 ++ 16 files changed, 234 insertions(+), 35 deletions(-) diff --git a/docker/build/x86_64/Dockerfile b/docker/build/x86_64/Dockerfile index 0452c2b27..2ffcf7050 100644 --- a/docker/build/x86_64/Dockerfile +++ b/docker/build/x86_64/Dockerfile @@ -36,7 +36,7 @@ RUN make build # Final Runnable Image FROM alpine:latest -RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates vips-tools COPY --from=backend /stash/stash /ffmpeg/ffmpeg /ffmpeg/ffprobe /usr/bin/ ENV STASH_CONFIG_FILE=/root/.stash/config.yml EXPOSE 9999 diff --git a/docker/ci/x86_64/Dockerfile b/docker/ci/x86_64/Dockerfile index 53f3efeb2..7e2fd24fa 100644 --- a/docker/ci/x86_64/Dockerfile +++ b/docker/ci/x86_64/Dockerfile @@ -11,7 +11,7 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then BIN=stash-pi; \ ENV DEBIAN_FRONTEND=noninteractive RUN apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-pip && pip3 install cloudscraper FROM ubuntu:20.04 as app -RUN apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-mechanicalsoup ffmpeg && rm -rf /var/lib/apt/lists/* +run apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-mechanicalsoup ffmpeg libvips-tools && rm -rf /var/lib/apt/lists/* COPY --from=prep /stash /usr/bin/ COPY --from=prep /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index 7912bee00..f9151fff9 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -18,6 +18,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { previewPreset maxTranscodeSize maxStreamingTranscodeSize + writeImageThumbnails apiKey username password diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 63fd730c5..25c80a79c 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -63,6 +63,8 @@ input ConfigGeneralInput { maxTranscodeSize: StreamingResolutionEnum """Max streaming transcode size""" maxStreamingTranscodeSize: StreamingResolutionEnum + """Write image thumbnails to disk when generating on the fly""" + writeImageThumbnails: Boolean """Username""" username: String """Password""" @@ -136,6 +138,8 @@ type ConfigGeneralResult { maxTranscodeSize: StreamingResolutionEnum """Max streaming transcode size""" maxStreamingTranscodeSize: StreamingResolutionEnum + """Write image thumbnails to disk when generating on the fly""" + writeImageThumbnails: Boolean! """API Key""" apiKey: String! """Username""" diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 28f293fd8..9a57ea9fe 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -47,6 +47,8 @@ input ScanMetadataInput { scanGenerateSprites: Boolean """Generate phashes during scan""" scanGeneratePhashes: Boolean + """Generate image thumbnails during scan""" + scanGenerateThumbnails: Boolean } input CleanMetadataInput { diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index 83f4c3bd3..5ebfb20f7 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -115,6 +115,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co c.Set(config.MaxStreamingTranscodeSize, input.MaxStreamingTranscodeSize.String()) } + if input.WriteImageThumbnails != nil { + c.Set(config.WriteImageThumbnails, *input.WriteImageThumbnails) + } + if input.Username != nil { c.Set(config.Username, input.Username) } diff --git a/pkg/api/resolver_query_configuration.go b/pkg/api/resolver_query_configuration.go index 27aaac925..55d0a5c99 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/pkg/api/resolver_query_configuration.go @@ -73,6 +73,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult { PreviewPreset: config.GetPreviewPreset(), MaxTranscodeSize: &maxTranscodeSize, MaxStreamingTranscodeSize: &maxStreamingTranscodeSize, + WriteImageThumbnails: config.IsWriteImageThumbnails(), APIKey: config.GetAPIKey(), Username: config.GetUsername(), Password: config.GetPasswordHash(), diff --git a/pkg/api/routes_image.go b/pkg/api/routes_image.go index 59b7382b6..97c7c7524 100644 --- a/pkg/api/routes_image.go +++ b/pkg/api/routes_image.go @@ -33,15 +33,33 @@ func (rs imageRoutes) Routes() chi.Router { // region Handlers func (rs imageRoutes) Thumbnail(w http.ResponseWriter, r *http.Request) { - image := r.Context().Value(imageKey).(*models.Image) - filepath := manager.GetInstance().Paths.Generated.GetThumbnailPath(image.Checksum, models.DefaultGthumbWidth) + img := r.Context().Value(imageKey).(*models.Image) + filepath := manager.GetInstance().Paths.Generated.GetThumbnailPath(img.Checksum, models.DefaultGthumbWidth) - // if the thumbnail doesn't exist, fall back to the original file + w.Header().Add("Cache-Control", "max-age=604800000") + + // if the thumbnail doesn't exist, encode on the fly exists, _ := utils.FileExists(filepath) if exists { http.ServeFile(w, r, filepath) } else { - rs.Image(w, r) + encoder := image.NewThumbnailEncoder(manager.GetInstance().FFMPEGPath) + data, err := encoder.GetThumbnail(img, models.DefaultGthumbWidth) + if err != nil { + logger.Errorf("error generating thumbnail for image: %s", err.Error()) + + // backwards compatibility - fallback to original image instead + rs.Image(w, r) + return + } + + // write the generated thumbnail to disk if enabled + if manager.GetInstance().Config.IsWriteImageThumbnails() { + if err := utils.WriteFile(filepath, data); err != nil { + logger.Errorf("error writing thumbnail for image %s: %s", img.Path, err) + } + } + w.Write(data) } } diff --git a/pkg/image/image.go b/pkg/image/image.go index 64b766e6d..fc9cbe22f 100644 --- a/pkg/image/image.go +++ b/pkg/image/image.go @@ -35,6 +35,18 @@ func GetSourceImage(i *models.Image) (image.Image, error) { return srcImage, nil } +func DecodeSourceImage(i *models.Image) (*image.Config, *string, error) { + f, err := openSourceImage(i.Path) + if err != nil { + return nil, nil, err + } + defer f.Close() + + config, format, err := image.DecodeConfig(f) + + return &config, &format, err +} + func CalculateMD5(path string) (string, error) { f, err := openSourceImage(path) if err != nil { @@ -154,15 +166,15 @@ func SetFileDetails(i *models.Image) error { return err } - src, _ := GetSourceImage(i) + config, _, err := DecodeSourceImage(i) - if src != nil { + if err == nil { i.Width = sql.NullInt64{ - Int64: int64(src.Bounds().Max.X), + Int64: int64(config.Width), Valid: true, } i.Height = sql.NullInt64{ - Int64: int64(src.Bounds().Max.Y), + Int64: int64(config.Height), Valid: true, } } diff --git a/pkg/image/thumbnail.go b/pkg/image/thumbnail.go index 107b77143..69f047f02 100644 --- a/pkg/image/thumbnail.go +++ b/pkg/image/thumbnail.go @@ -2,39 +2,126 @@ package image import ( "bytes" - "image" - "image/jpeg" + "errors" + "fmt" + "os/exec" + "strings" + "sync" - "github.com/disintegration/imaging" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" ) -func ThumbnailNeeded(srcImage image.Image, maxSize int) bool { - dim := srcImage.Bounds().Max - w := dim.X - h := dim.Y +var vipsPath string +var once sync.Once - return w > maxSize || h > maxSize +type ThumbnailEncoder struct { + FFMPEGPath string + VipsPath string +} + +func GetVipsPath() string { + once.Do(func() { + vipsPath, _ = exec.LookPath("vips") + }) + return vipsPath +} + +func NewThumbnailEncoder(ffmpegPath string) ThumbnailEncoder { + return ThumbnailEncoder{ + FFMPEGPath: ffmpegPath, + VipsPath: GetVipsPath(), + } } // GetThumbnail returns the thumbnail image of the provided image resized to // the provided max size. It resizes based on the largest X/Y direction. // It returns nil and an error if an error occurs reading, decoding or encoding // the image. -func GetThumbnail(srcImage image.Image, maxSize int) ([]byte, error) { - var resizedImage image.Image - - // if height is longer then resize by height instead of width - dim := srcImage.Bounds().Max - if dim.Y > dim.X { - resizedImage = imaging.Resize(srcImage, 0, maxSize, imaging.Box) - } else { - resizedImage = imaging.Resize(srcImage, maxSize, 0, imaging.Box) - } - - buf := new(bytes.Buffer) - err := jpeg.Encode(buf, resizedImage, nil) +func (e *ThumbnailEncoder) GetThumbnail(img *models.Image, maxSize int) ([]byte, error) { + reader, err := openSourceImage(img.Path) if err != nil { return nil, err } - return buf.Bytes(), nil + + buf := new(bytes.Buffer) + buf.ReadFrom(reader) + + _, format, err := DecodeSourceImage(img) + if err != nil { + return nil, err + } + + if format != nil && *format == "gif" { + return buf.Bytes(), nil + } + + if e.VipsPath != "" { + return e.getVipsThumbnail(buf, maxSize) + } else { + return e.getFFMPEGThumbnail(buf, format, maxSize, img.Path) + } +} + +func (e *ThumbnailEncoder) getVipsThumbnail(image *bytes.Buffer, maxSize int) ([]byte, error) { + args := []string{ + "thumbnail_source", + "[descriptor=0]", + ".jpg[Q=70,strip]", + fmt.Sprint(maxSize), + "--size", "down", + } + data, err := e.run(e.VipsPath, args, image) + + return []byte(data), err +} + +func (e *ThumbnailEncoder) getFFMPEGThumbnail(image *bytes.Buffer, format *string, maxDimensions int, path string) ([]byte, error) { + // ffmpeg spends a long sniffing image format when data is piped through stdio, so we pass the format explicitly instead + ffmpegformat := "" + if format != nil && *format == "jpeg" { + ffmpegformat = "mjpeg" + } else if format != nil && *format == "png" { + ffmpegformat = "png_pipe" + } else if format != nil && *format == "webp" { + ffmpegformat = "webp_pipe" + } else { + return nil, errors.New("unsupported image format") + } + + args := []string{ + "-f", ffmpegformat, + "-i", "-", + "-vf", fmt.Sprintf("scale=%v:%v:force_original_aspect_ratio=decrease", maxDimensions, maxDimensions), + "-c:v", "mjpeg", + "-q:v", "5", + "-f", "image2pipe", + "-", + } + data, err := e.run(e.FFMPEGPath, args, image) + + return []byte(data), err +} + +func (e *ThumbnailEncoder) run(path string, args []string, stdin *bytes.Buffer) (string, error) { + cmd := exec.Command(path, args...) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.Stdin = stdin + + if err := cmd.Start(); err != nil { + return "", err + } + + err := cmd.Wait() + + if err != nil { + // error message should be in the stderr stream + logger.Errorf("image encoder error when running command <%s>: %s", strings.Join(cmd.Args, " "), stderr.String()) + return stdout.String(), err + } + + return stdout.String(), nil } diff --git a/pkg/manager/config/config.go b/pkg/manager/config/config.go index d17daf8b7..0629f8d87 100644 --- a/pkg/manager/config/config.go +++ b/pkg/manager/config/config.go @@ -84,6 +84,9 @@ const previewExcludeStartDefault = "0" const PreviewExcludeEnd = "preview_exclude_end" const previewExcludeEndDefault = "0" +const WriteImageThumbnails = "write_image_thumbnails" +const writeImageThumbnailsDefault = true + const Host = "host" const Port = "port" const ExternalHost = "external_host" @@ -595,6 +598,14 @@ func (i *Instance) GetMaxStreamingTranscodeSize() models.StreamingResolutionEnum return models.StreamingResolutionEnum(ret) } +// IsWriteImageThumbnails returns true if image thumbnails should be written +// to disk after generating on the fly. +func (i *Instance) IsWriteImageThumbnails() bool { + i.RLock() + defer i.RUnlock() + return viper.GetBool(WriteImageThumbnails) +} + func (i *Instance) GetAPIKey() string { i.RLock() defer i.RUnlock() @@ -968,6 +979,8 @@ func (i *Instance) setDefaultValues() error { viper.SetDefault(PreviewAudio, previewAudioDefault) viper.SetDefault(SoundOnPreview, false) + viper.SetDefault(WriteImageThumbnails, writeImageThumbnailsDefault) + viper.SetDefault(Database, defaultDatabaseFilePath) // Set generated to the metadata path for backwards compat diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index 47146847a..f76f14497 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -103,6 +103,7 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) { GenerateImagePreview: utils.IsTrue(input.ScanGenerateImagePreviews), GenerateSprite: utils.IsTrue(input.ScanGenerateSprites), GeneratePhash: utils.IsTrue(input.ScanGeneratePhashes), + GenerateThumbnails: utils.IsTrue(input.ScanGenerateThumbnails), progress: progress, CaseSensitiveFs: csFs, ctx: ctx, @@ -221,6 +222,7 @@ type ScanTask struct { GeneratePhash bool GeneratePreview bool GenerateImagePreview bool + GenerateThumbnails bool zipGallery *models.Gallery progress *job.Progress CaseSensitiveFs bool @@ -1275,20 +1277,26 @@ func (t *ScanTask) associateImageWithFolderGallery(imageID int, qb models.Galler } func (t *ScanTask) generateThumbnail(i *models.Image) { + if !t.GenerateThumbnails { + return + } + thumbPath := GetInstance().Paths.Generated.GetThumbnailPath(i.Checksum, models.DefaultGthumbWidth) exists, _ := utils.FileExists(thumbPath) if exists { return } - srcImage, err := image.GetSourceImage(i) + config, _, err := image.DecodeSourceImage(i) if err != nil { logger.Errorf("error reading image %s: %s", i.Path, err.Error()) return } - if image.ThumbnailNeeded(srcImage, models.DefaultGthumbWidth) { - data, err := image.GetThumbnail(srcImage, models.DefaultGthumbWidth) + if config.Height > models.DefaultGthumbWidth || config.Width > models.DefaultGthumbWidth { + encoder := image.NewThumbnailEncoder(instance.FFMPEGPath) + data, err := encoder.GetThumbnail(i, models.DefaultGthumbWidth) + if err != nil { logger.Errorf("error getting thumbnail for image %s: %s", i.Path, err.Error()) return diff --git a/ui/v2.5/src/components/Changelog/versions/v0100.md b/ui/v2.5/src/components/Changelog/versions/v0100.md index caab78f40..bc3612504 100644 --- a/ui/v2.5/src/components/Changelog/versions/v0100.md +++ b/ui/v2.5/src/components/Changelog/versions/v0100.md @@ -13,6 +13,7 @@ * Support filtering Movies by Performers. ([#1675](https://github.com/stashapp/stash/pull/1675)) ### 🎨 Improvements +* Optimised image thumbnail generation (optionally using `libvips`) and made optional. ([#1655](https://github.com/stashapp/stash/pull/1655)) * Added missing image table indexes, resulting in a significant performance improvement. ([#1740](https://github.com/stashapp/stash/pull/1740)) * Support scraper script logging to specific log levels. ([#1648](https://github.com/stashapp/stash/pull/1648)) * Added sv-SE language option. ([#1691](https://github.com/stashapp/stash/pull/1691)) diff --git a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx index 99c926a23..0b75c1766 100644 --- a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx @@ -106,6 +106,7 @@ export const SettingsConfigurationPanel: React.FC = () => { const [maxStreamingTranscodeSize, setMaxStreamingTranscodeSize] = useState< GQL.StreamingResolutionEnum | undefined >(undefined); + const [writeImageThumbnails, setWriteImageThumbnails] = useState(true); const [username, setUsername] = useState(undefined); const [password, setPassword] = useState(undefined); const [maxSessionAge, setMaxSessionAge] = useState(0); @@ -157,6 +158,7 @@ export const SettingsConfigurationPanel: React.FC = () => { previewPreset: (previewPreset as GQL.PreviewPreset) ?? undefined, maxTranscodeSize, maxStreamingTranscodeSize, + writeImageThumbnails, username, password, maxSessionAge, @@ -203,6 +205,7 @@ export const SettingsConfigurationPanel: React.FC = () => { setMaxStreamingTranscodeSize( conf.general.maxStreamingTranscodeSize ?? undefined ); + setWriteImageThumbnails(conf.general.writeImageThumbnails); setUsername(conf.general.username); setPassword(conf.general.password); setMaxSessionAge(conf.general.maxSessionAge); @@ -834,6 +837,30 @@ export const SettingsConfigurationPanel: React.FC = () => { +
+ + +

{intl.formatMessage({ id: "images" })}

+ + + setWriteImageThumbnails(!writeImageThumbnails)} + /> + + {intl.formatMessage({ + id: "config.ui.images.options.write_image_thumbnails.description", + })} + + + + +
+

{intl.formatMessage({ id: "performers" })}

diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx index b1c72d53f..8ae9a62ef 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx @@ -46,6 +46,9 @@ export const SettingsTasksPanel: React.FC = () => { const [scanGeneratePhashes, setScanGeneratePhashes] = useState( false ); + const [scanGenerateThumbnails, setScanGenerateThumbnails] = useState( + false + ); const [cleanDryRun, setCleanDryRun] = useState(false); const [ scanGenerateImagePreviews, @@ -161,6 +164,7 @@ export const SettingsTasksPanel: React.FC = () => { scanGenerateImagePreviews, scanGenerateSprites, scanGeneratePhashes, + scanGenerateThumbnails, }); Toast.success({ content: intl.formatMessage( @@ -403,6 +407,14 @@ export const SettingsTasksPanel: React.FC = () => { })} onChange={() => setScanGeneratePhashes(!scanGeneratePhashes)} /> + setScanGenerateThumbnails(!scanGenerateThumbnails)} + />