Improve image scanning performance and thumbnail generation (#1655)

* Improve image scanning performance and thumbnail generation
* Add vips-tools to build image
* Add option to write generated thumbnails to disk
* Fallback to image if thumbnail generation fails

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

docker/build/x86_64/Dockerfile

@@ -36,7 +36,7 @@ RUN make build
 
 # Final Runnable Image
 FROM alpine:latest
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates vips-tools
 COPY --from=backend /stash/stash /ffmpeg/ffmpeg /ffmpeg/ffprobe /usr/bin/
 ENV STASH_CONFIG_FILE=/root/.stash/config.yml
 EXPOSE 9999

docker/ci/x86_64/Dockerfile

@@ -11,7 +11,7 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ];   then BIN=stash-pi; \
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-pip && pip3 install cloudscraper
 FROM ubuntu:20.04 as app
-RUN apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-mechanicalsoup ffmpeg && rm -rf /var/lib/apt/lists/*
+run apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-mechanicalsoup ffmpeg libvips-tools && rm -rf /var/lib/apt/lists/*
 COPY --from=prep /stash /usr/bin/
 COPY --from=prep /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages
 

graphql/documents/data/config.graphql

@@ -18,6 +18,7 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   previewPreset
   maxTranscodeSize
   maxStreamingTranscodeSize
+  writeImageThumbnails
   apiKey
   username
   password

graphql/schema/types/config.graphql

@@ -63,6 +63,8 @@ input ConfigGeneralInput {
   maxTranscodeSize: StreamingResolutionEnum
   """Max streaming transcode size"""
   maxStreamingTranscodeSize: StreamingResolutionEnum
+  """Write image thumbnails to disk when generating on the fly"""
+  writeImageThumbnails: Boolean
   """Username"""
   username: String
   """Password"""
@@ -136,6 +138,8 @@ type ConfigGeneralResult {
   maxTranscodeSize: StreamingResolutionEnum
   """Max streaming transcode size"""
   maxStreamingTranscodeSize: StreamingResolutionEnum
+  """Write image thumbnails to disk when generating on the fly"""
+  writeImageThumbnails: Boolean!
   """API Key"""
   apiKey: String!
   """Username"""

graphql/schema/types/metadata.graphql

@@ -47,6 +47,8 @@ input ScanMetadataInput {
   scanGenerateSprites: Boolean
   """Generate phashes during scan"""
   scanGeneratePhashes: Boolean
+  """Generate image thumbnails during scan"""
+  scanGenerateThumbnails: Boolean
 }
 
 input CleanMetadataInput {

pkg/api/resolver_mutation_configure.go

@@ -115,6 +115,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		c.Set(config.MaxStreamingTranscodeSize, input.MaxStreamingTranscodeSize.String())
 	}
 
+	if input.WriteImageThumbnails != nil {
+		c.Set(config.WriteImageThumbnails, *input.WriteImageThumbnails)
+	}
+
 	if input.Username != nil {
 		c.Set(config.Username, input.Username)
 	}

pkg/api/resolver_query_configuration.go

@@ -73,6 +73,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		PreviewPreset:                config.GetPreviewPreset(),
 		MaxTranscodeSize:             &maxTranscodeSize,
 		MaxStreamingTranscodeSize:    &maxStreamingTranscodeSize,
+		WriteImageThumbnails:         config.IsWriteImageThumbnails(),
 		APIKey:                       config.GetAPIKey(),
 		Username:                     config.GetUsername(),
 		Password:                     config.GetPasswordHash(),

pkg/api/routes_image.go

@@ -33,15 +33,33 @@ func (rs imageRoutes) Routes() chi.Router {
 // region Handlers
 
 func (rs imageRoutes) Thumbnail(w http.ResponseWriter, r *http.Request) {
-	image := r.Context().Value(imageKey).(*models.Image)
-	filepath := manager.GetInstance().Paths.Generated.GetThumbnailPath(image.Checksum, models.DefaultGthumbWidth)
+	img := r.Context().Value(imageKey).(*models.Image)
+	filepath := manager.GetInstance().Paths.Generated.GetThumbnailPath(img.Checksum, models.DefaultGthumbWidth)
 
-	// if the thumbnail doesn't exist, fall back to the original file
+	w.Header().Add("Cache-Control", "max-age=604800000")
+
+	// if the thumbnail doesn't exist, encode on the fly
 	exists, _ := utils.FileExists(filepath)
 	if exists {
 		http.ServeFile(w, r, filepath)
 	} else {
+		encoder := image.NewThumbnailEncoder(manager.GetInstance().FFMPEGPath)
+		data, err := encoder.GetThumbnail(img, models.DefaultGthumbWidth)
+		if err != nil {
+			logger.Errorf("error generating thumbnail for image: %s", err.Error())
+
+			// backwards compatibility - fallback to original image instead
 			rs.Image(w, r)
+			return
+		}
+
+		// write the generated thumbnail to disk if enabled
+		if manager.GetInstance().Config.IsWriteImageThumbnails() {
+			if err := utils.WriteFile(filepath, data); err != nil {
+				logger.Errorf("error writing thumbnail for image %s: %s", img.Path, err)
+			}
+		}
+		w.Write(data)
 	}
 }
 

pkg/image/image.go

@@ -35,6 +35,18 @@ func GetSourceImage(i *models.Image) (image.Image, error) {
 	return srcImage, nil
 }
 
+func DecodeSourceImage(i *models.Image) (*image.Config, *string, error) {
+	f, err := openSourceImage(i.Path)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer f.Close()
+
+	config, format, err := image.DecodeConfig(f)
+
+	return &config, &format, err
+}
+
 func CalculateMD5(path string) (string, error) {
 	f, err := openSourceImage(path)
 	if err != nil {
@@ -154,15 +166,15 @@ func SetFileDetails(i *models.Image) error {
 		return err
 	}
 
-	src, _ := GetSourceImage(i)
+	config, _, err := DecodeSourceImage(i)
 
-	if src != nil {
+	if err == nil {
 		i.Width = sql.NullInt64{
-			Int64: int64(src.Bounds().Max.X),
+			Int64: int64(config.Width),
 			Valid: true,
 		}
 		i.Height = sql.NullInt64{
-			Int64: int64(src.Bounds().Max.Y),
+			Int64: int64(config.Height),
 			Valid: true,
 		}
 	}

pkg/image/thumbnail.go

@@ -2,39 +2,126 @@ package image
 
 import (
 	"bytes"
-	"image"
-	"image/jpeg"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+	"sync"
 
-	"github.com/disintegration/imaging"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
 )
 
-func ThumbnailNeeded(srcImage image.Image, maxSize int) bool {
-	dim := srcImage.Bounds().Max
-	w := dim.X
-	h := dim.Y
+var vipsPath string
+var once sync.Once
 
-	return w > maxSize || h > maxSize
+type ThumbnailEncoder struct {
+	FFMPEGPath string
+	VipsPath   string
+}
+
+func GetVipsPath() string {
+	once.Do(func() {
+		vipsPath, _ = exec.LookPath("vips")
+	})
+	return vipsPath
+}
+
+func NewThumbnailEncoder(ffmpegPath string) ThumbnailEncoder {
+	return ThumbnailEncoder{
+		FFMPEGPath: ffmpegPath,
+		VipsPath:   GetVipsPath(),
+	}
 }
 
 // GetThumbnail returns the thumbnail image of the provided image resized to
 // the provided max size. It resizes based on the largest X/Y direction.
 // It returns nil and an error if an error occurs reading, decoding or encoding
 // the image.
-func GetThumbnail(srcImage image.Image, maxSize int) ([]byte, error) {
-	var resizedImage image.Image
-
-	// if height is longer then resize by height instead of width
-	dim := srcImage.Bounds().Max
-	if dim.Y > dim.X {
-		resizedImage = imaging.Resize(srcImage, 0, maxSize, imaging.Box)
-	} else {
-		resizedImage = imaging.Resize(srcImage, maxSize, 0, imaging.Box)
-	}
-
-	buf := new(bytes.Buffer)
-	err := jpeg.Encode(buf, resizedImage, nil)
+func (e *ThumbnailEncoder) GetThumbnail(img *models.Image, maxSize int) ([]byte, error) {
+	reader, err := openSourceImage(img.Path)
 	if err != nil {
 		return nil, err
 	}
+
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(reader)
+
+	_, format, err := DecodeSourceImage(img)
+	if err != nil {
+		return nil, err
+	}
+
+	if format != nil && *format == "gif" {
 		return buf.Bytes(), nil
+	}
+
+	if e.VipsPath != "" {
+		return e.getVipsThumbnail(buf, maxSize)
+	} else {
+		return e.getFFMPEGThumbnail(buf, format, maxSize, img.Path)
+	}
+}
+
+func (e *ThumbnailEncoder) getVipsThumbnail(image *bytes.Buffer, maxSize int) ([]byte, error) {
+	args := []string{
+		"thumbnail_source",
+		"[descriptor=0]",
+		".jpg[Q=70,strip]",
+		fmt.Sprint(maxSize),
+		"--size", "down",
+	}
+	data, err := e.run(e.VipsPath, args, image)
+
+	return []byte(data), err
+}
+
+func (e *ThumbnailEncoder) getFFMPEGThumbnail(image *bytes.Buffer, format *string, maxDimensions int, path string) ([]byte, error) {
+	// ffmpeg spends a long sniffing image format when data is piped through stdio, so we pass the format explicitly instead
+	ffmpegformat := ""
+	if format != nil && *format == "jpeg" {
+		ffmpegformat = "mjpeg"
+	} else if format != nil && *format == "png" {
+		ffmpegformat = "png_pipe"
+	} else if format != nil && *format == "webp" {
+		ffmpegformat = "webp_pipe"
+	} else {
+		return nil, errors.New("unsupported image format")
+	}
+
+	args := []string{
+		"-f", ffmpegformat,
+		"-i", "-",
+		"-vf", fmt.Sprintf("scale=%v:%v:force_original_aspect_ratio=decrease", maxDimensions, maxDimensions),
+		"-c:v", "mjpeg",
+		"-q:v", "5",
+		"-f", "image2pipe",
+		"-",
+	}
+	data, err := e.run(e.FFMPEGPath, args, image)
+
+	return []byte(data), err
+}
+
+func (e *ThumbnailEncoder) run(path string, args []string, stdin *bytes.Buffer) (string, error) {
+	cmd := exec.Command(path, args...)
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	cmd.Stdin = stdin
+
+	if err := cmd.Start(); err != nil {
+		return "", err
+	}
+
+	err := cmd.Wait()
+
+	if err != nil {
+		// error message should be in the stderr stream
+		logger.Errorf("image encoder error when running command <%s>: %s", strings.Join(cmd.Args, " "), stderr.String())
+		return stdout.String(), err
+	}
+
+	return stdout.String(), nil
 }

pkg/manager/config/config.go

@@ -84,6 +84,9 @@ const previewExcludeStartDefault = "0"
 const PreviewExcludeEnd = "preview_exclude_end"
 const previewExcludeEndDefault = "0"
 
+const WriteImageThumbnails = "write_image_thumbnails"
+const writeImageThumbnailsDefault = true
+
 const Host = "host"
 const Port = "port"
 const ExternalHost = "external_host"
@@ -595,6 +598,14 @@ func (i *Instance) GetMaxStreamingTranscodeSize() models.StreamingResolutionEnum
 	return models.StreamingResolutionEnum(ret)
 }
 
+// IsWriteImageThumbnails returns true if image thumbnails should be written
+// to disk after generating on the fly.
+func (i *Instance) IsWriteImageThumbnails() bool {
+	i.RLock()
+	defer i.RUnlock()
+	return viper.GetBool(WriteImageThumbnails)
+}
+
 func (i *Instance) GetAPIKey() string {
 	i.RLock()
 	defer i.RUnlock()
@@ -968,6 +979,8 @@ func (i *Instance) setDefaultValues() error {
 	viper.SetDefault(PreviewAudio, previewAudioDefault)
 	viper.SetDefault(SoundOnPreview, false)
 
+	viper.SetDefault(WriteImageThumbnails, writeImageThumbnailsDefault)
+
 	viper.SetDefault(Database, defaultDatabaseFilePath)
 
 	// Set generated to the metadata path for backwards compat

pkg/manager/task_scan.go

@@ -103,6 +103,7 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) {
 				GenerateImagePreview: utils.IsTrue(input.ScanGenerateImagePreviews),
 				GenerateSprite:       utils.IsTrue(input.ScanGenerateSprites),
 				GeneratePhash:        utils.IsTrue(input.ScanGeneratePhashes),
+				GenerateThumbnails:   utils.IsTrue(input.ScanGenerateThumbnails),
 				progress:             progress,
 				CaseSensitiveFs:      csFs,
 				ctx:                  ctx,
@@ -221,6 +222,7 @@ type ScanTask struct {
 	GeneratePhash        bool
 	GeneratePreview      bool
 	GenerateImagePreview bool
+	GenerateThumbnails   bool
 	zipGallery           *models.Gallery
 	progress             *job.Progress
 	CaseSensitiveFs      bool
@@ -1275,20 +1277,26 @@ func (t *ScanTask) associateImageWithFolderGallery(imageID int, qb models.Galler
 }
 
 func (t *ScanTask) generateThumbnail(i *models.Image) {
+	if !t.GenerateThumbnails {
+		return
+	}
+
 	thumbPath := GetInstance().Paths.Generated.GetThumbnailPath(i.Checksum, models.DefaultGthumbWidth)
 	exists, _ := utils.FileExists(thumbPath)
 	if exists {
 		return
 	}
 
-	srcImage, err := image.GetSourceImage(i)
+	config, _, err := image.DecodeSourceImage(i)
 	if err != nil {
 		logger.Errorf("error reading image %s: %s", i.Path, err.Error())
 		return
 	}
 
-	if image.ThumbnailNeeded(srcImage, models.DefaultGthumbWidth) {
-		data, err := image.GetThumbnail(srcImage, models.DefaultGthumbWidth)
+	if config.Height > models.DefaultGthumbWidth || config.Width > models.DefaultGthumbWidth {
+		encoder := image.NewThumbnailEncoder(instance.FFMPEGPath)
+		data, err := encoder.GetThumbnail(i, models.DefaultGthumbWidth)
+
 		if err != nil {
 			logger.Errorf("error getting thumbnail for image %s: %s", i.Path, err.Error())
 			return

ui/v2.5/src/components/Changelog/versions/v0100.md

@@ -13,6 +13,7 @@
 * Support filtering Movies by Performers. ([#1675](https://github.com/stashapp/stash/pull/1675))
 
 ### 🎨 Improvements
+* Optimised image thumbnail generation (optionally using `libvips`) and made optional. ([#1655](https://github.com/stashapp/stash/pull/1655))
 * Added missing image table indexes, resulting in a significant performance improvement. ([#1740](https://github.com/stashapp/stash/pull/1740))
 * Support scraper script logging to specific log levels. ([#1648](https://github.com/stashapp/stash/pull/1648))
 * Added sv-SE language option. ([#1691](https://github.com/stashapp/stash/pull/1691))

ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx

@@ -106,6 +106,7 @@ export const SettingsConfigurationPanel: React.FC = () => {
   const [maxStreamingTranscodeSize, setMaxStreamingTranscodeSize] = useState<
     GQL.StreamingResolutionEnum | undefined
   >(undefined);
+  const [writeImageThumbnails, setWriteImageThumbnails] = useState(true);
   const [username, setUsername] = useState<string | undefined>(undefined);
   const [password, setPassword] = useState<string | undefined>(undefined);
   const [maxSessionAge, setMaxSessionAge] = useState<number>(0);
@@ -157,6 +158,7 @@ export const SettingsConfigurationPanel: React.FC = () => {
     previewPreset: (previewPreset as GQL.PreviewPreset) ?? undefined,
     maxTranscodeSize,
     maxStreamingTranscodeSize,
+    writeImageThumbnails,
     username,
     password,
     maxSessionAge,
@@ -203,6 +205,7 @@ export const SettingsConfigurationPanel: React.FC = () => {
       setMaxStreamingTranscodeSize(
         conf.general.maxStreamingTranscodeSize ?? undefined
       );
+      setWriteImageThumbnails(conf.general.writeImageThumbnails);
       setUsername(conf.general.username);
       setPassword(conf.general.password);
       setMaxSessionAge(conf.general.maxSessionAge);
@@ -834,6 +837,30 @@ export const SettingsConfigurationPanel: React.FC = () => {
         </Form.Group>
       </Form.Group>
 
+      <hr />
+
+      <Form.Group>
+        <h4>{intl.formatMessage({ id: "images" })}</h4>
+
+        <Form.Group>
+          <Form.Check
+            id="write-image-thumbnails"
+            checked={writeImageThumbnails}
+            label={intl.formatMessage({
+              id: "config.ui.images.options.write_image_thumbnails.heading",
+            })}
+            onChange={() => setWriteImageThumbnails(!writeImageThumbnails)}
+          />
+          <Form.Text className="text-muted">
+            {intl.formatMessage({
+              id: "config.ui.images.options.write_image_thumbnails.description",
+            })}
+          </Form.Text>
+        </Form.Group>
+      </Form.Group>
+
+      <hr />
+
       <Form.Group>
         <h4>{intl.formatMessage({ id: "performers" })}</h4>
         <Form.Group>

ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx

@@ -46,6 +46,9 @@ export const SettingsTasksPanel: React.FC = () => {
   const [scanGeneratePhashes, setScanGeneratePhashes] = useState<boolean>(
     false
   );
+  const [scanGenerateThumbnails, setScanGenerateThumbnails] = useState<boolean>(
+    false
+  );
   const [cleanDryRun, setCleanDryRun] = useState<boolean>(false);
   const [
     scanGenerateImagePreviews,
@@ -161,6 +164,7 @@ export const SettingsTasksPanel: React.FC = () => {
         scanGenerateImagePreviews,
         scanGenerateSprites,
         scanGeneratePhashes,
+        scanGenerateThumbnails,
       });
       Toast.success({
         content: intl.formatMessage(
@@ -403,6 +407,14 @@ export const SettingsTasksPanel: React.FC = () => {
           })}
           onChange={() => setScanGeneratePhashes(!scanGeneratePhashes)}
         />
+        <Form.Check
+          id="scan-generate-thumbnails"
+          checked={scanGenerateThumbnails}
+          label={intl.formatMessage({
+            id: "config.tasks.generate_thumbnails_during_scan",
+          })}
+          onChange={() => setScanGenerateThumbnails(!scanGenerateThumbnails)}
+        />
       </Form.Group>
       <Form.Group>
         <Button

ui/v2.5/src/locales/en-GB.json

@@ -282,6 +282,7 @@
       "generate_previews_during_scan": "Generate image previews during scan (animated WebP previews, only required if Preview Type is set to Animated Image)",
       "generate_sprites_during_scan": "Generate sprites during scan (for the scene scrubber)",
       "generate_video_previews_during_scan": "Generate previews during scan (video previews which play when hovering over a scene)",
+      "generate_thumbnails_during_scan": "Generate thumbnails for images during scan.",
       "generated_content": "Generated Content",
       "import_from_exported_json": "Import from exported JSON in the metadata directory. Wipes the existing database.",
       "incremental_import": "Incremental import from a supplied export zip file.",
@@ -327,6 +328,14 @@
         "description": "Time offset in milliseconds for interactive scripts playback.",
         "heading": "Funscript Offset (ms)"
       },
+      "images": {
+        "options": {
+          "write_image_thumbnails": {
+            "heading": "Write image thumbnails",
+            "description": "Write image thumbnails to disk when generated on-the-fly"
+          }
+        }
+      },
       "language": {
         "heading": "Language"
       },