Merge 0e32f33681 into 061d21dede

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/disintegration/imaging v1.6.2
 	github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d
 	github.com/doug-martin/goqu/v9 v9.18.0
+	github.com/fsnotify/fsnotify v1.9.0
 	github.com/go-chi/chi/v5 v5.2.2
 	github.com/go-chi/cors v1.2.1
 	github.com/go-chi/httplog v0.3.1
@@ -77,7 +78,6 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect
-	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect

graphql/schema/types/config.graphql

@@ -143,6 +143,8 @@ input ConfigGeneralInput {
   writeImageThumbnails: Boolean
   "Create Image Clips from Video extensions when Videos are disabled in Library"
   createImageClipsFromVideos: Boolean
+  "Watch library directories for changes"
+  autoScanWatch: Boolean
   "Username"
   username: String
   "Password"
@@ -269,6 +271,8 @@ type ConfigGeneralResult {
   writeImageThumbnails: Boolean!
   "Create Image Clips from Video extensions when Videos are disabled in Library"
   createImageClipsFromVideos: Boolean!
+  "Watch library directories for changes"
+  autoScanWatch: Boolean!
   "API Key"
   apiKey: String!
   "Username"

internal/api/resolver_mutation_configure.go

@@ -299,6 +299,8 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen
 	r.setConfigBool(config.WriteImageThumbnails, input.WriteImageThumbnails)
 	r.setConfigBool(config.CreateImageClipsFromVideos, input.CreateImageClipsFromVideos)
 
+	r.setConfigBool(config.AutoScanWatch, input.AutoScanWatch)
+
 	if input.GalleryCoverRegex != nil {
 		_, err := regexp.Compile(*input.GalleryCoverRegex)
 		if err != nil {
@@ -451,6 +453,7 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen
 	if refreshPluginSource {
 		manager.GetInstance().RefreshPluginSourceManager()
 	}
+	manager.GetInstance().RefreshFileWatcher()
 
 	return makeConfigGeneralResult(), nil
 }

internal/api/resolver_query_configuration.go

@@ -107,6 +107,7 @@ func makeConfigGeneralResult() *ConfigGeneralResult {
 		MaxStreamingTranscodeSize:     &maxStreamingTranscodeSize,
 		WriteImageThumbnails:          config.IsWriteImageThumbnails(),
 		CreateImageClipsFromVideos:    config.IsCreateImageClipsFromVideos(),
+		AutoScanWatch:                 config.GetAutoScanWatch(),
 		GalleryCoverRegex:             config.GetGalleryCoverRegex(),
 		APIKey:                        config.GetAPIKey(),
 		Username:                      config.GetUsername(),

internal/manager/config/config.go

@@ -110,6 +110,10 @@ const (
 	CreateImageClipsFromVideos        = "create_image_clip_from_videos"
 	createImageClipsFromVideosDefault = false
 
+	// AutoScanWatch enables filesystem watcher to auto-trigger scans
+	AutoScanWatch        = "auto_scan_watch"
+	AutoScanWatchDefault = false
+
 	Host        = "host"
 	hostDefault = "0.0.0.0"
 
@@ -212,6 +216,7 @@ const (
 	ImageLightboxDisableAnimation           = "image_lightbox.disable_animation"
 
 	UI             = "ui"
+	UIScanSettings = "ui.taskDefaults.scan"
 
 	defaultImageLightboxSlideshowDelay = 5
 
@@ -1070,6 +1075,11 @@ func (i *Config) IsCreateImageClipsFromVideos() bool {
 	return i.getBool(CreateImageClipsFromVideos)
 }
 
+// GetAutoScanWatch returns whether filesystem watcher should be enabled.
+func (i *Config) GetAutoScanWatch() bool {
+	return i.getBool(AutoScanWatch)
+}
+
 func (i *Config) GetAPIKey() string {
 	return i.getString(ApiKey)
 }
@@ -1328,6 +1338,25 @@ func (i *Config) SetUIConfiguration(v map[string]interface{}) {
 	i.set(UI, v)
 }
 
+// GetUIScanSettings returns the UI Scan task settings.
+// Returns nil if the settings could not be unmarshalled, or if it
+// has not been set.
+func (i *Config) GetUIScanSettings() *ScanMetadataOptions {
+	i.RLock()
+	defer i.RUnlock()
+	v := i.forKey(UIScanSettings)
+
+	if v.Exists(UIScanSettings) && v.Get(UIScanSettings) != nil {
+		var ret ScanMetadataOptions
+		if err := v.Unmarshal(UIScanSettings, &ret); err != nil {
+			return nil
+		}
+		return &ret
+	}
+
+	return nil
+}
+
 func (i *Config) GetCSSPath() string {
 	// use custom.css in the same directory as the config file
 	configFileUsed := i.GetConfigFile()
@@ -1820,6 +1849,8 @@ func (i *Config) setDefaultValues() {
 	i.setDefault(WriteImageThumbnails, writeImageThumbnailsDefault)
 	i.setDefault(CreateImageClipsFromVideos, createImageClipsFromVideosDefault)
 
+	i.setDefault(AutoScanWatch, AutoScanWatchDefault)
+
 	i.setDefault(Database, defaultDatabaseFilePath)
 
 	i.setDefault(dangerousAllowPublicWithoutAuth, dangerousAllowPublicWithoutAuthDefault)

internal/manager/init.go

@@ -246,6 +246,7 @@ func (s *Manager) postInit(ctx context.Context) error {
 
 	s.RefreshFFMpeg(ctx)
 	s.RefreshStreamManager()
+	s.RefreshFileWatcher()
 
 	return nil
 }

internal/manager/manager.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sync"
 	"time"
 
 	"github.com/remeh/sizedwaitgroup"
@@ -69,6 +70,9 @@ type Manager struct {
 	GroupService   GroupService
 
 	scanSubs *subscriptionManager
+
+	fileWatcherCancel context.CancelFunc
+	fileWatcherMu     sync.Mutex
 }
 
 var instance *Manager

internal/manager/watcher.go

@@ -0,0 +1,261 @@
+package manager
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/stashapp/stash/pkg/file"
+	file_image "github.com/stashapp/stash/pkg/file/image"
+	"github.com/stashapp/stash/pkg/file/video"
+	"github.com/stashapp/stash/pkg/fsutil"
+	"github.com/stashapp/stash/pkg/job"
+	"github.com/stashapp/stash/pkg/logger"
+)
+
+const (
+	// How long to wait for write events to finish before scanning file
+	defaultWriteDebounce = 30
+	// The maximum number of tasks on a single scan
+	taskQueueSize = 200
+)
+
+// findAssociatedVideo attempts to map an event path an associated video.
+func findAssociatedVideo(eventPath string, s *Manager) *string {
+	dir := filepath.Dir(eventPath)
+
+	findWithExts := func(prefix string) *string {
+		for _, ve := range s.Config.GetVideoExtensions() {
+			cand := filepath.Join(dir, prefix+"."+ve)
+			if _, err := os.Stat(cand); err == nil {
+				return &cand
+			}
+		}
+		return nil
+	}
+
+	// funscript
+	if fsutil.MatchExtension(eventPath, []string{"funscript"}) {
+		prefix := strings.TrimSuffix(filepath.Base(eventPath), filepath.Ext(eventPath))
+		return findWithExts(prefix)
+	}
+
+	// captions (.srt, .vtt, etc.)
+	if fsutil.MatchExtension(eventPath, video.CaptionExts) {
+		prefix := strings.TrimSuffix(video.GetCaptionPrefix(eventPath), ".")
+		return findWithExts(prefix)
+	}
+
+	return nil
+}
+
+// clearWatcherCancelLock clears the stored file watcher cancel func.
+// Caller must hold s.fileWatcherMu.
+func (s *Manager) clearWatcherCancelLock() {
+	if s.fileWatcherCancel != nil {
+		s.fileWatcherCancel()
+		s.fileWatcherCancel = nil
+	}
+}
+
+// shouldScheduleScan determines whether the raw event path should trigger a
+// scan.
+func shouldScheduleScan(rawPath string, s *Manager) *string {
+	// If the event itself is a video/image/zip we scan it directly.
+	if useAsVideo(rawPath) || useAsImage(rawPath) || isZip(rawPath) {
+		return &rawPath
+	}
+
+	// Otherwise try to map captions/funscripts to an associated video.
+	return findAssociatedVideo(rawPath, s)
+}
+
+// makeScanner constructs a configured file.Scanner used by the watcher.
+func makeScanner(s *Manager) *file.Scanner {
+	return &file.Scanner{
+		Repository: file.NewRepository(s.Repository),
+		FileDecorators: []file.Decorator{
+			&file.FilteredDecorator{
+				Decorator: &video.Decorator{FFProbe: s.FFProbe},
+				Filter:    file.FilterFunc(videoFileFilter),
+			},
+			&file.FilteredDecorator{
+				Decorator: &file_image.Decorator{FFProbe: s.FFProbe},
+				Filter:    file.FilterFunc(imageFileFilter),
+			},
+		},
+		FingerprintCalculator: &fingerprintCalculator{s.Config},
+		FS:                    &file.OsFS{},
+	}
+}
+
+// runScan performs the scan job for the given path. It is invoked by
+// the debounce timers once the debounce period expires.
+func runScan(ctx context.Context, s *Manager, p string) {
+	// quick existence check - skip if file no longer exists
+	_, err := os.Stat(p)
+	if err != nil {
+		return
+	}
+
+	scanner := makeScanner(s)
+
+	// create and add job
+	j := job.MakeJobExec(func(ctx context.Context, progress *job.Progress) error {
+		tq := job.NewTaskQueue(ctx, progress, taskQueueSize, s.Config.GetParallelTasksWithAutoDetection())
+
+		// Build scan input using the ui scan settings from config (if present).
+		var scanInput ScanMetadataInput
+		if ds := s.Config.GetUIScanSettings(); ds != nil {
+			scanInput.ScanMetadataOptions = *ds
+		}
+
+		scanHandler := getScanHandlers(scanInput, tq, progress)
+		scanOptions := file.ScanOptions{
+			Paths:             []string{p},
+			ZipFileExtensions: s.Config.GetGalleryExtensions(),
+			ScanFilters:       []file.PathFilter{newScanFilter(s.Config, s.Repository, time.Time{})},
+			ParallelTasks:     s.Config.GetParallelTasksWithAutoDetection(),
+		}
+
+		scanner.Scan(ctx, scanHandler, scanOptions, progress)
+		tq.Close()
+		return nil
+	})
+
+	s.JobManager.Add(ctx, "FS change detected - scanning...", j)
+}
+
+// RefreshFileWatcher starts a filesystem watcher for configured stash paths.
+// It will schedule a single-file scan job when files are created/modified.
+func (s *Manager) RefreshFileWatcher() {
+	// restart/cancel existing watcher if present
+	s.fileWatcherMu.Lock()
+	s.clearWatcherCancelLock()
+
+	// if disabled in config, do nothing
+	if !s.Config.GetAutoScanWatch() {
+		s.fileWatcherMu.Unlock()
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	s.fileWatcherCancel = cancel
+	s.fileWatcherMu.Unlock()
+
+	// don't block postInit on watcher startup
+	go func() {
+		watcher, err := fsnotify.NewWatcher()
+		if err != nil {
+			logger.Errorf("could not start fsnotify watcher: %v", err)
+			// ensure we clear cancel so future restarts will try again
+			s.fileWatcherMu.Lock()
+			s.clearWatcherCancelLock()
+			s.fileWatcherMu.Unlock()
+			return
+		}
+
+		// ensure watcher is closed when context cancels
+		go func() {
+			<-ctx.Done()
+			watcher.Close()
+		}()
+
+		// add all stash dirs recursively
+		stashPaths := s.Config.GetStashPaths()
+		for _, st := range stashPaths {
+			if st == nil || st.Path == "" {
+				continue
+			}
+			// walk directories and add watches
+			_ = filepath.WalkDir(st.Path, func(path string, dEntry fs.DirEntry, err error) error {
+				if err != nil {
+					return nil
+				}
+				if dEntry.IsDir() {
+					_ = watcher.Add(path)
+				}
+				return nil
+			})
+		}
+
+		// debounce map to avoid repeated scans. Use timers so multiple rapid events
+		// reset the wait and only schedule a single scan when events quiet down.
+		debounce := defaultWriteDebounce * time.Second
+		var mu sync.Mutex
+		timers := make(map[string]*time.Timer)
+
+		// event loop
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case ev, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+
+				// interested in Write/Create/Rename
+				if ev.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Rename) == 0 {
+					continue
+				}
+
+				// schedule single-file scan job with debounce
+				rawPath := ev.Name
+
+				info, err := os.Stat(rawPath)
+				if err != nil {
+					continue
+				}
+
+				// Map events like captions/funscript to the corresponding video file
+				var pptr *string
+
+				isDir := info.IsDir()
+				if isDir {
+					pptr = &rawPath
+				} else {
+					pptr = shouldScheduleScan(rawPath, s)
+				}
+
+				if pptr == nil {
+					continue
+				}
+
+				// schedule actual scan to run after debounce period; bind effective path
+				p := *pptr
+
+				mu.Lock()
+				if t, ok := timers[p]; ok {
+					t.Stop()
+				}
+
+				// capture p for closure
+				timers[p] = time.AfterFunc(debounce, func() {
+					mu.Lock()
+					defer mu.Unlock()
+					delete(timers, p)
+
+					// add watches for newly created directories
+					if isDir {
+						_ = watcher.Add(rawPath)
+					}
+
+					runScan(ctx, s, p)
+				})
+				mu.Unlock()
+
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return
+				}
+				logger.Errorf("fsnotify error: %v", err)
+			}
+		}
+	}()
+}

pkg/file/video/caption.go

@@ -59,8 +59,8 @@ func IsLangInCaptions(lang string, ext string, captions []*models.VideoCaption)
 	return false
 }
 
-// getCaptionPrefix returns the prefix used to search for video files for the provided caption path
-func getCaptionPrefix(captionPath string) string {
+// GetCaptionPrefix returns the prefix used to search for video files for the provided caption path
+func GetCaptionPrefix(captionPath string) string {
 	basename := strings.TrimSuffix(captionPath, filepath.Ext(captionPath)) // caption filename without the extension
 
 	// a caption file can be something like scene_filename.srt or scene_filename.en.srt
@@ -94,7 +94,7 @@ type CaptionUpdater interface {
 func AssociateCaptions(ctx context.Context, captionPath string, txnMgr txn.Manager, fqb models.FileFinder, w CaptionUpdater) {
 	captionLang := getCaptionsLangFromPath(captionPath)
 
-	captionPrefix := getCaptionPrefix(captionPath)
+	captionPrefix := GetCaptionPrefix(captionPath)
 	if err := txn.WithTxn(ctx, txnMgr, func(ctx context.Context) error {
 		var err error
 		files, er := fqb.FindAllByPath(ctx, captionPrefix+"*", true)

pkg/file/video/caption_test.go

@@ -42,7 +42,7 @@ var testCases = []testCase{
 
 func TestGenerateCaptionCandidates(t *testing.T) {
 	for _, c := range testCases {
-		assert.Equal(t, c.expectedResult, getCaptionPrefix(c.captionPath))
+		assert.Equal(t, c.expectedResult, GetCaptionPrefix(c.captionPath))
 	}
 }
 

ui/v2.5/graphql/data/config.graphql

@@ -30,6 +30,7 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   maxStreamingTranscodeSize
   writeImageThumbnails
   createImageClipsFromVideos
+  autoScanWatch
   apiKey
   username
   password

ui/v2.5/src/components/Settings/SettingsLibraryPanel.tsx

@@ -34,7 +34,15 @@ export const SettingsLibraryPanel: React.FC = () => {
       <StashSetting
         value={general.stashes ?? []}
         onChange={(v) => saveGeneral({ stashes: v })}
+      >
+        <BooleanSetting
+          id="auto-watch-directories"
+          headingID="config.general.auto_watch.heading"
+          subHeadingID="config.general.auto_watch.description"
+          checked={general.autoScanWatch ?? false}
+          onChange={(v) => saveGeneral({ autoScanWatch: v })}
         />
+      </StashSetting>
 
       <SettingSection headingID="config.library.media_content_extensions">
         <StringSetting

ui/v2.5/src/components/Settings/StashConfiguration.tsx

@@ -1,5 +1,5 @@
 import { faEllipsisV } from "@fortawesome/free-solid-svg-icons";
-import React, { useState } from "react";
+import React, { useState, PropsWithChildren } from "react";
 import { Button, Form, Row, Col, Dropdown } from "react-bootstrap";
 import { FormattedMessage } from "react-intl";
 import { Icon } from "src/components/Shared/Icon";
@@ -193,7 +193,11 @@ interface IStashSetting {
   onChange: (v: GQL.StashConfigInput[]) => void;
 }
 
-export const StashSetting: React.FC<IStashSetting> = ({ value, onChange }) => {
+export const StashSetting: React.FC<PropsWithChildren<IStashSetting>> = ({
+  value,
+  onChange,
+  children,
+}) => {
   return (
     <SettingSection
       id="stashes"
@@ -201,6 +205,7 @@ export const StashSetting: React.FC<IStashSetting> = ({ value, onChange }) => {
       subHeadingID="config.general.directory_locations_to_your_content"
     >
       <StashConfiguration stashes={value} setStashes={(v) => onChange(v)} />
+      {children}
     </SettingSection>
   );
 };

ui/v2.5/src/locales/en-GB.json

@@ -432,7 +432,11 @@
       "sqlite_location": "File location for the SQLite database (requires restart). WARNING: storing the database on a different system to where the Stash server is run from (i.e. over the network) is unsupported!",
       "video_ext_desc": "Comma-delimited list of file extensions that will be identified as videos.",
       "video_ext_head": "Video Extensions",
-      "video_head": "Video"
+      "video_head": "Video",
+      "auto_watch": {
+        "heading": "Filesystem Watcher",
+        "description": "When enabled, watches your configured library paths and automatically schedules a scan when files change. The watcher uses your scan settings when scheduling these scans."
+      }
     },
     "library": {
       "exclusions": "Exclusions",