From 0e32f33681da398779b7a02d7918b1c590ae7afd Mon Sep 17 00:00:00 2001 From: Nodude <75137537+NodudeWasTaken@users.noreply.github.com> Date: Tue, 18 Nov 2025 20:32:21 +0100 Subject: [PATCH] Use fsnotify to watch filesystem for changes --- go.mod | 2 +- graphql/schema/types/config.graphql | 4 + internal/api/resolver_mutation_configure.go | 3 + internal/api/resolver_query_configuration.go | 1 + internal/manager/config/config.go | 33 ++- internal/manager/init.go | 1 + internal/manager/manager.go | 4 + internal/manager/watcher.go | 261 ++++++++++++++++++ pkg/file/video/caption.go | 6 +- pkg/file/video/caption_test.go | 2 +- ui/v2.5/graphql/data/config.graphql | 1 + .../Settings/SettingsLibraryPanel.tsx | 10 +- .../Settings/StashConfiguration.tsx | 9 +- ui/v2.5/src/locales/en-GB.json | 6 +- 14 files changed, 333 insertions(+), 10 deletions(-) create mode 100644 internal/manager/watcher.go diff --git a/go.mod b/go.mod index 4d6b78dc6..0d337edd2 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d github.com/doug-martin/goqu/v9 v9.18.0 + github.com/fsnotify/fsnotify v1.9.0 github.com/go-chi/chi/v5 v5.2.2 github.com/go-chi/cors v1.2.1 github.com/go-chi/httplog v0.3.1 @@ -77,7 +78,6 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 6a1ac72be..37348ee78 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -141,6 +141,8 @@ input ConfigGeneralInput { writeImageThumbnails: Boolean "Create Image Clips from Video extensions when Videos are disabled in Library" createImageClipsFromVideos: Boolean + "Watch library directories for changes" + autoScanWatch: Boolean "Username" username: String "Password" @@ -265,6 +267,8 @@ type ConfigGeneralResult { writeImageThumbnails: Boolean! "Create Image Clips from Video extensions when Videos are disabled in Library" createImageClipsFromVideos: Boolean! + "Watch library directories for changes" + autoScanWatch: Boolean! "API Key" apiKey: String! "Username" diff --git a/internal/api/resolver_mutation_configure.go b/internal/api/resolver_mutation_configure.go index 3299c01a8..f1405e400 100644 --- a/internal/api/resolver_mutation_configure.go +++ b/internal/api/resolver_mutation_configure.go @@ -290,6 +290,8 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen r.setConfigBool(config.WriteImageThumbnails, input.WriteImageThumbnails) r.setConfigBool(config.CreateImageClipsFromVideos, input.CreateImageClipsFromVideos) + r.setConfigBool(config.AutoScanWatch, input.AutoScanWatch) + if input.GalleryCoverRegex != nil { _, err := regexp.Compile(*input.GalleryCoverRegex) if err != nil { @@ -442,6 +444,7 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen if refreshPluginSource { manager.GetInstance().RefreshPluginSourceManager() } + manager.GetInstance().RefreshFileWatcher() return makeConfigGeneralResult(), nil } diff --git a/internal/api/resolver_query_configuration.go b/internal/api/resolver_query_configuration.go index 7213f8447..9764162aa 100644 --- a/internal/api/resolver_query_configuration.go +++ b/internal/api/resolver_query_configuration.go @@ -106,6 +106,7 @@ func makeConfigGeneralResult() *ConfigGeneralResult { MaxStreamingTranscodeSize: &maxStreamingTranscodeSize, WriteImageThumbnails: config.IsWriteImageThumbnails(), CreateImageClipsFromVideos: config.IsCreateImageClipsFromVideos(), + AutoScanWatch: config.GetAutoScanWatch(), GalleryCoverRegex: config.GetGalleryCoverRegex(), APIKey: config.GetAPIKey(), Username: config.GetUsername(), diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index eda863663..37e8feb19 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -110,6 +110,10 @@ const ( CreateImageClipsFromVideos = "create_image_clip_from_videos" createImageClipsFromVideosDefault = false + // AutoScanWatch enables filesystem watcher to auto-trigger scans + AutoScanWatch = "auto_scan_watch" + AutoScanWatchDefault = false + Host = "host" hostDefault = "0.0.0.0" @@ -210,7 +214,8 @@ const ( ImageLightboxScrollModeKey = "image_lightbox.scroll_mode" ImageLightboxScrollAttemptsBeforeChange = "image_lightbox.scroll_attempts_before_change" - UI = "ui" + UI = "ui" + UIScanSettings = "ui.taskDefaults.scan" defaultImageLightboxSlideshowDelay = 5 @@ -1066,6 +1071,11 @@ func (i *Config) IsCreateImageClipsFromVideos() bool { return i.getBool(CreateImageClipsFromVideos) } +// GetAutoScanWatch returns whether filesystem watcher should be enabled. +func (i *Config) GetAutoScanWatch() bool { + return i.getBool(AutoScanWatch) +} + func (i *Config) GetAPIKey() string { return i.getString(ApiKey) } @@ -1320,6 +1330,25 @@ func (i *Config) SetUIConfiguration(v map[string]interface{}) { i.set(UI, v) } +// GetUIScanSettings returns the UI Scan task settings. +// Returns nil if the settings could not be unmarshalled, or if it +// has not been set. +func (i *Config) GetUIScanSettings() *ScanMetadataOptions { + i.RLock() + defer i.RUnlock() + v := i.forKey(UIScanSettings) + + if v.Exists(UIScanSettings) && v.Get(UIScanSettings) != nil { + var ret ScanMetadataOptions + if err := v.Unmarshal(UIScanSettings, &ret); err != nil { + return nil + } + return &ret + } + + return nil +} + func (i *Config) GetCSSPath() string { // use custom.css in the same directory as the config file configFileUsed := i.GetConfigFile() @@ -1804,6 +1833,8 @@ func (i *Config) setDefaultValues() { i.setDefault(WriteImageThumbnails, writeImageThumbnailsDefault) i.setDefault(CreateImageClipsFromVideos, createImageClipsFromVideosDefault) + i.setDefault(AutoScanWatch, AutoScanWatchDefault) + i.setDefault(Database, defaultDatabaseFilePath) i.setDefault(dangerousAllowPublicWithoutAuth, dangerousAllowPublicWithoutAuthDefault) diff --git a/internal/manager/init.go b/internal/manager/init.go index dd1640ed3..7a404eb12 100644 --- a/internal/manager/init.go +++ b/internal/manager/init.go @@ -246,6 +246,7 @@ func (s *Manager) postInit(ctx context.Context) error { s.RefreshFFMpeg(ctx) s.RefreshStreamManager() + s.RefreshFileWatcher() return nil } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 2d47fd907..05c4f8266 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "runtime" + "sync" "time" "github.com/remeh/sizedwaitgroup" @@ -69,6 +70,9 @@ type Manager struct { GroupService GroupService scanSubs *subscriptionManager + + fileWatcherCancel context.CancelFunc + fileWatcherMu sync.Mutex } var instance *Manager diff --git a/internal/manager/watcher.go b/internal/manager/watcher.go new file mode 100644 index 000000000..272b9b4be --- /dev/null +++ b/internal/manager/watcher.go @@ -0,0 +1,261 @@ +package manager + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/stashapp/stash/pkg/file" + file_image "github.com/stashapp/stash/pkg/file/image" + "github.com/stashapp/stash/pkg/file/video" + "github.com/stashapp/stash/pkg/fsutil" + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" +) + +const ( + // How long to wait for write events to finish before scanning file + defaultWriteDebounce = 30 + // The maximum number of tasks on a single scan + taskQueueSize = 200 +) + +// findAssociatedVideo attempts to map an event path an associated video. +func findAssociatedVideo(eventPath string, s *Manager) *string { + dir := filepath.Dir(eventPath) + + findWithExts := func(prefix string) *string { + for _, ve := range s.Config.GetVideoExtensions() { + cand := filepath.Join(dir, prefix+"."+ve) + if _, err := os.Stat(cand); err == nil { + return &cand + } + } + return nil + } + + // funscript + if fsutil.MatchExtension(eventPath, []string{"funscript"}) { + prefix := strings.TrimSuffix(filepath.Base(eventPath), filepath.Ext(eventPath)) + return findWithExts(prefix) + } + + // captions (.srt, .vtt, etc.) + if fsutil.MatchExtension(eventPath, video.CaptionExts) { + prefix := strings.TrimSuffix(video.GetCaptionPrefix(eventPath), ".") + return findWithExts(prefix) + } + + return nil +} + +// clearWatcherCancelLock clears the stored file watcher cancel func. +// Caller must hold s.fileWatcherMu. +func (s *Manager) clearWatcherCancelLock() { + if s.fileWatcherCancel != nil { + s.fileWatcherCancel() + s.fileWatcherCancel = nil + } +} + +// shouldScheduleScan determines whether the raw event path should trigger a +// scan. +func shouldScheduleScan(rawPath string, s *Manager) *string { + // If the event itself is a video/image/zip we scan it directly. + if useAsVideo(rawPath) || useAsImage(rawPath) || isZip(rawPath) { + return &rawPath + } + + // Otherwise try to map captions/funscripts to an associated video. + return findAssociatedVideo(rawPath, s) +} + +// makeScanner constructs a configured file.Scanner used by the watcher. +func makeScanner(s *Manager) *file.Scanner { + return &file.Scanner{ + Repository: file.NewRepository(s.Repository), + FileDecorators: []file.Decorator{ + &file.FilteredDecorator{ + Decorator: &video.Decorator{FFProbe: s.FFProbe}, + Filter: file.FilterFunc(videoFileFilter), + }, + &file.FilteredDecorator{ + Decorator: &file_image.Decorator{FFProbe: s.FFProbe}, + Filter: file.FilterFunc(imageFileFilter), + }, + }, + FingerprintCalculator: &fingerprintCalculator{s.Config}, + FS: &file.OsFS{}, + } +} + +// runScan performs the scan job for the given path. It is invoked by +// the debounce timers once the debounce period expires. +func runScan(ctx context.Context, s *Manager, p string) { + // quick existence check - skip if file no longer exists + _, err := os.Stat(p) + if err != nil { + return + } + + scanner := makeScanner(s) + + // create and add job + j := job.MakeJobExec(func(ctx context.Context, progress *job.Progress) error { + tq := job.NewTaskQueue(ctx, progress, taskQueueSize, s.Config.GetParallelTasksWithAutoDetection()) + + // Build scan input using the ui scan settings from config (if present). + var scanInput ScanMetadataInput + if ds := s.Config.GetUIScanSettings(); ds != nil { + scanInput.ScanMetadataOptions = *ds + } + + scanHandler := getScanHandlers(scanInput, tq, progress) + scanOptions := file.ScanOptions{ + Paths: []string{p}, + ZipFileExtensions: s.Config.GetGalleryExtensions(), + ScanFilters: []file.PathFilter{newScanFilter(s.Config, s.Repository, time.Time{})}, + ParallelTasks: s.Config.GetParallelTasksWithAutoDetection(), + } + + scanner.Scan(ctx, scanHandler, scanOptions, progress) + tq.Close() + return nil + }) + + s.JobManager.Add(ctx, "FS change detected - scanning...", j) +} + +// RefreshFileWatcher starts a filesystem watcher for configured stash paths. +// It will schedule a single-file scan job when files are created/modified. +func (s *Manager) RefreshFileWatcher() { + // restart/cancel existing watcher if present + s.fileWatcherMu.Lock() + s.clearWatcherCancelLock() + + // if disabled in config, do nothing + if !s.Config.GetAutoScanWatch() { + s.fileWatcherMu.Unlock() + return + } + + ctx, cancel := context.WithCancel(context.Background()) + s.fileWatcherCancel = cancel + s.fileWatcherMu.Unlock() + + // don't block postInit on watcher startup + go func() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + logger.Errorf("could not start fsnotify watcher: %v", err) + // ensure we clear cancel so future restarts will try again + s.fileWatcherMu.Lock() + s.clearWatcherCancelLock() + s.fileWatcherMu.Unlock() + return + } + + // ensure watcher is closed when context cancels + go func() { + <-ctx.Done() + watcher.Close() + }() + + // add all stash dirs recursively + stashPaths := s.Config.GetStashPaths() + for _, st := range stashPaths { + if st == nil || st.Path == "" { + continue + } + // walk directories and add watches + _ = filepath.WalkDir(st.Path, func(path string, dEntry fs.DirEntry, err error) error { + if err != nil { + return nil + } + if dEntry.IsDir() { + _ = watcher.Add(path) + } + return nil + }) + } + + // debounce map to avoid repeated scans. Use timers so multiple rapid events + // reset the wait and only schedule a single scan when events quiet down. + debounce := defaultWriteDebounce * time.Second + var mu sync.Mutex + timers := make(map[string]*time.Timer) + + // event loop + for { + select { + case <-ctx.Done(): + return + case ev, ok := <-watcher.Events: + if !ok { + return + } + + // interested in Write/Create/Rename + if ev.Op&(fsnotify.Write|fsnotify.Create|fsnotify.Rename) == 0 { + continue + } + + // schedule single-file scan job with debounce + rawPath := ev.Name + + info, err := os.Stat(rawPath) + if err != nil { + continue + } + + // Map events like captions/funscript to the corresponding video file + var pptr *string + + isDir := info.IsDir() + if isDir { + pptr = &rawPath + } else { + pptr = shouldScheduleScan(rawPath, s) + } + + if pptr == nil { + continue + } + + // schedule actual scan to run after debounce period; bind effective path + p := *pptr + + mu.Lock() + if t, ok := timers[p]; ok { + t.Stop() + } + + // capture p for closure + timers[p] = time.AfterFunc(debounce, func() { + mu.Lock() + defer mu.Unlock() + delete(timers, p) + + // add watches for newly created directories + if isDir { + _ = watcher.Add(rawPath) + } + + runScan(ctx, s, p) + }) + mu.Unlock() + + case err, ok := <-watcher.Errors: + if !ok { + return + } + logger.Errorf("fsnotify error: %v", err) + } + } + }() +} diff --git a/pkg/file/video/caption.go b/pkg/file/video/caption.go index bec3db6fd..8f7fa3bcc 100644 --- a/pkg/file/video/caption.go +++ b/pkg/file/video/caption.go @@ -59,8 +59,8 @@ func IsLangInCaptions(lang string, ext string, captions []*models.VideoCaption) return false } -// getCaptionPrefix returns the prefix used to search for video files for the provided caption path -func getCaptionPrefix(captionPath string) string { +// GetCaptionPrefix returns the prefix used to search for video files for the provided caption path +func GetCaptionPrefix(captionPath string) string { basename := strings.TrimSuffix(captionPath, filepath.Ext(captionPath)) // caption filename without the extension // a caption file can be something like scene_filename.srt or scene_filename.en.srt @@ -94,7 +94,7 @@ type CaptionUpdater interface { func AssociateCaptions(ctx context.Context, captionPath string, txnMgr txn.Manager, fqb models.FileFinder, w CaptionUpdater) { captionLang := getCaptionsLangFromPath(captionPath) - captionPrefix := getCaptionPrefix(captionPath) + captionPrefix := GetCaptionPrefix(captionPath) if err := txn.WithTxn(ctx, txnMgr, func(ctx context.Context) error { var err error files, er := fqb.FindAllByPath(ctx, captionPrefix+"*") diff --git a/pkg/file/video/caption_test.go b/pkg/file/video/caption_test.go index 7c6f301da..91964d17c 100644 --- a/pkg/file/video/caption_test.go +++ b/pkg/file/video/caption_test.go @@ -42,7 +42,7 @@ var testCases = []testCase{ func TestGenerateCaptionCandidates(t *testing.T) { for _, c := range testCases { - assert.Equal(t, c.expectedResult, getCaptionPrefix(c.captionPath)) + assert.Equal(t, c.expectedResult, GetCaptionPrefix(c.captionPath)) } } diff --git a/ui/v2.5/graphql/data/config.graphql b/ui/v2.5/graphql/data/config.graphql index 192fb8053..08401e3d1 100644 --- a/ui/v2.5/graphql/data/config.graphql +++ b/ui/v2.5/graphql/data/config.graphql @@ -29,6 +29,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { maxStreamingTranscodeSize writeImageThumbnails createImageClipsFromVideos + autoScanWatch apiKey username password diff --git a/ui/v2.5/src/components/Settings/SettingsLibraryPanel.tsx b/ui/v2.5/src/components/Settings/SettingsLibraryPanel.tsx index 4a8e55a60..abf0cd107 100644 --- a/ui/v2.5/src/components/Settings/SettingsLibraryPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsLibraryPanel.tsx @@ -34,7 +34,15 @@ export const SettingsLibraryPanel: React.FC = () => { saveGeneral({ stashes: v })} - /> + > + saveGeneral({ autoScanWatch: v })} + /> + void; } -export const StashSetting: React.FC = ({ value, onChange }) => { +export const StashSetting: React.FC> = ({ + value, + onChange, + children, +}) => { return ( = ({ value, onChange }) => { subHeadingID="config.general.directory_locations_to_your_content" > onChange(v)} /> + {children} ); }; diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 6a230736a..ef11e8cbe 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -425,7 +425,11 @@ "sqlite_location": "File location for the SQLite database (requires restart). WARNING: storing the database on a different system to where the Stash server is run from (i.e. over the network) is unsupported!", "video_ext_desc": "Comma-delimited list of file extensions that will be identified as videos.", "video_ext_head": "Video Extensions", - "video_head": "Video" + "video_head": "Video", + "auto_watch": { + "heading": "Filesystem Watcher", + "description": "When enabled, watches your configured library paths and automatically schedules a scan when files change. The watcher uses your scan settings when scheduling these scans." + } }, "library": { "exclusions": "Exclusions",