From 152bdb22cdf7b37ffd635715902cde621f4d297d Mon Sep 17 00:00:00 2001 From: Matt Stone Date: Thu, 8 Jan 2026 18:33:50 +1000 Subject: [PATCH 1/6] Tests and implementation for stashignore --- go.mod | 1 + go.sum | 2 + internal/manager/scan_stashignore_test.go | 489 ++++++++++++++++++++ pkg/file/stashignore.go | 190 ++++++++ pkg/file/stashignore_test.go | 523 ++++++++++++++++++++++ 5 files changed, 1205 insertions(+) create mode 100644 internal/manager/scan_stashignore_test.go create mode 100644 pkg/file/stashignore.go create mode 100644 pkg/file/stashignore_test.go diff --git a/go.mod b/go.mod index 705fe40ed..84bf36454 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd + github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cast v1.6.0 github.com/spf13/pflag v1.0.6 diff --git a/go.sum b/go.sum index 750acd9ab..277647764 100644 --- a/go.sum +++ b/go.sum @@ -541,6 +541,8 @@ github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= diff --git a/internal/manager/scan_stashignore_test.go b/internal/manager/scan_stashignore_test.go new file mode 100644 index 000000000..b8c5fb947 --- /dev/null +++ b/internal/manager/scan_stashignore_test.go @@ -0,0 +1,489 @@ +//go:build integration +// +build integration + +package manager + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/sqlite" + "github.com/stashapp/stash/pkg/txn" + + // Necessary to register custom migrations. + _ "github.com/stashapp/stash/pkg/sqlite/migrations" +) + +// mockFingerprintCalculator returns empty fingerprints. +type mockFingerprintCalculator struct{} + +func (m *mockFingerprintCalculator) CalculateFingerprints(f *models.BaseFile, o file.Opener, useExisting bool) ([]models.Fingerprint, error) { + // Return a simple fingerprint based on path for testing. + return []models.Fingerprint{ + { + Type: models.FingerprintTypeMD5, + Fingerprint: fmt.Sprintf("md5-%s", f.Basename), + }, + }, nil +} + +// mockProgressReporter does nothing. +type mockProgressReporter struct{} + +func (m *mockProgressReporter) AddTotal(total int) {} +func (m *mockProgressReporter) Increment() {} +func (m *mockProgressReporter) Definite() {} +func (m *mockProgressReporter) ExecuteTask(description string, fn func()) { fn() } + +// createTestFileOnDisk creates a file with some content. +func createTestFileOnDisk(t *testing.T, dir, name string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("failed to create directory for %s: %v", path, err) + } + // Write some content so the file has a non-zero size. + if err := os.WriteFile(path, []byte("test content for "+name), 0644); err != nil { + t.Fatalf("failed to create file %s: %v", path, err) + } + return path +} + +// createStashIgnoreFile creates a .stashignore file with the given content. +func createStashIgnoreFile(t *testing.T, dir, content string) { + t.Helper() + path := filepath.Join(dir, ".stashignore") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("failed to create .stashignore: %v", err) + } +} + +// setupTestDatabase creates a temporary SQLite database for testing. +func setupScanTestDatabase(t *testing.T) (*sqlite.Database, func()) { + t.Helper() + + // Initialise empty config - needed by some migrations. + _ = config.InitializeEmpty() + + // Create temporary database file. + f, err := os.CreateTemp("", "stash-scan-test-*.sqlite") + if err != nil { + t.Fatalf("failed to create temp database file: %v", err) + } + f.Close() + dbFile := f.Name() + + db := sqlite.NewDatabase() + db.SetBlobStoreOptions(sqlite.BlobStoreOptions{ + UseDatabase: true, + }) + + if err := db.Open(dbFile); err != nil { + os.Remove(dbFile) + t.Fatalf("failed to open database: %v", err) + } + + cleanup := func() { + db.Close() + os.Remove(dbFile) + } + + return db, cleanup +} + +func TestScannerWithStashIgnore(t *testing.T) { + // Setup test database. + db, cleanup := setupScanTestDatabase(t) + defer cleanup() + + // Create temp directory structure. + tmpDir := t.TempDir() + + // Create test files. + createTestFileOnDisk(t, tmpDir, "video1.mp4") + createTestFileOnDisk(t, tmpDir, "video2.mp4") + createTestFileOnDisk(t, tmpDir, "ignore_me.mp4") + createTestFileOnDisk(t, tmpDir, "subdir/video3.mp4") + createTestFileOnDisk(t, tmpDir, "subdir/skip_this.mp4") + createTestFileOnDisk(t, tmpDir, "excluded_dir/video4.mp4") + createTestFileOnDisk(t, tmpDir, "temp/processing.mp4") + + // Create .stashignore file. + stashignore := `# Ignore specific files +ignore_me.mp4 +subdir/skip_this.mp4 + +# Ignore directories +excluded_dir/ +temp/ +` + createStashIgnoreFile(t, tmpDir, stashignore) + + // Create scanner. + repo := file.NewRepository(db.Repository()) + scanner := &file.Scanner{ + FS: &file.OsFS{}, + Repository: repo, + FingerprintCalculator: &mockFingerprintCalculator{}, + } + + // Create stashignore filter. + stashIgnoreFilter := file.NewStashIgnoreFilter(tmpDir) + + // Run scan. + ctx := context.Background() + scanner.Scan(ctx, nil, file.ScanOptions{ + Paths: []string{tmpDir}, + ScanFilters: []file.PathFilter{stashIgnoreFilter}, + ParallelTasks: 1, + }, &mockProgressReporter{}) + + // Verify results by checking what's in the database. + var scannedPaths []string + err := txn.WithTxn(ctx, db, func(ctx context.Context) error { + // Check folders by path. + checkDirs := []string{ + filepath.Join(tmpDir, "subdir"), + filepath.Join(tmpDir, "excluded_dir"), + filepath.Join(tmpDir, "temp"), + } + + for _, dir := range checkDirs { + f, err := db.Folder.FindByPath(ctx, dir, true) + if err != nil { + return fmt.Errorf("checking folder %s: %w", dir, err) + } + if f != nil { + relPath, _ := filepath.Rel(tmpDir, dir) + scannedPaths = append(scannedPaths, "dir:"+relPath) + } + } + + // Check specific files. + checkFiles := []string{ + filepath.Join(tmpDir, "video1.mp4"), + filepath.Join(tmpDir, "video2.mp4"), + filepath.Join(tmpDir, "ignore_me.mp4"), + filepath.Join(tmpDir, "subdir/video3.mp4"), + filepath.Join(tmpDir, "subdir/skip_this.mp4"), + filepath.Join(tmpDir, "excluded_dir/video4.mp4"), + filepath.Join(tmpDir, "temp/processing.mp4"), + } + + for _, path := range checkFiles { + f, err := db.File.FindByPath(ctx, path, true) + if err != nil { + return fmt.Errorf("checking file %s: %w", path, err) + } + if f != nil { + relPath, _ := filepath.Rel(tmpDir, path) + scannedPaths = append(scannedPaths, "file:"+relPath) + } + } + + return nil + }) + + if err != nil { + t.Fatalf("failed to verify scan results: %v", err) + } + + sort.Strings(scannedPaths) + + // Expected: video1.mp4, video2.mp4, subdir/video3.mp4, and their folders. + // NOT expected: ignore_me.mp4, subdir/skip_this.mp4, excluded_dir/*, temp/*. + expectedPaths := []string{ + "dir:subdir", + "file:subdir/video3.mp4", + "file:video1.mp4", + "file:video2.mp4", + } + sort.Strings(expectedPaths) + + if len(scannedPaths) != len(expectedPaths) { + t.Errorf("scanned path count mismatch:\nexpected %d: %v\nactual %d: %v", + len(expectedPaths), expectedPaths, len(scannedPaths), scannedPaths) + return + } + + for i := range expectedPaths { + if scannedPaths[i] != expectedPaths[i] { + t.Errorf("path mismatch at index %d:\nexpected: %s\nactual: %s", + i, expectedPaths[i], scannedPaths[i]) + } + } +} + +func TestScannerWithNestedStashIgnore(t *testing.T) { + // Setup test database. + db, cleanup := setupScanTestDatabase(t) + defer cleanup() + + // Create temp directory structure. + tmpDir := t.TempDir() + + // Create test files. + createTestFileOnDisk(t, tmpDir, "root.mp4") + createTestFileOnDisk(t, tmpDir, "root.tmp") + createTestFileOnDisk(t, tmpDir, "subdir/sub.mp4") + createTestFileOnDisk(t, tmpDir, "subdir/sub.log") + createTestFileOnDisk(t, tmpDir, "subdir/sub.tmp") + + // Root .stashignore excludes *.tmp. + createStashIgnoreFile(t, tmpDir, "*.tmp\n") + + // Subdir .stashignore excludes *.log. + createStashIgnoreFile(t, filepath.Join(tmpDir, "subdir"), "*.log\n") + + // Create scanner. + repo := file.NewRepository(db.Repository()) + scanner := &file.Scanner{ + FS: &file.OsFS{}, + Repository: repo, + FingerprintCalculator: &mockFingerprintCalculator{}, + } + + // Create stashignore filter. + stashIgnoreFilter := file.NewStashIgnoreFilter(tmpDir) + + // Run scan. + ctx := context.Background() + scanner.Scan(ctx, nil, file.ScanOptions{ + Paths: []string{tmpDir}, + ScanFilters: []file.PathFilter{stashIgnoreFilter}, + ParallelTasks: 1, + }, &mockProgressReporter{}) + + // Verify results. + var scannedFiles []string + err := txn.WithTxn(ctx, db, func(ctx context.Context) error { + checkFiles := []string{ + filepath.Join(tmpDir, "root.mp4"), + filepath.Join(tmpDir, "root.tmp"), + filepath.Join(tmpDir, "subdir/sub.mp4"), + filepath.Join(tmpDir, "subdir/sub.log"), + filepath.Join(tmpDir, "subdir/sub.tmp"), + } + + for _, path := range checkFiles { + f, err := db.File.FindByPath(ctx, path, true) + if err != nil { + return fmt.Errorf("checking file %s: %w", path, err) + } + if f != nil { + relPath, _ := filepath.Rel(tmpDir, path) + scannedFiles = append(scannedFiles, relPath) + } + } + + return nil + }) + + if err != nil { + t.Fatalf("failed to verify scan results: %v", err) + } + + sort.Strings(scannedFiles) + + // Expected: root.mp4, subdir/sub.mp4. + // NOT expected: root.tmp (root ignore), subdir/sub.log (subdir ignore), subdir/sub.tmp (root ignore). + expectedFiles := []string{ + "root.mp4", + "subdir/sub.mp4", + } + sort.Strings(expectedFiles) + + if len(scannedFiles) != len(expectedFiles) { + t.Errorf("scanned file count mismatch:\nexpected %d: %v\nactual %d: %v", + len(expectedFiles), expectedFiles, len(scannedFiles), scannedFiles) + return + } + + for i := range expectedFiles { + if scannedFiles[i] != expectedFiles[i] { + t.Errorf("file mismatch at index %d:\nexpected: %s\nactual: %s", + i, expectedFiles[i], scannedFiles[i]) + } + } +} + +func TestScannerWithoutStashIgnore(t *testing.T) { + // Setup test database. + db, cleanup := setupScanTestDatabase(t) + defer cleanup() + + // Create temp directory structure (no .stashignore). + tmpDir := t.TempDir() + + // Create test files. + createTestFileOnDisk(t, tmpDir, "video1.mp4") + createTestFileOnDisk(t, tmpDir, "video2.mp4") + createTestFileOnDisk(t, tmpDir, "subdir/video3.mp4") + + // Create scanner. + repo := file.NewRepository(db.Repository()) + scanner := &file.Scanner{ + FS: &file.OsFS{}, + Repository: repo, + FingerprintCalculator: &mockFingerprintCalculator{}, + } + + // Create stashignore filter (but no .stashignore file exists). + stashIgnoreFilter := file.NewStashIgnoreFilter(tmpDir) + + // Run scan. + ctx := context.Background() + scanner.Scan(ctx, nil, file.ScanOptions{ + Paths: []string{tmpDir}, + ScanFilters: []file.PathFilter{stashIgnoreFilter}, + ParallelTasks: 1, + }, &mockProgressReporter{}) + + // Verify all files were scanned. + var scannedFiles []string + err := txn.WithTxn(ctx, db, func(ctx context.Context) error { + checkFiles := []string{ + filepath.Join(tmpDir, "video1.mp4"), + filepath.Join(tmpDir, "video2.mp4"), + filepath.Join(tmpDir, "subdir/video3.mp4"), + } + + for _, path := range checkFiles { + f, err := db.File.FindByPath(ctx, path, true) + if err != nil { + return fmt.Errorf("checking file %s: %w", path, err) + } + if f != nil { + relPath, _ := filepath.Rel(tmpDir, path) + scannedFiles = append(scannedFiles, relPath) + } + } + + return nil + }) + + if err != nil { + t.Fatalf("failed to verify scan results: %v", err) + } + + sort.Strings(scannedFiles) + + // All files should be scanned. + expectedFiles := []string{ + "subdir/video3.mp4", + "video1.mp4", + "video2.mp4", + } + sort.Strings(expectedFiles) + + if len(scannedFiles) != len(expectedFiles) { + t.Errorf("scanned file count mismatch:\nexpected %d: %v\nactual %d: %v", + len(expectedFiles), expectedFiles, len(scannedFiles), scannedFiles) + return + } + + for i := range expectedFiles { + if scannedFiles[i] != expectedFiles[i] { + t.Errorf("file mismatch at index %d:\nexpected: %s\nactual: %s", + i, expectedFiles[i], scannedFiles[i]) + } + } +} + +func TestScannerWithNegationPattern(t *testing.T) { + // Setup test database. + db, cleanup := setupScanTestDatabase(t) + defer cleanup() + + // Create temp directory structure. + tmpDir := t.TempDir() + + // Create test files. + createTestFileOnDisk(t, tmpDir, "file1.tmp") + createTestFileOnDisk(t, tmpDir, "file2.tmp") + createTestFileOnDisk(t, tmpDir, "keep_this.tmp") + createTestFileOnDisk(t, tmpDir, "video.mp4") + + // Create .stashignore with negation. + stashignore := `*.tmp +!keep_this.tmp +` + createStashIgnoreFile(t, tmpDir, stashignore) + + // Create scanner. + repo := file.NewRepository(db.Repository()) + scanner := &file.Scanner{ + FS: &file.OsFS{}, + Repository: repo, + FingerprintCalculator: &mockFingerprintCalculator{}, + } + + // Create stashignore filter. + stashIgnoreFilter := file.NewStashIgnoreFilter(tmpDir) + + // Run scan. + ctx := context.Background() + scanner.Scan(ctx, nil, file.ScanOptions{ + Paths: []string{tmpDir}, + ScanFilters: []file.PathFilter{stashIgnoreFilter}, + ParallelTasks: 1, + }, &mockProgressReporter{}) + + // Verify results. + var scannedFiles []string + err := txn.WithTxn(ctx, db, func(ctx context.Context) error { + checkFiles := []string{ + filepath.Join(tmpDir, "file1.tmp"), + filepath.Join(tmpDir, "file2.tmp"), + filepath.Join(tmpDir, "keep_this.tmp"), + filepath.Join(tmpDir, "video.mp4"), + } + + for _, path := range checkFiles { + f, err := db.File.FindByPath(ctx, path, true) + if err != nil { + return fmt.Errorf("checking file %s: %w", path, err) + } + if f != nil { + relPath, _ := filepath.Rel(tmpDir, path) + scannedFiles = append(scannedFiles, relPath) + } + } + + return nil + }) + + if err != nil { + t.Fatalf("failed to verify scan results: %v", err) + } + + sort.Strings(scannedFiles) + + // Expected: keep_this.tmp (negated), video.mp4. + // NOT expected: file1.tmp, file2.tmp. + expectedFiles := []string{ + "keep_this.tmp", + "video.mp4", + } + sort.Strings(expectedFiles) + + if len(scannedFiles) != len(expectedFiles) { + t.Errorf("scanned file count mismatch:\nexpected %d: %v\nactual %d: %v", + len(expectedFiles), expectedFiles, len(scannedFiles), scannedFiles) + return + } + + for i := range expectedFiles { + if scannedFiles[i] != expectedFiles[i] { + t.Errorf("file mismatch at index %d:\nexpected: %s\nactual: %s", + i, expectedFiles[i], scannedFiles[i]) + } + } +} diff --git a/pkg/file/stashignore.go b/pkg/file/stashignore.go new file mode 100644 index 000000000..3581bda8e --- /dev/null +++ b/pkg/file/stashignore.go @@ -0,0 +1,190 @@ +package file + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + + ignore "github.com/sabhiram/go-gitignore" +) + +const stashIgnoreFilename = ".stashignore" + +// StashIgnoreFilter implements PathFilter to exclude files/directories +// based on .stashignore files with gitignore-style patterns. +type StashIgnoreFilter struct { + // root is the root directory being scanned. + root string + + // cache stores compiled ignore patterns per directory. + cache sync.Map // map[string]*ignoreEntry +} + +// ignoreEntry holds the compiled ignore patterns for a directory. +type ignoreEntry struct { + // patterns is the compiled gitignore matcher for this directory. + patterns *ignore.GitIgnore + // dir is the directory this entry applies to. + dir string +} + +// NewStashIgnoreFilter creates a new StashIgnoreFilter for the given root directory. +func NewStashIgnoreFilter(root string) *StashIgnoreFilter { + return &StashIgnoreFilter{ + root: root, + } +} + +// Accept returns true if the path should be included in the scan. +// It checks for .stashignore files in the directory hierarchy and +// applies gitignore-style pattern matching. +func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.FileInfo) bool { + // Always accept .stashignore files themselves so they can be read. + if filepath.Base(path) == stashIgnoreFilename { + return true + } + + // Get the directory containing this path. + var dir string + if info.IsDir() { + dir = filepath.Dir(path) + } else { + dir = filepath.Dir(path) + } + + // Collect all applicable ignore entries from root to this directory. + entries := f.collectIgnoreEntries(dir) + + // Check if any pattern matches (and isn't negated). + relPath, err := filepath.Rel(f.root, path) + if err != nil { + // If we can't get relative path, accept the file. + return true + } + + // Normalise to forward slashes for consistent matching. + relPath = filepath.ToSlash(relPath) + + // For directories, also check with trailing slash. + if info.IsDir() { + relPath = relPath + "/" + } + + // Check each ignore entry in order (from root to most specific). + // Later entries can override earlier ones with negation patterns. + ignored := false + for _, entry := range entries { + // Get path relative to the ignore file's directory. + entryRelPath, err := filepath.Rel(entry.dir, path) + if err != nil { + continue + } + entryRelPath = filepath.ToSlash(entryRelPath) + if info.IsDir() { + entryRelPath = entryRelPath + "/" + } + + if entry.patterns.MatchesPath(entryRelPath) { + ignored = true + } + // Check negation by testing without the directory suffix. + // The library handles negation internally. + } + + return !ignored +} + +// collectIgnoreEntries gathers all ignore entries from root to the given directory. +func (f *StashIgnoreFilter) collectIgnoreEntries(dir string) []*ignoreEntry { + var entries []*ignoreEntry + + // Walk from root to current directory. + current := f.root + relDir, err := filepath.Rel(f.root, dir) + if err != nil { + return entries + } + + // Check root directory first. + if entry := f.getOrLoadIgnoreEntry(current); entry != nil { + entries = append(entries, entry) + } + + // Then check each subdirectory. + if relDir != "." { + parts := strings.Split(filepath.ToSlash(relDir), "/") + for _, part := range parts { + current = filepath.Join(current, part) + if entry := f.getOrLoadIgnoreEntry(current); entry != nil { + entries = append(entries, entry) + } + } + } + + return entries +} + +// getOrLoadIgnoreEntry returns the cached ignore entry for a directory, or loads it. +func (f *StashIgnoreFilter) getOrLoadIgnoreEntry(dir string) *ignoreEntry { + // Check cache first. + if cached, ok := f.cache.Load(dir); ok { + entry := cached.(*ignoreEntry) + if entry.patterns == nil { + return nil // Cached negative result. + } + return entry + } + + // Try to load .stashignore from this directory. + stashIgnorePath := filepath.Join(dir, stashIgnoreFilename) + patterns, err := f.loadIgnoreFile(stashIgnorePath, dir) + if err != nil { + // Cache negative result. + f.cache.Store(dir, &ignoreEntry{patterns: nil, dir: dir}) + return nil + } + + entry := &ignoreEntry{ + patterns: patterns, + dir: dir, + } + f.cache.Store(dir, entry) + return entry +} + +// loadIgnoreFile loads and compiles a .stashignore file. +func (f *StashIgnoreFilter) loadIgnoreFile(path string, dir string) (*ignore.GitIgnore, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + lines := strings.Split(string(data), "\n") + var patterns []string + + for _, line := range lines { + // Trim trailing whitespace (but preserve leading for patterns). + line = strings.TrimRight(line, " \t\r") + + // Skip empty lines. + if line == "" { + continue + } + + // Skip comments (but not escaped #). + if strings.HasPrefix(line, "#") && !strings.HasPrefix(line, "\\#") { + continue + } + + patterns = append(patterns, line) + } + + if len(patterns) == 0 { + return nil, os.ErrNotExist + } + + return ignore.CompileIgnoreLines(patterns...), nil +} diff --git a/pkg/file/stashignore_test.go b/pkg/file/stashignore_test.go new file mode 100644 index 000000000..9103c5da6 --- /dev/null +++ b/pkg/file/stashignore_test.go @@ -0,0 +1,523 @@ +package file + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "sort" + "testing" +) + +// Helper to create an empty file. +func createTestFile(t *testing.T, dir, name string) { + t.Helper() + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("failed to create directory for %s: %v", path, err) + } + if err := os.WriteFile(path, []byte{}, 0644); err != nil { + t.Fatalf("failed to create file %s: %v", path, err) + } +} + +// Helper to create a file with content. +func createTestFileWithContent(t *testing.T, dir, name, content string) { + t.Helper() + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("failed to create directory for %s: %v", path, err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("failed to create file %s: %v", path, err) + } +} + +// Helper to create a directory. +func createTestDir(t *testing.T, dir, name string) { + t.Helper() + path := filepath.Join(dir, name) + if err := os.MkdirAll(path, 0755); err != nil { + t.Fatalf("failed to create directory %s: %v", path, err) + } +} + +// walkAndFilter walks the directory tree and returns paths accepted by the filter. +// Returns paths relative to root for easier assertion. +func walkAndFilter(t *testing.T, root string, filter *StashIgnoreFilter) []string { + t.Helper() + var accepted []string + ctx := context.Background() + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Skip the root directory itself. + if path == root { + return nil + } + + info, err := d.Info() + if err != nil { + return err + } + + if filter.Accept(ctx, path, info) { + relPath, _ := filepath.Rel(root, path) + accepted = append(accepted, relPath) + } else if info.IsDir() { + // If directory is rejected, skip it. + return filepath.SkipDir + } + + return nil + }) + + if err != nil { + t.Fatalf("walk failed: %v", err) + } + + sort.Strings(accepted) + return accepted +} + +// assertPathsEqual checks that the accepted paths match expected. +func assertPathsEqual(t *testing.T, expected, actual []string) { + t.Helper() + sort.Strings(expected) + + if len(expected) != len(actual) { + t.Errorf("path count mismatch:\nexpected %d: %v\nactual %d: %v", len(expected), expected, len(actual), actual) + return + } + + for i := range expected { + if expected[i] != actual[i] { + t.Errorf("path mismatch at index %d:\nexpected: %s\nactual: %s", i, expected[i], actual[i]) + } + } +} + +func TestStashIgnore_ExactFilename(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestFile(t, tmpDir, "video2.mp4") + createTestFile(t, tmpDir, "ignore_me.mp4") + + // Create .stashignore that excludes exact filename. + createTestFileWithContent(t, tmpDir, ".stashignore", "ignore_me.mp4\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "video1.mp4", + "video2.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_WildcardPattern(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestFile(t, tmpDir, "video2.mp4") + createTestFile(t, tmpDir, "temp1.tmp") + createTestFile(t, tmpDir, "temp2.tmp") + createTestFile(t, tmpDir, "notes.log") + + // Create .stashignore that excludes by extension. + createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n*.log\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "video1.mp4", + "video2.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_DirectoryExclusion(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestDir(t, tmpDir, "excluded_dir") + createTestFile(t, tmpDir, "excluded_dir/video2.mp4") + createTestFile(t, tmpDir, "excluded_dir/video3.mp4") + createTestDir(t, tmpDir, "included_dir") + createTestFile(t, tmpDir, "included_dir/video4.mp4") + + // Create .stashignore that excludes a directory. + createTestFileWithContent(t, tmpDir, ".stashignore", "excluded_dir/\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "included_dir", + "included_dir/video4.mp4", + "video1.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_NegationPattern(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "file1.tmp") + createTestFile(t, tmpDir, "file2.tmp") + createTestFile(t, tmpDir, "keep_this.tmp") + + // Create .stashignore that excludes *.tmp but keeps one. + createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n!keep_this.tmp\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "keep_this.tmp", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_CommentsAndEmptyLines(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestFile(t, tmpDir, "ignore_me.mp4") + + // Create .stashignore with comments and empty lines. + stashignore := `# This is a comment +ignore_me.mp4 + +# Another comment + +` + createTestFileWithContent(t, tmpDir, ".stashignore", stashignore) + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "video1.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_NestedStashIgnoreFiles(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "root_video.mp4") + createTestFile(t, tmpDir, "root_ignore.tmp") + createTestDir(t, tmpDir, "subdir") + createTestFile(t, tmpDir, "subdir/sub_video.mp4") + createTestFile(t, tmpDir, "subdir/sub_ignore.log") + createTestFile(t, tmpDir, "subdir/also_tmp.tmp") + + // Root .stashignore excludes *.tmp. + createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n") + + // Subdir .stashignore excludes *.log. + createTestFileWithContent(t, tmpDir, "subdir/.stashignore", "*.log\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + // *.tmp from root should apply everywhere. + // *.log from subdir should only apply in subdir. + expected := []string{ + ".stashignore", + "root_video.mp4", + "subdir", + "subdir/.stashignore", + "subdir/sub_video.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_PathPattern(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestDir(t, tmpDir, "subdir") + createTestFile(t, tmpDir, "subdir/video2.mp4") + createTestFile(t, tmpDir, "subdir/skip_this.mp4") + + // Create .stashignore that excludes a specific path. + createTestFileWithContent(t, tmpDir, ".stashignore", "subdir/skip_this.mp4\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "subdir", + "subdir/video2.mp4", + "video1.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_DoubleStarPattern(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestDir(t, tmpDir, "a") + createTestFile(t, tmpDir, "a/video2.mp4") + createTestDir(t, tmpDir, "a/temp") + createTestFile(t, tmpDir, "a/temp/video3.mp4") + createTestDir(t, tmpDir, "a/b") + createTestDir(t, tmpDir, "a/b/temp") + createTestFile(t, tmpDir, "a/b/temp/video4.mp4") + + // Create .stashignore that excludes temp directories at any level. + createTestFileWithContent(t, tmpDir, ".stashignore", "**/temp/\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "a", + "a/b", + "a/video2.mp4", + "video1.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_LeadingSlashPattern(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "ignore.mp4") + createTestDir(t, tmpDir, "subdir") + createTestFile(t, tmpDir, "subdir/ignore.mp4") + + // Create .stashignore that excludes only at root level. + createTestFileWithContent(t, tmpDir, ".stashignore", "/ignore.mp4\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + // Only root ignore.mp4 should be excluded. + expected := []string{ + ".stashignore", + "subdir", + "subdir/ignore.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_NoStashIgnoreFile(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files without any .stashignore. + createTestFile(t, tmpDir, "video1.mp4") + createTestFile(t, tmpDir, "video2.mp4") + createTestDir(t, tmpDir, "subdir") + createTestFile(t, tmpDir, "subdir/video3.mp4") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + // All files should be accepted. + expected := []string{ + "subdir", + "subdir/video3.mp4", + "video1.mp4", + "video2.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_HiddenDirectories(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files including hidden directory. + createTestFile(t, tmpDir, "video1.mp4") + createTestDir(t, tmpDir, ".hidden") + createTestFile(t, tmpDir, ".hidden/video2.mp4") + + // Create .stashignore that excludes hidden directories. + createTestFileWithContent(t, tmpDir, ".stashignore", ".*\n!.stashignore\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "video1.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_MultiplePatternsSameLine(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestFile(t, tmpDir, "file.tmp") + createTestFile(t, tmpDir, "file.log") + createTestFile(t, tmpDir, "file.bak") + + // Each pattern should be on its own line. + createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n*.log\n*.bak\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "video1.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_TrailingSpaces(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestFile(t, tmpDir, "ignore_me.mp4") + + // Pattern with trailing spaces (should be trimmed). + createTestFileWithContent(t, tmpDir, ".stashignore", "ignore_me.mp4 \n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "video1.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_EscapedHash(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files. + createTestFile(t, tmpDir, "video1.mp4") + createTestFile(t, tmpDir, "#filename.mp4") + + // Escaped hash should match literal # character. + createTestFileWithContent(t, tmpDir, ".stashignore", "\\#filename.mp4\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "video1.mp4", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_CaseSensitiveMatching(t *testing.T) { + tmpDir := t.TempDir() + + // Create test files - use distinct names that work on all filesystems. + createTestFile(t, tmpDir, "video_lower.mp4") + createTestFile(t, tmpDir, "VIDEO_UPPER.mp4") + createTestFile(t, tmpDir, "other.avi") + + // Pattern should match exactly (case-sensitive). + createTestFileWithContent(t, tmpDir, ".stashignore", "video_lower.mp4\n") + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + // Only exact match is excluded. + expected := []string{ + ".stashignore", + "VIDEO_UPPER.mp4", + "other.avi", + } + + assertPathsEqual(t, expected, accepted) +} + +func TestStashIgnore_ComplexScenario(t *testing.T) { + tmpDir := t.TempDir() + + // Create a complex directory structure. + createTestFile(t, tmpDir, "video1.mp4") + createTestFile(t, tmpDir, "video2.avi") + createTestFile(t, tmpDir, "thumbnail.jpg") + createTestFile(t, tmpDir, "metadata.nfo") + createTestDir(t, tmpDir, "movies") + createTestFile(t, tmpDir, "movies/movie1.mp4") + createTestFile(t, tmpDir, "movies/movie1.nfo") + createTestDir(t, tmpDir, "movies/.thumbnails") + createTestFile(t, tmpDir, "movies/.thumbnails/thumb1.jpg") + createTestDir(t, tmpDir, "temp") + createTestFile(t, tmpDir, "temp/processing.mp4") + createTestDir(t, tmpDir, "backup") + createTestFile(t, tmpDir, "backup/video1.mp4.bak") + + // Complex .stashignore. + stashignore := `# Ignore metadata files +*.nfo + +# Ignore hidden directories +.* +!.stashignore + +# Ignore temp and backup directories +temp/ +backup/ + +# But keep thumbnails in specific location +!movies/.thumbnails/ +` + createTestFileWithContent(t, tmpDir, ".stashignore", stashignore) + + filter := NewStashIgnoreFilter(tmpDir) + accepted := walkAndFilter(t, tmpDir, filter) + + expected := []string{ + ".stashignore", + "movies", + "movies/.thumbnails", + "movies/.thumbnails/thumb1.jpg", + "movies/movie1.mp4", + "thumbnail.jpg", + "video1.mp4", + "video2.avi", + } + + assertPathsEqual(t, expected, accepted) +} From cc3431dd04508b2512551f9f62bc0e820b6a8678 Mon Sep 17 00:00:00 2001 From: Matt Stone Date: Thu, 8 Jan 2026 19:22:33 +1000 Subject: [PATCH 2/6] Feedback changes --- internal/manager/scan_stashignore_test.go | 40 +++++-- internal/manager/task_scan.go | 8 ++ pkg/file/stashignore.go | 131 ++++++++++++---------- pkg/file/stashignore_test.go | 34 +++--- ui/v2.5/src/docs/en/Manual/Tasks.md | 33 ++++++ 5 files changed, 162 insertions(+), 84 deletions(-) diff --git a/internal/manager/scan_stashignore_test.go b/internal/manager/scan_stashignore_test.go index b8c5fb947..73db71fec 100644 --- a/internal/manager/scan_stashignore_test.go +++ b/internal/manager/scan_stashignore_test.go @@ -6,6 +6,7 @@ package manager import ( "context" "fmt" + "io/fs" "os" "path/filepath" "sort" @@ -42,6 +43,17 @@ func (m *mockProgressReporter) Increment() {} func (m *mockProgressReporter) Definite() {} func (m *mockProgressReporter) ExecuteTask(description string, fn func()) { fn() } +// stashIgnorePathFilter wraps StashIgnoreFilter to implement PathFilter for testing. +// It provides a fixed library root for the filter. +type stashIgnorePathFilter struct { + filter *file.StashIgnoreFilter + libraryRoot string +} + +func (f *stashIgnorePathFilter) Accept(ctx context.Context, path string, info fs.FileInfo) bool { + return f.filter.Accept(ctx, path, info, f.libraryRoot) +} + // createTestFileOnDisk creates a file with some content. func createTestFileOnDisk(t *testing.T, dir, name string) string { t.Helper() @@ -134,8 +146,11 @@ temp/ FingerprintCalculator: &mockFingerprintCalculator{}, } - // Create stashignore filter. - stashIgnoreFilter := file.NewStashIgnoreFilter(tmpDir) + // Create stashignore filter with library root. + stashIgnoreFilter := &stashIgnorePathFilter{ + filter: file.NewStashIgnoreFilter(), + libraryRoot: tmpDir, + } // Run scan. ctx := context.Background() @@ -250,8 +265,11 @@ func TestScannerWithNestedStashIgnore(t *testing.T) { FingerprintCalculator: &mockFingerprintCalculator{}, } - // Create stashignore filter. - stashIgnoreFilter := file.NewStashIgnoreFilter(tmpDir) + // Create stashignore filter with library root. + stashIgnoreFilter := &stashIgnorePathFilter{ + filter: file.NewStashIgnoreFilter(), + libraryRoot: tmpDir, + } // Run scan. ctx := context.Background() @@ -335,8 +353,11 @@ func TestScannerWithoutStashIgnore(t *testing.T) { FingerprintCalculator: &mockFingerprintCalculator{}, } - // Create stashignore filter (but no .stashignore file exists). - stashIgnoreFilter := file.NewStashIgnoreFilter(tmpDir) + // Create stashignore filter with library root (but no .stashignore file exists). + stashIgnoreFilter := &stashIgnorePathFilter{ + filter: file.NewStashIgnoreFilter(), + libraryRoot: tmpDir, + } // Run scan. ctx := context.Background() @@ -425,8 +446,11 @@ func TestScannerWithNegationPattern(t *testing.T) { FingerprintCalculator: &mockFingerprintCalculator{}, } - // Create stashignore filter. - stashIgnoreFilter := file.NewStashIgnoreFilter(tmpDir) + // Create stashignore filter with library root. + stashIgnoreFilter := &stashIgnorePathFilter{ + filter: file.NewStashIgnoreFilter(), + libraryRoot: tmpDir, + } // Run scan. ctx := context.Background() diff --git a/internal/manager/task_scan.go b/internal/manager/task_scan.go index 6f7f34b3c..12ab69932 100644 --- a/internal/manager/task_scan.go +++ b/internal/manager/task_scan.go @@ -254,6 +254,7 @@ type scanFilter struct { videoExcludeRegex []*regexp.Regexp imageExcludeRegex []*regexp.Regexp minModTime time.Time + stashIgnoreFilter *file.StashIgnoreFilter } func newScanFilter(c *config.Config, repo models.Repository, minModTime time.Time) *scanFilter { @@ -267,6 +268,7 @@ func newScanFilter(c *config.Config, repo models.Repository, minModTime time.Tim videoExcludeRegex: generateRegexps(c.GetExcludes()), imageExcludeRegex: generateRegexps(c.GetImageExcludes()), minModTime: minModTime, + stashIgnoreFilter: file.NewStashIgnoreFilter(), } } @@ -287,6 +289,12 @@ func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo) return false } + // Check .stashignore files, bounded to the library root. + if !f.stashIgnoreFilter.Accept(ctx, path, info, s.Path) { + logger.Debugf("Skipping %s due to .stashignore", path) + return false + } + isVideoFile := useAsVideo(path) isImageFile := useAsImage(path) isZipFile := fsutil.MatchExtension(path, f.zipExt) diff --git a/pkg/file/stashignore.go b/pkg/file/stashignore.go index 3581bda8e..db7b7d3fa 100644 --- a/pkg/file/stashignore.go +++ b/pkg/file/stashignore.go @@ -9,6 +9,7 @@ import ( "sync" ignore "github.com/sabhiram/go-gitignore" + "github.com/stashapp/stash/pkg/logger" ) const stashIgnoreFilename = ".stashignore" @@ -16,9 +17,6 @@ const stashIgnoreFilename = ".stashignore" // StashIgnoreFilter implements PathFilter to exclude files/directories // based on .stashignore files with gitignore-style patterns. type StashIgnoreFilter struct { - // root is the root directory being scanned. - root string - // cache stores compiled ignore patterns per directory. cache sync.Map // map[string]*ignoreEntry } @@ -31,46 +29,36 @@ type ignoreEntry struct { dir string } -// NewStashIgnoreFilter creates a new StashIgnoreFilter for the given root directory. -func NewStashIgnoreFilter(root string) *StashIgnoreFilter { - return &StashIgnoreFilter{ - root: root, - } +// NewStashIgnoreFilter creates a new StashIgnoreFilter. +func NewStashIgnoreFilter() *StashIgnoreFilter { + return &StashIgnoreFilter{} } // Accept returns true if the path should be included in the scan. // It checks for .stashignore files in the directory hierarchy and // applies gitignore-style pattern matching. -func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.FileInfo) bool { +// The libraryRoot parameter bounds the search for .stashignore files - +// only directories within the library root are checked. +func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.FileInfo, libraryRoot string) bool { // Always accept .stashignore files themselves so they can be read. if filepath.Base(path) == stashIgnoreFilename { return true } - // Get the directory containing this path. - var dir string - if info.IsDir() { - dir = filepath.Dir(path) - } else { - dir = filepath.Dir(path) - } - - // Collect all applicable ignore entries from root to this directory. - entries := f.collectIgnoreEntries(dir) - - // Check if any pattern matches (and isn't negated). - relPath, err := filepath.Rel(f.root, path) - if err != nil { - // If we can't get relative path, accept the file. + // If no library root provided, accept the file (safety fallback). + if libraryRoot == "" { return true } - // Normalise to forward slashes for consistent matching. - relPath = filepath.ToSlash(relPath) + // Get the directory containing this path. + dir := filepath.Dir(path) - // For directories, also check with trailing slash. - if info.IsDir() { - relPath = relPath + "/" + // Collect all applicable ignore entries from library root to this directory. + entries := f.collectIgnoreEntries(dir, libraryRoot) + + // If no .stashignore files found, accept the file. + if len(entries) == 0 { + return true } // Check each ignore entry in order (from root to most specific). @@ -90,43 +78,65 @@ func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.Fil if entry.patterns.MatchesPath(entryRelPath) { ignored = true } - // Check negation by testing without the directory suffix. - // The library handles negation internally. } return !ignored } -// collectIgnoreEntries gathers all ignore entries from root to the given directory. -func (f *StashIgnoreFilter) collectIgnoreEntries(dir string) []*ignoreEntry { +// collectIgnoreEntries gathers all ignore entries from library root to the given directory. +// It walks up the directory tree from dir to libraryRoot and returns entries in order +// from root to most specific. +func (f *StashIgnoreFilter) collectIgnoreEntries(dir string, libraryRoot string) []*ignoreEntry { + // Collect directories from library root down to current dir. + var dirs []string + + // Clean paths for consistent comparison. + dir = filepath.Clean(dir) + libraryRoot = filepath.Clean(libraryRoot) + + // Walk up from dir to library root, collecting directories. + current := dir + for { + // Check if we're still within the library root. + if !isPathInOrEqual(libraryRoot, current) { + break + } + + dirs = append([]string{current}, dirs...) // Prepend to maintain root-to-leaf order. + + // Stop if we've reached the library root. + if current == libraryRoot { + break + } + + parent := filepath.Dir(current) + if parent == current { + // Reached filesystem root without finding library root. + break + } + current = parent + } + + // Check each directory for .stashignore files. var entries []*ignoreEntry - - // Walk from root to current directory. - current := f.root - relDir, err := filepath.Rel(f.root, dir) - if err != nil { - return entries - } - - // Check root directory first. - if entry := f.getOrLoadIgnoreEntry(current); entry != nil { - entries = append(entries, entry) - } - - // Then check each subdirectory. - if relDir != "." { - parts := strings.Split(filepath.ToSlash(relDir), "/") - for _, part := range parts { - current = filepath.Join(current, part) - if entry := f.getOrLoadIgnoreEntry(current); entry != nil { - entries = append(entries, entry) - } + for _, d := range dirs { + if entry := f.getOrLoadIgnoreEntry(d); entry != nil { + entries = append(entries, entry) } } return entries } +// isPathInOrEqual checks if path is equal to or inside root. +func isPathInOrEqual(root, path string) bool { + if path == root { + return true + } + // Check if path starts with root + separator. + return strings.HasPrefix(path, root+string(filepath.Separator)) +} + // getOrLoadIgnoreEntry returns the cached ignore entry for a directory, or loads it. func (f *StashIgnoreFilter) getOrLoadIgnoreEntry(dir string) *ignoreEntry { // Check cache first. @@ -140,13 +150,15 @@ func (f *StashIgnoreFilter) getOrLoadIgnoreEntry(dir string) *ignoreEntry { // Try to load .stashignore from this directory. stashIgnorePath := filepath.Join(dir, stashIgnoreFilename) - patterns, err := f.loadIgnoreFile(stashIgnorePath, dir) - if err != nil { - // Cache negative result. + patterns, err := f.loadIgnoreFile(stashIgnorePath) + if err != nil || patterns == nil { + // Cache negative result (file doesn't exist or has no patterns). f.cache.Store(dir, &ignoreEntry{patterns: nil, dir: dir}) return nil } + logger.Debugf("Loaded .stashignore from %s", dir) + entry := &ignoreEntry{ patterns: patterns, dir: dir, @@ -156,7 +168,7 @@ func (f *StashIgnoreFilter) getOrLoadIgnoreEntry(dir string) *ignoreEntry { } // loadIgnoreFile loads and compiles a .stashignore file. -func (f *StashIgnoreFilter) loadIgnoreFile(path string, dir string) (*ignore.GitIgnore, error) { +func (f *StashIgnoreFilter) loadIgnoreFile(path string) (*ignore.GitIgnore, error) { data, err := os.ReadFile(path) if err != nil { return nil, err @@ -183,7 +195,8 @@ func (f *StashIgnoreFilter) loadIgnoreFile(path string, dir string) (*ignore.Git } if len(patterns) == 0 { - return nil, os.ErrNotExist + // File exists but has no patterns (e.g., only comments). + return nil, nil } return ignore.CompileIgnoreLines(patterns...), nil diff --git a/pkg/file/stashignore_test.go b/pkg/file/stashignore_test.go index 9103c5da6..5297f544b 100644 --- a/pkg/file/stashignore_test.go +++ b/pkg/file/stashignore_test.go @@ -64,7 +64,7 @@ func walkAndFilter(t *testing.T, root string, filter *StashIgnoreFilter) []strin return err } - if filter.Accept(ctx, path, info) { + if filter.Accept(ctx, path, info, root) { relPath, _ := filepath.Rel(root, path) accepted = append(accepted, relPath) } else if info.IsDir() { @@ -111,7 +111,7 @@ func TestStashIgnore_ExactFilename(t *testing.T) { // Create .stashignore that excludes exact filename. createTestFileWithContent(t, tmpDir, ".stashignore", "ignore_me.mp4\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -136,7 +136,7 @@ func TestStashIgnore_WildcardPattern(t *testing.T) { // Create .stashignore that excludes by extension. createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n*.log\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -162,7 +162,7 @@ func TestStashIgnore_DirectoryExclusion(t *testing.T) { // Create .stashignore that excludes a directory. createTestFileWithContent(t, tmpDir, ".stashignore", "excluded_dir/\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -186,7 +186,7 @@ func TestStashIgnore_NegationPattern(t *testing.T) { // Create .stashignore that excludes *.tmp but keeps one. createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n!keep_this.tmp\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -213,7 +213,7 @@ ignore_me.mp4 ` createTestFileWithContent(t, tmpDir, ".stashignore", stashignore) - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -241,7 +241,7 @@ func TestStashIgnore_NestedStashIgnoreFiles(t *testing.T) { // Subdir .stashignore excludes *.log. createTestFileWithContent(t, tmpDir, "subdir/.stashignore", "*.log\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) // *.tmp from root should apply everywhere. @@ -269,7 +269,7 @@ func TestStashIgnore_PathPattern(t *testing.T) { // Create .stashignore that excludes a specific path. createTestFileWithContent(t, tmpDir, ".stashignore", "subdir/skip_this.mp4\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -298,7 +298,7 @@ func TestStashIgnore_DoubleStarPattern(t *testing.T) { // Create .stashignore that excludes temp directories at any level. createTestFileWithContent(t, tmpDir, ".stashignore", "**/temp/\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -323,7 +323,7 @@ func TestStashIgnore_LeadingSlashPattern(t *testing.T) { // Create .stashignore that excludes only at root level. createTestFileWithContent(t, tmpDir, ".stashignore", "/ignore.mp4\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) // Only root ignore.mp4 should be excluded. @@ -345,7 +345,7 @@ func TestStashIgnore_NoStashIgnoreFile(t *testing.T) { createTestDir(t, tmpDir, "subdir") createTestFile(t, tmpDir, "subdir/video3.mp4") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) // All files should be accepted. @@ -370,7 +370,7 @@ func TestStashIgnore_HiddenDirectories(t *testing.T) { // Create .stashignore that excludes hidden directories. createTestFileWithContent(t, tmpDir, ".stashignore", ".*\n!.stashignore\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -393,7 +393,7 @@ func TestStashIgnore_MultiplePatternsSameLine(t *testing.T) { // Each pattern should be on its own line. createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n*.log\n*.bak\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -414,7 +414,7 @@ func TestStashIgnore_TrailingSpaces(t *testing.T) { // Pattern with trailing spaces (should be trimmed). createTestFileWithContent(t, tmpDir, ".stashignore", "ignore_me.mp4 \n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -435,7 +435,7 @@ func TestStashIgnore_EscapedHash(t *testing.T) { // Escaped hash should match literal # character. createTestFileWithContent(t, tmpDir, ".stashignore", "\\#filename.mp4\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ @@ -457,7 +457,7 @@ func TestStashIgnore_CaseSensitiveMatching(t *testing.T) { // Pattern should match exactly (case-sensitive). createTestFileWithContent(t, tmpDir, ".stashignore", "video_lower.mp4\n") - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) // Only exact match is excluded. @@ -505,7 +505,7 @@ backup/ ` createTestFileWithContent(t, tmpDir, ".stashignore", stashignore) - filter := NewStashIgnoreFilter(tmpDir) + filter := NewStashIgnoreFilter() accepted := walkAndFilter(t, tmpDir, filter) expected := []string{ diff --git a/ui/v2.5/src/docs/en/Manual/Tasks.md b/ui/v2.5/src/docs/en/Manual/Tasks.md index aa46f72bb..4334f0898 100644 --- a/ui/v2.5/src/docs/en/Manual/Tasks.md +++ b/ui/v2.5/src/docs/en/Manual/Tasks.md @@ -10,6 +10,39 @@ Stash currently identifies files by performing a quick file hash. This means tha Stash currently ignores duplicate files. If two files contain identical content, only the first one it comes across is used. +### Ignoring Files with .stashignore + +You can create `.stashignore` files to exclude specific files or directories from being scanned. These files use gitignore-style pattern matching syntax. + +Place a `.stashignore` file in any directory within your library. The patterns in that file will apply to all files and subdirectories within that directory. You can have multiple `.stashignore` files at different levels of your directory hierarchy - patterns from parent directories cascade down to child directories. + +**Supported patterns:** + +| Pattern | Description | +|---------|-------------| +| `filename.mp4` | Ignore a specific file. | +| `*.tmp` | Ignore all files with a specific extension. | +| `temp/` | Ignore a directory and all its contents. | +| `**/cache/` | Ignore directories named "cache" at any level. | +| `!important.mp4` | Negation - do not ignore this file even if it matches a previous pattern. | +| `# comment` | Lines starting with # are comments. | +| `\#filename` | Use backslash to match a literal # character. | + +**Example .stashignore file:** + +``` +# Ignore temporary files +*.tmp +*.log + +# Ignore specific directories +temp/ +.thumbnails/ + +# But keep this specific file +!important.tmp +``` + The scan task accepts the following options: | Option | Description | From 7851a5884f05b8d655b165a029dbefa21cc0427d Mon Sep 17 00:00:00 2001 From: Matt Stone Date: Thu, 8 Jan 2026 19:28:13 +1000 Subject: [PATCH 3/6] Linting fix --- pkg/file/stashignore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/file/stashignore.go b/pkg/file/stashignore.go index db7b7d3fa..c00b7a6de 100644 --- a/pkg/file/stashignore.go +++ b/pkg/file/stashignore.go @@ -72,7 +72,7 @@ func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.Fil } entryRelPath = filepath.ToSlash(entryRelPath) if info.IsDir() { - entryRelPath = entryRelPath + "/" + entryRelPath += "/" } if entry.patterns.MatchesPath(entryRelPath) { From 5e56d17cb3c7dd37eed59ba710b1fc8ac0a72902 Mon Sep 17 00:00:00 2001 From: Matt Stone Date: Wed, 14 Jan 2026 19:52:51 +1000 Subject: [PATCH 4/6] Remove unnecessary file exclusion --- pkg/file/stashignore.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/file/stashignore.go b/pkg/file/stashignore.go index c00b7a6de..ccf1b4505 100644 --- a/pkg/file/stashignore.go +++ b/pkg/file/stashignore.go @@ -40,11 +40,6 @@ func NewStashIgnoreFilter() *StashIgnoreFilter { // The libraryRoot parameter bounds the search for .stashignore files - // only directories within the library root are checked. func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.FileInfo, libraryRoot string) bool { - // Always accept .stashignore files themselves so they can be read. - if filepath.Base(path) == stashIgnoreFilename { - return true - } - // If no library root provided, accept the file (safety fallback). if libraryRoot == "" { return true From b716297ee4f7749bddbd408f0a9c676b4c232abf Mon Sep 17 00:00:00 2001 From: Matt Stone Date: Wed, 14 Jan 2026 19:55:43 +1000 Subject: [PATCH 5/6] Logging for unreadable .stashignore --- pkg/file/stashignore.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/file/stashignore.go b/pkg/file/stashignore.go index ccf1b4505..be3571ca5 100644 --- a/pkg/file/stashignore.go +++ b/pkg/file/stashignore.go @@ -146,8 +146,15 @@ func (f *StashIgnoreFilter) getOrLoadIgnoreEntry(dir string) *ignoreEntry { // Try to load .stashignore from this directory. stashIgnorePath := filepath.Join(dir, stashIgnoreFilename) patterns, err := f.loadIgnoreFile(stashIgnorePath) - if err != nil || patterns == nil { - // Cache negative result (file doesn't exist or has no patterns). + if err != nil { + if !os.IsNotExist(err) { + logger.Warnf("Failed to load .stashignore from %s: %v", dir, err) + } + f.cache.Store(dir, &ignoreEntry{patterns: nil, dir: dir}) + return nil + } + if patterns == nil { + // File exists but has no patterns (empty or only comments). f.cache.Store(dir, &ignoreEntry{patterns: nil, dir: dir}) return nil } From 30698fa56b3b82f618cd51a7dd8b4c47f89e7fc0 Mon Sep 17 00:00:00 2001 From: Matt Stone Date: Thu, 15 Jan 2026 09:45:05 +1000 Subject: [PATCH 6/6] LRU cache optimisation --- pkg/file/stashignore.go | 64 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/pkg/file/stashignore.go b/pkg/file/stashignore.go index be3571ca5..160b5c224 100644 --- a/pkg/file/stashignore.go +++ b/pkg/file/stashignore.go @@ -8,17 +8,26 @@ import ( "strings" "sync" + lru "github.com/hashicorp/golang-lru/v2" ignore "github.com/sabhiram/go-gitignore" "github.com/stashapp/stash/pkg/logger" ) const stashIgnoreFilename = ".stashignore" +// entriesCacheSize is the size of the LRU cache for collected ignore entries. +// This cache stores the computed list of ignore entries per directory, avoiding +// repeated directory tree walks for files in the same directory. +const entriesCacheSize = 500 + // StashIgnoreFilter implements PathFilter to exclude files/directories // based on .stashignore files with gitignore-style patterns. type StashIgnoreFilter struct { // cache stores compiled ignore patterns per directory. cache sync.Map // map[string]*ignoreEntry + // entriesCache stores collected ignore entries per (dir, libraryRoot) pair. + // This avoids recomputing the entry list for every file in the same directory. + entriesCache *lru.Cache[string, []*ignoreEntry] } // ignoreEntry holds the compiled ignore patterns for a directory. @@ -31,7 +40,12 @@ type ignoreEntry struct { // NewStashIgnoreFilter creates a new StashIgnoreFilter. func NewStashIgnoreFilter() *StashIgnoreFilter { - return &StashIgnoreFilter{} + // Create the LRU cache for collected entries. + // Ignore error as it only fails if size <= 0. + entriesCache, _ := lru.New[string, []*ignoreEntry](entriesCacheSize) + return &StashIgnoreFilter{ + entriesCache: entriesCache, + } } // Accept returns true if the path should be included in the scan. @@ -80,16 +94,44 @@ func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.Fil // collectIgnoreEntries gathers all ignore entries from library root to the given directory. // It walks up the directory tree from dir to libraryRoot and returns entries in order -// from root to most specific. +// from root to most specific. Results are cached to avoid repeated computation for +// files in the same directory. func (f *StashIgnoreFilter) collectIgnoreEntries(dir string, libraryRoot string) []*ignoreEntry { - // Collect directories from library root down to current dir. - var dirs []string - - // Clean paths for consistent comparison. + // Clean paths for consistent comparison and cache key generation. dir = filepath.Clean(dir) libraryRoot = filepath.Clean(libraryRoot) + // Build cache key from dir and libraryRoot. + cacheKey := dir + "\x00" + libraryRoot + + // Check the entries cache first. + if cached, ok := f.entriesCache.Get(cacheKey); ok { + return cached + } + + // Try subdirectory shortcut: if parent's entries are cached, extend them. + if dir != libraryRoot { + parent := filepath.Dir(dir) + if isPathInOrEqual(libraryRoot, parent) { + parentKey := parent + "\x00" + libraryRoot + if parentEntries, ok := f.entriesCache.Get(parentKey); ok { + // Parent is cached - just check if current dir has a .stashignore. + entries := parentEntries + if entry := f.getOrLoadIgnoreEntry(dir); entry != nil { + // Copy parent slice and append to avoid mutating cached slice. + entries = make([]*ignoreEntry, len(parentEntries), len(parentEntries)+1) + copy(entries, parentEntries) + entries = append(entries, entry) + } + f.entriesCache.Add(cacheKey, entries) + return entries + } + } + } + + // No cache hit - compute from scratch. // Walk up from dir to library root, collecting directories. + var dirs []string current := dir for { // Check if we're still within the library root. @@ -97,7 +139,7 @@ func (f *StashIgnoreFilter) collectIgnoreEntries(dir string, libraryRoot string) break } - dirs = append([]string{current}, dirs...) // Prepend to maintain root-to-leaf order. + dirs = append(dirs, current) // Stop if we've reached the library root. if current == libraryRoot { @@ -112,6 +154,11 @@ func (f *StashIgnoreFilter) collectIgnoreEntries(dir string, libraryRoot string) current = parent } + // Reverse to get root-to-leaf order. + for i, j := 0, len(dirs)-1; i < j; i, j = i+1, j-1 { + dirs[i], dirs[j] = dirs[j], dirs[i] + } + // Check each directory for .stashignore files. var entries []*ignoreEntry for _, d := range dirs { @@ -120,6 +167,9 @@ func (f *StashIgnoreFilter) collectIgnoreEntries(dir string, libraryRoot string) } } + // Cache the result. + f.entriesCache.Add(cacheKey, entries) + return entries }