mirror of
https://github.com/stashapp/stash.git
synced 2026-02-07 16:05:47 +01:00
Merge 30698fa56b into 8dec195c2d
This commit is contained in:
Matt Stone
GitHub
commit
de507f0a49
7 changed files with 1335 additions and 0 deletions
1
go.mod
1
go.mod
|
|
@ -44,6 +44,7 @@ require (
|
|||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/spf13/cast v1.6.0
|
||||
github.com/spf13/pflag v1.0.6
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -537,6 +537,8 @@ github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK
|
|||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||
github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
|
||||
github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
|
|
|
|||
513
internal/manager/scan_stashignore_test.go
Normal file
513
internal/manager/scan_stashignore_test.go
Normal file
|
|
@ -0,0 +1,513 @@
|
|||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stashapp/stash/internal/manager/config"
|
||||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/sqlite"
|
||||
"github.com/stashapp/stash/pkg/txn"
|
||||
|
||||
// Necessary to register custom migrations.
|
||||
_ "github.com/stashapp/stash/pkg/sqlite/migrations"
|
||||
)
|
||||
|
||||
// mockFingerprintCalculator returns empty fingerprints.
|
||||
type mockFingerprintCalculator struct{}
|
||||
|
||||
func (m *mockFingerprintCalculator) CalculateFingerprints(f *models.BaseFile, o file.Opener, useExisting bool) ([]models.Fingerprint, error) {
|
||||
// Return a simple fingerprint based on path for testing.
|
||||
return []models.Fingerprint{
|
||||
{
|
||||
Type: models.FingerprintTypeMD5,
|
||||
Fingerprint: fmt.Sprintf("md5-%s", f.Basename),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// mockProgressReporter does nothing.
|
||||
type mockProgressReporter struct{}
|
||||
|
||||
func (m *mockProgressReporter) AddTotal(total int) {}
|
||||
func (m *mockProgressReporter) Increment() {}
|
||||
func (m *mockProgressReporter) Definite() {}
|
||||
func (m *mockProgressReporter) ExecuteTask(description string, fn func()) { fn() }
|
||||
|
||||
// stashIgnorePathFilter wraps StashIgnoreFilter to implement PathFilter for testing.
|
||||
// It provides a fixed library root for the filter.
|
||||
type stashIgnorePathFilter struct {
|
||||
filter *file.StashIgnoreFilter
|
||||
libraryRoot string
|
||||
}
|
||||
|
||||
func (f *stashIgnorePathFilter) Accept(ctx context.Context, path string, info fs.FileInfo) bool {
|
||||
return f.filter.Accept(ctx, path, info, f.libraryRoot)
|
||||
}
|
||||
|
||||
// createTestFileOnDisk creates a file with some content.
|
||||
func createTestFileOnDisk(t *testing.T, dir, name string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
t.Fatalf("failed to create directory for %s: %v", path, err)
|
||||
}
|
||||
// Write some content so the file has a non-zero size.
|
||||
if err := os.WriteFile(path, []byte("test content for "+name), 0644); err != nil {
|
||||
t.Fatalf("failed to create file %s: %v", path, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// createStashIgnoreFile creates a .stashignore file with the given content.
|
||||
func createStashIgnoreFile(t *testing.T, dir, content string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, ".stashignore")
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to create .stashignore: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// setupTestDatabase creates a temporary SQLite database for testing.
|
||||
func setupScanTestDatabase(t *testing.T) (*sqlite.Database, func()) {
|
||||
t.Helper()
|
||||
|
||||
// Initialise empty config - needed by some migrations.
|
||||
_ = config.InitializeEmpty()
|
||||
|
||||
// Create temporary database file.
|
||||
f, err := os.CreateTemp("", "stash-scan-test-*.sqlite")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create temp database file: %v", err)
|
||||
}
|
||||
f.Close()
|
||||
dbFile := f.Name()
|
||||
|
||||
db := sqlite.NewDatabase()
|
||||
db.SetBlobStoreOptions(sqlite.BlobStoreOptions{
|
||||
UseDatabase: true,
|
||||
})
|
||||
|
||||
if err := db.Open(dbFile); err != nil {
|
||||
os.Remove(dbFile)
|
||||
t.Fatalf("failed to open database: %v", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
db.Close()
|
||||
os.Remove(dbFile)
|
||||
}
|
||||
|
||||
return db, cleanup
|
||||
}
|
||||
|
||||
func TestScannerWithStashIgnore(t *testing.T) {
|
||||
// Setup test database.
|
||||
db, cleanup := setupScanTestDatabase(t)
|
||||
defer cleanup()
|
||||
|
||||
// Create temp directory structure.
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFileOnDisk(t, tmpDir, "video1.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "video2.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "ignore_me.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "subdir/video3.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "subdir/skip_this.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "excluded_dir/video4.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "temp/processing.mp4")
|
||||
|
||||
// Create .stashignore file.
|
||||
stashignore := `# Ignore specific files
|
||||
ignore_me.mp4
|
||||
subdir/skip_this.mp4
|
||||
|
||||
# Ignore directories
|
||||
excluded_dir/
|
||||
temp/
|
||||
`
|
||||
createStashIgnoreFile(t, tmpDir, stashignore)
|
||||
|
||||
// Create scanner.
|
||||
repo := file.NewRepository(db.Repository())
|
||||
scanner := &file.Scanner{
|
||||
FS: &file.OsFS{},
|
||||
Repository: repo,
|
||||
FingerprintCalculator: &mockFingerprintCalculator{},
|
||||
}
|
||||
|
||||
// Create stashignore filter with library root.
|
||||
stashIgnoreFilter := &stashIgnorePathFilter{
|
||||
filter: file.NewStashIgnoreFilter(),
|
||||
libraryRoot: tmpDir,
|
||||
}
|
||||
|
||||
// Run scan.
|
||||
ctx := context.Background()
|
||||
scanner.Scan(ctx, nil, file.ScanOptions{
|
||||
Paths: []string{tmpDir},
|
||||
ScanFilters: []file.PathFilter{stashIgnoreFilter},
|
||||
ParallelTasks: 1,
|
||||
}, &mockProgressReporter{})
|
||||
|
||||
// Verify results by checking what's in the database.
|
||||
var scannedPaths []string
|
||||
err := txn.WithTxn(ctx, db, func(ctx context.Context) error {
|
||||
// Check folders by path.
|
||||
checkDirs := []string{
|
||||
filepath.Join(tmpDir, "subdir"),
|
||||
filepath.Join(tmpDir, "excluded_dir"),
|
||||
filepath.Join(tmpDir, "temp"),
|
||||
}
|
||||
|
||||
for _, dir := range checkDirs {
|
||||
f, err := db.Folder.FindByPath(ctx, dir, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking folder %s: %w", dir, err)
|
||||
}
|
||||
if f != nil {
|
||||
relPath, _ := filepath.Rel(tmpDir, dir)
|
||||
scannedPaths = append(scannedPaths, "dir:"+relPath)
|
||||
}
|
||||
}
|
||||
|
||||
// Check specific files.
|
||||
checkFiles := []string{
|
||||
filepath.Join(tmpDir, "video1.mp4"),
|
||||
filepath.Join(tmpDir, "video2.mp4"),
|
||||
filepath.Join(tmpDir, "ignore_me.mp4"),
|
||||
filepath.Join(tmpDir, "subdir/video3.mp4"),
|
||||
filepath.Join(tmpDir, "subdir/skip_this.mp4"),
|
||||
filepath.Join(tmpDir, "excluded_dir/video4.mp4"),
|
||||
filepath.Join(tmpDir, "temp/processing.mp4"),
|
||||
}
|
||||
|
||||
for _, path := range checkFiles {
|
||||
f, err := db.File.FindByPath(ctx, path, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking file %s: %w", path, err)
|
||||
}
|
||||
if f != nil {
|
||||
relPath, _ := filepath.Rel(tmpDir, path)
|
||||
scannedPaths = append(scannedPaths, "file:"+relPath)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("failed to verify scan results: %v", err)
|
||||
}
|
||||
|
||||
sort.Strings(scannedPaths)
|
||||
|
||||
// Expected: video1.mp4, video2.mp4, subdir/video3.mp4, and their folders.
|
||||
// NOT expected: ignore_me.mp4, subdir/skip_this.mp4, excluded_dir/*, temp/*.
|
||||
expectedPaths := []string{
|
||||
"dir:subdir",
|
||||
"file:subdir/video3.mp4",
|
||||
"file:video1.mp4",
|
||||
"file:video2.mp4",
|
||||
}
|
||||
sort.Strings(expectedPaths)
|
||||
|
||||
if len(scannedPaths) != len(expectedPaths) {
|
||||
t.Errorf("scanned path count mismatch:\nexpected %d: %v\nactual %d: %v",
|
||||
len(expectedPaths), expectedPaths, len(scannedPaths), scannedPaths)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range expectedPaths {
|
||||
if scannedPaths[i] != expectedPaths[i] {
|
||||
t.Errorf("path mismatch at index %d:\nexpected: %s\nactual: %s",
|
||||
i, expectedPaths[i], scannedPaths[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerWithNestedStashIgnore(t *testing.T) {
|
||||
// Setup test database.
|
||||
db, cleanup := setupScanTestDatabase(t)
|
||||
defer cleanup()
|
||||
|
||||
// Create temp directory structure.
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFileOnDisk(t, tmpDir, "root.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "root.tmp")
|
||||
createTestFileOnDisk(t, tmpDir, "subdir/sub.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "subdir/sub.log")
|
||||
createTestFileOnDisk(t, tmpDir, "subdir/sub.tmp")
|
||||
|
||||
// Root .stashignore excludes *.tmp.
|
||||
createStashIgnoreFile(t, tmpDir, "*.tmp\n")
|
||||
|
||||
// Subdir .stashignore excludes *.log.
|
||||
createStashIgnoreFile(t, filepath.Join(tmpDir, "subdir"), "*.log\n")
|
||||
|
||||
// Create scanner.
|
||||
repo := file.NewRepository(db.Repository())
|
||||
scanner := &file.Scanner{
|
||||
FS: &file.OsFS{},
|
||||
Repository: repo,
|
||||
FingerprintCalculator: &mockFingerprintCalculator{},
|
||||
}
|
||||
|
||||
// Create stashignore filter with library root.
|
||||
stashIgnoreFilter := &stashIgnorePathFilter{
|
||||
filter: file.NewStashIgnoreFilter(),
|
||||
libraryRoot: tmpDir,
|
||||
}
|
||||
|
||||
// Run scan.
|
||||
ctx := context.Background()
|
||||
scanner.Scan(ctx, nil, file.ScanOptions{
|
||||
Paths: []string{tmpDir},
|
||||
ScanFilters: []file.PathFilter{stashIgnoreFilter},
|
||||
ParallelTasks: 1,
|
||||
}, &mockProgressReporter{})
|
||||
|
||||
// Verify results.
|
||||
var scannedFiles []string
|
||||
err := txn.WithTxn(ctx, db, func(ctx context.Context) error {
|
||||
checkFiles := []string{
|
||||
filepath.Join(tmpDir, "root.mp4"),
|
||||
filepath.Join(tmpDir, "root.tmp"),
|
||||
filepath.Join(tmpDir, "subdir/sub.mp4"),
|
||||
filepath.Join(tmpDir, "subdir/sub.log"),
|
||||
filepath.Join(tmpDir, "subdir/sub.tmp"),
|
||||
}
|
||||
|
||||
for _, path := range checkFiles {
|
||||
f, err := db.File.FindByPath(ctx, path, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking file %s: %w", path, err)
|
||||
}
|
||||
if f != nil {
|
||||
relPath, _ := filepath.Rel(tmpDir, path)
|
||||
scannedFiles = append(scannedFiles, relPath)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("failed to verify scan results: %v", err)
|
||||
}
|
||||
|
||||
sort.Strings(scannedFiles)
|
||||
|
||||
// Expected: root.mp4, subdir/sub.mp4.
|
||||
// NOT expected: root.tmp (root ignore), subdir/sub.log (subdir ignore), subdir/sub.tmp (root ignore).
|
||||
expectedFiles := []string{
|
||||
"root.mp4",
|
||||
"subdir/sub.mp4",
|
||||
}
|
||||
sort.Strings(expectedFiles)
|
||||
|
||||
if len(scannedFiles) != len(expectedFiles) {
|
||||
t.Errorf("scanned file count mismatch:\nexpected %d: %v\nactual %d: %v",
|
||||
len(expectedFiles), expectedFiles, len(scannedFiles), scannedFiles)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range expectedFiles {
|
||||
if scannedFiles[i] != expectedFiles[i] {
|
||||
t.Errorf("file mismatch at index %d:\nexpected: %s\nactual: %s",
|
||||
i, expectedFiles[i], scannedFiles[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerWithoutStashIgnore(t *testing.T) {
|
||||
// Setup test database.
|
||||
db, cleanup := setupScanTestDatabase(t)
|
||||
defer cleanup()
|
||||
|
||||
// Create temp directory structure (no .stashignore).
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFileOnDisk(t, tmpDir, "video1.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "video2.mp4")
|
||||
createTestFileOnDisk(t, tmpDir, "subdir/video3.mp4")
|
||||
|
||||
// Create scanner.
|
||||
repo := file.NewRepository(db.Repository())
|
||||
scanner := &file.Scanner{
|
||||
FS: &file.OsFS{},
|
||||
Repository: repo,
|
||||
FingerprintCalculator: &mockFingerprintCalculator{},
|
||||
}
|
||||
|
||||
// Create stashignore filter with library root (but no .stashignore file exists).
|
||||
stashIgnoreFilter := &stashIgnorePathFilter{
|
||||
filter: file.NewStashIgnoreFilter(),
|
||||
libraryRoot: tmpDir,
|
||||
}
|
||||
|
||||
// Run scan.
|
||||
ctx := context.Background()
|
||||
scanner.Scan(ctx, nil, file.ScanOptions{
|
||||
Paths: []string{tmpDir},
|
||||
ScanFilters: []file.PathFilter{stashIgnoreFilter},
|
||||
ParallelTasks: 1,
|
||||
}, &mockProgressReporter{})
|
||||
|
||||
// Verify all files were scanned.
|
||||
var scannedFiles []string
|
||||
err := txn.WithTxn(ctx, db, func(ctx context.Context) error {
|
||||
checkFiles := []string{
|
||||
filepath.Join(tmpDir, "video1.mp4"),
|
||||
filepath.Join(tmpDir, "video2.mp4"),
|
||||
filepath.Join(tmpDir, "subdir/video3.mp4"),
|
||||
}
|
||||
|
||||
for _, path := range checkFiles {
|
||||
f, err := db.File.FindByPath(ctx, path, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking file %s: %w", path, err)
|
||||
}
|
||||
if f != nil {
|
||||
relPath, _ := filepath.Rel(tmpDir, path)
|
||||
scannedFiles = append(scannedFiles, relPath)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("failed to verify scan results: %v", err)
|
||||
}
|
||||
|
||||
sort.Strings(scannedFiles)
|
||||
|
||||
// All files should be scanned.
|
||||
expectedFiles := []string{
|
||||
"subdir/video3.mp4",
|
||||
"video1.mp4",
|
||||
"video2.mp4",
|
||||
}
|
||||
sort.Strings(expectedFiles)
|
||||
|
||||
if len(scannedFiles) != len(expectedFiles) {
|
||||
t.Errorf("scanned file count mismatch:\nexpected %d: %v\nactual %d: %v",
|
||||
len(expectedFiles), expectedFiles, len(scannedFiles), scannedFiles)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range expectedFiles {
|
||||
if scannedFiles[i] != expectedFiles[i] {
|
||||
t.Errorf("file mismatch at index %d:\nexpected: %s\nactual: %s",
|
||||
i, expectedFiles[i], scannedFiles[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerWithNegationPattern(t *testing.T) {
|
||||
// Setup test database.
|
||||
db, cleanup := setupScanTestDatabase(t)
|
||||
defer cleanup()
|
||||
|
||||
// Create temp directory structure.
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFileOnDisk(t, tmpDir, "file1.tmp")
|
||||
createTestFileOnDisk(t, tmpDir, "file2.tmp")
|
||||
createTestFileOnDisk(t, tmpDir, "keep_this.tmp")
|
||||
createTestFileOnDisk(t, tmpDir, "video.mp4")
|
||||
|
||||
// Create .stashignore with negation.
|
||||
stashignore := `*.tmp
|
||||
!keep_this.tmp
|
||||
`
|
||||
createStashIgnoreFile(t, tmpDir, stashignore)
|
||||
|
||||
// Create scanner.
|
||||
repo := file.NewRepository(db.Repository())
|
||||
scanner := &file.Scanner{
|
||||
FS: &file.OsFS{},
|
||||
Repository: repo,
|
||||
FingerprintCalculator: &mockFingerprintCalculator{},
|
||||
}
|
||||
|
||||
// Create stashignore filter with library root.
|
||||
stashIgnoreFilter := &stashIgnorePathFilter{
|
||||
filter: file.NewStashIgnoreFilter(),
|
||||
libraryRoot: tmpDir,
|
||||
}
|
||||
|
||||
// Run scan.
|
||||
ctx := context.Background()
|
||||
scanner.Scan(ctx, nil, file.ScanOptions{
|
||||
Paths: []string{tmpDir},
|
||||
ScanFilters: []file.PathFilter{stashIgnoreFilter},
|
||||
ParallelTasks: 1,
|
||||
}, &mockProgressReporter{})
|
||||
|
||||
// Verify results.
|
||||
var scannedFiles []string
|
||||
err := txn.WithTxn(ctx, db, func(ctx context.Context) error {
|
||||
checkFiles := []string{
|
||||
filepath.Join(tmpDir, "file1.tmp"),
|
||||
filepath.Join(tmpDir, "file2.tmp"),
|
||||
filepath.Join(tmpDir, "keep_this.tmp"),
|
||||
filepath.Join(tmpDir, "video.mp4"),
|
||||
}
|
||||
|
||||
for _, path := range checkFiles {
|
||||
f, err := db.File.FindByPath(ctx, path, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking file %s: %w", path, err)
|
||||
}
|
||||
if f != nil {
|
||||
relPath, _ := filepath.Rel(tmpDir, path)
|
||||
scannedFiles = append(scannedFiles, relPath)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("failed to verify scan results: %v", err)
|
||||
}
|
||||
|
||||
sort.Strings(scannedFiles)
|
||||
|
||||
// Expected: keep_this.tmp (negated), video.mp4.
|
||||
// NOT expected: file1.tmp, file2.tmp.
|
||||
expectedFiles := []string{
|
||||
"keep_this.tmp",
|
||||
"video.mp4",
|
||||
}
|
||||
sort.Strings(expectedFiles)
|
||||
|
||||
if len(scannedFiles) != len(expectedFiles) {
|
||||
t.Errorf("scanned file count mismatch:\nexpected %d: %v\nactual %d: %v",
|
||||
len(expectedFiles), expectedFiles, len(scannedFiles), scannedFiles)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range expectedFiles {
|
||||
if scannedFiles[i] != expectedFiles[i] {
|
||||
t.Errorf("file mismatch at index %d:\nexpected: %s\nactual: %s",
|
||||
i, expectedFiles[i], scannedFiles[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -515,6 +515,7 @@ type scanFilter struct {
|
|||
videoExcludeRegex []*regexp.Regexp
|
||||
imageExcludeRegex []*regexp.Regexp
|
||||
minModTime time.Time
|
||||
stashIgnoreFilter *file.StashIgnoreFilter
|
||||
}
|
||||
|
||||
func newScanFilter(c *config.Config, repo models.Repository, minModTime time.Time) *scanFilter {
|
||||
|
|
@ -528,6 +529,7 @@ func newScanFilter(c *config.Config, repo models.Repository, minModTime time.Tim
|
|||
videoExcludeRegex: generateRegexps(c.GetExcludes()),
|
||||
imageExcludeRegex: generateRegexps(c.GetImageExcludes()),
|
||||
minModTime: minModTime,
|
||||
stashIgnoreFilter: file.NewStashIgnoreFilter(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -548,6 +550,12 @@ func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo)
|
|||
return false
|
||||
}
|
||||
|
||||
// Check .stashignore files, bounded to the library root.
|
||||
if !f.stashIgnoreFilter.Accept(ctx, path, info, s.Path) {
|
||||
logger.Debugf("Skipping %s due to .stashignore", path)
|
||||
return false
|
||||
}
|
||||
|
||||
isVideoFile := useAsVideo(path)
|
||||
isImageFile := useAsImage(path)
|
||||
isZipFile := fsutil.MatchExtension(path, f.zipExt)
|
||||
|
|
|
|||
255
pkg/file/stashignore.go
Normal file
255
pkg/file/stashignore.go
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
)
|
||||
|
||||
const stashIgnoreFilename = ".stashignore"
|
||||
|
||||
// entriesCacheSize is the size of the LRU cache for collected ignore entries.
|
||||
// This cache stores the computed list of ignore entries per directory, avoiding
|
||||
// repeated directory tree walks for files in the same directory.
|
||||
const entriesCacheSize = 500
|
||||
|
||||
// StashIgnoreFilter implements PathFilter to exclude files/directories
|
||||
// based on .stashignore files with gitignore-style patterns.
|
||||
type StashIgnoreFilter struct {
|
||||
// cache stores compiled ignore patterns per directory.
|
||||
cache sync.Map // map[string]*ignoreEntry
|
||||
// entriesCache stores collected ignore entries per (dir, libraryRoot) pair.
|
||||
// This avoids recomputing the entry list for every file in the same directory.
|
||||
entriesCache *lru.Cache[string, []*ignoreEntry]
|
||||
}
|
||||
|
||||
// ignoreEntry holds the compiled ignore patterns for a directory.
|
||||
type ignoreEntry struct {
|
||||
// patterns is the compiled gitignore matcher for this directory.
|
||||
patterns *ignore.GitIgnore
|
||||
// dir is the directory this entry applies to.
|
||||
dir string
|
||||
}
|
||||
|
||||
// NewStashIgnoreFilter creates a new StashIgnoreFilter.
|
||||
func NewStashIgnoreFilter() *StashIgnoreFilter {
|
||||
// Create the LRU cache for collected entries.
|
||||
// Ignore error as it only fails if size <= 0.
|
||||
entriesCache, _ := lru.New[string, []*ignoreEntry](entriesCacheSize)
|
||||
return &StashIgnoreFilter{
|
||||
entriesCache: entriesCache,
|
||||
}
|
||||
}
|
||||
|
||||
// Accept returns true if the path should be included in the scan.
|
||||
// It checks for .stashignore files in the directory hierarchy and
|
||||
// applies gitignore-style pattern matching.
|
||||
// The libraryRoot parameter bounds the search for .stashignore files -
|
||||
// only directories within the library root are checked.
|
||||
func (f *StashIgnoreFilter) Accept(ctx context.Context, path string, info fs.FileInfo, libraryRoot string) bool {
|
||||
// If no library root provided, accept the file (safety fallback).
|
||||
if libraryRoot == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Get the directory containing this path.
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
// Collect all applicable ignore entries from library root to this directory.
|
||||
entries := f.collectIgnoreEntries(dir, libraryRoot)
|
||||
|
||||
// If no .stashignore files found, accept the file.
|
||||
if len(entries) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check each ignore entry in order (from root to most specific).
|
||||
// Later entries can override earlier ones with negation patterns.
|
||||
ignored := false
|
||||
for _, entry := range entries {
|
||||
// Get path relative to the ignore file's directory.
|
||||
entryRelPath, err := filepath.Rel(entry.dir, path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
entryRelPath = filepath.ToSlash(entryRelPath)
|
||||
if info.IsDir() {
|
||||
entryRelPath += "/"
|
||||
}
|
||||
|
||||
if entry.patterns.MatchesPath(entryRelPath) {
|
||||
ignored = true
|
||||
}
|
||||
}
|
||||
|
||||
return !ignored
|
||||
}
|
||||
|
||||
// collectIgnoreEntries gathers all ignore entries from library root to the given directory.
|
||||
// It walks up the directory tree from dir to libraryRoot and returns entries in order
|
||||
// from root to most specific. Results are cached to avoid repeated computation for
|
||||
// files in the same directory.
|
||||
func (f *StashIgnoreFilter) collectIgnoreEntries(dir string, libraryRoot string) []*ignoreEntry {
|
||||
// Clean paths for consistent comparison and cache key generation.
|
||||
dir = filepath.Clean(dir)
|
||||
libraryRoot = filepath.Clean(libraryRoot)
|
||||
|
||||
// Build cache key from dir and libraryRoot.
|
||||
cacheKey := dir + "\x00" + libraryRoot
|
||||
|
||||
// Check the entries cache first.
|
||||
if cached, ok := f.entriesCache.Get(cacheKey); ok {
|
||||
return cached
|
||||
}
|
||||
|
||||
// Try subdirectory shortcut: if parent's entries are cached, extend them.
|
||||
if dir != libraryRoot {
|
||||
parent := filepath.Dir(dir)
|
||||
if isPathInOrEqual(libraryRoot, parent) {
|
||||
parentKey := parent + "\x00" + libraryRoot
|
||||
if parentEntries, ok := f.entriesCache.Get(parentKey); ok {
|
||||
// Parent is cached - just check if current dir has a .stashignore.
|
||||
entries := parentEntries
|
||||
if entry := f.getOrLoadIgnoreEntry(dir); entry != nil {
|
||||
// Copy parent slice and append to avoid mutating cached slice.
|
||||
entries = make([]*ignoreEntry, len(parentEntries), len(parentEntries)+1)
|
||||
copy(entries, parentEntries)
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
f.entriesCache.Add(cacheKey, entries)
|
||||
return entries
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No cache hit - compute from scratch.
|
||||
// Walk up from dir to library root, collecting directories.
|
||||
var dirs []string
|
||||
current := dir
|
||||
for {
|
||||
// Check if we're still within the library root.
|
||||
if !isPathInOrEqual(libraryRoot, current) {
|
||||
break
|
||||
}
|
||||
|
||||
dirs = append(dirs, current)
|
||||
|
||||
// Stop if we've reached the library root.
|
||||
if current == libraryRoot {
|
||||
break
|
||||
}
|
||||
|
||||
parent := filepath.Dir(current)
|
||||
if parent == current {
|
||||
// Reached filesystem root without finding library root.
|
||||
break
|
||||
}
|
||||
current = parent
|
||||
}
|
||||
|
||||
// Reverse to get root-to-leaf order.
|
||||
for i, j := 0, len(dirs)-1; i < j; i, j = i+1, j-1 {
|
||||
dirs[i], dirs[j] = dirs[j], dirs[i]
|
||||
}
|
||||
|
||||
// Check each directory for .stashignore files.
|
||||
var entries []*ignoreEntry
|
||||
for _, d := range dirs {
|
||||
if entry := f.getOrLoadIgnoreEntry(d); entry != nil {
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
}
|
||||
|
||||
// Cache the result.
|
||||
f.entriesCache.Add(cacheKey, entries)
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
// isPathInOrEqual checks if path is equal to or inside root.
|
||||
func isPathInOrEqual(root, path string) bool {
|
||||
if path == root {
|
||||
return true
|
||||
}
|
||||
// Check if path starts with root + separator.
|
||||
return strings.HasPrefix(path, root+string(filepath.Separator))
|
||||
}
|
||||
|
||||
// getOrLoadIgnoreEntry returns the cached ignore entry for a directory, or loads it.
|
||||
func (f *StashIgnoreFilter) getOrLoadIgnoreEntry(dir string) *ignoreEntry {
|
||||
// Check cache first.
|
||||
if cached, ok := f.cache.Load(dir); ok {
|
||||
entry := cached.(*ignoreEntry)
|
||||
if entry.patterns == nil {
|
||||
return nil // Cached negative result.
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
// Try to load .stashignore from this directory.
|
||||
stashIgnorePath := filepath.Join(dir, stashIgnoreFilename)
|
||||
patterns, err := f.loadIgnoreFile(stashIgnorePath)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
logger.Warnf("Failed to load .stashignore from %s: %v", dir, err)
|
||||
}
|
||||
f.cache.Store(dir, &ignoreEntry{patterns: nil, dir: dir})
|
||||
return nil
|
||||
}
|
||||
if patterns == nil {
|
||||
// File exists but has no patterns (empty or only comments).
|
||||
f.cache.Store(dir, &ignoreEntry{patterns: nil, dir: dir})
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Debugf("Loaded .stashignore from %s", dir)
|
||||
|
||||
entry := &ignoreEntry{
|
||||
patterns: patterns,
|
||||
dir: dir,
|
||||
}
|
||||
f.cache.Store(dir, entry)
|
||||
return entry
|
||||
}
|
||||
|
||||
// loadIgnoreFile loads and compiles a .stashignore file.
|
||||
func (f *StashIgnoreFilter) loadIgnoreFile(path string) (*ignore.GitIgnore, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
lines := strings.Split(string(data), "\n")
|
||||
var patterns []string
|
||||
|
||||
for _, line := range lines {
|
||||
// Trim trailing whitespace (but preserve leading for patterns).
|
||||
line = strings.TrimRight(line, " \t\r")
|
||||
|
||||
// Skip empty lines.
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip comments (but not escaped #).
|
||||
if strings.HasPrefix(line, "#") && !strings.HasPrefix(line, "\\#") {
|
||||
continue
|
||||
}
|
||||
|
||||
patterns = append(patterns, line)
|
||||
}
|
||||
|
||||
if len(patterns) == 0 {
|
||||
// File exists but has no patterns (e.g., only comments).
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return ignore.CompileIgnoreLines(patterns...), nil
|
||||
}
|
||||
523
pkg/file/stashignore_test.go
Normal file
523
pkg/file/stashignore_test.go
Normal file
|
|
@ -0,0 +1,523 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Helper to create an empty file.
|
||||
func createTestFile(t *testing.T, dir, name string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
t.Fatalf("failed to create directory for %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte{}, 0644); err != nil {
|
||||
t.Fatalf("failed to create file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to create a file with content.
|
||||
func createTestFileWithContent(t *testing.T, dir, name, content string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
t.Fatalf("failed to create directory for %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to create file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to create a directory.
|
||||
func createTestDir(t *testing.T, dir, name string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
t.Fatalf("failed to create directory %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// walkAndFilter walks the directory tree and returns paths accepted by the filter.
|
||||
// Returns paths relative to root for easier assertion.
|
||||
func walkAndFilter(t *testing.T, root string, filter *StashIgnoreFilter) []string {
|
||||
t.Helper()
|
||||
var accepted []string
|
||||
ctx := context.Background()
|
||||
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Skip the root directory itself.
|
||||
if path == root {
|
||||
return nil
|
||||
}
|
||||
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if filter.Accept(ctx, path, info, root) {
|
||||
relPath, _ := filepath.Rel(root, path)
|
||||
accepted = append(accepted, relPath)
|
||||
} else if info.IsDir() {
|
||||
// If directory is rejected, skip it.
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("walk failed: %v", err)
|
||||
}
|
||||
|
||||
sort.Strings(accepted)
|
||||
return accepted
|
||||
}
|
||||
|
||||
// assertPathsEqual checks that the accepted paths match expected.
|
||||
func assertPathsEqual(t *testing.T, expected, actual []string) {
|
||||
t.Helper()
|
||||
sort.Strings(expected)
|
||||
|
||||
if len(expected) != len(actual) {
|
||||
t.Errorf("path count mismatch:\nexpected %d: %v\nactual %d: %v", len(expected), expected, len(actual), actual)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range expected {
|
||||
if expected[i] != actual[i] {
|
||||
t.Errorf("path mismatch at index %d:\nexpected: %s\nactual: %s", i, expected[i], actual[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStashIgnore_ExactFilename(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestFile(t, tmpDir, "video2.mp4")
|
||||
createTestFile(t, tmpDir, "ignore_me.mp4")
|
||||
|
||||
// Create .stashignore that excludes exact filename.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "ignore_me.mp4\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"video1.mp4",
|
||||
"video2.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_WildcardPattern(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestFile(t, tmpDir, "video2.mp4")
|
||||
createTestFile(t, tmpDir, "temp1.tmp")
|
||||
createTestFile(t, tmpDir, "temp2.tmp")
|
||||
createTestFile(t, tmpDir, "notes.log")
|
||||
|
||||
// Create .stashignore that excludes by extension.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n*.log\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"video1.mp4",
|
||||
"video2.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_DirectoryExclusion(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestDir(t, tmpDir, "excluded_dir")
|
||||
createTestFile(t, tmpDir, "excluded_dir/video2.mp4")
|
||||
createTestFile(t, tmpDir, "excluded_dir/video3.mp4")
|
||||
createTestDir(t, tmpDir, "included_dir")
|
||||
createTestFile(t, tmpDir, "included_dir/video4.mp4")
|
||||
|
||||
// Create .stashignore that excludes a directory.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "excluded_dir/\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"included_dir",
|
||||
"included_dir/video4.mp4",
|
||||
"video1.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_NegationPattern(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "file1.tmp")
|
||||
createTestFile(t, tmpDir, "file2.tmp")
|
||||
createTestFile(t, tmpDir, "keep_this.tmp")
|
||||
|
||||
// Create .stashignore that excludes *.tmp but keeps one.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n!keep_this.tmp\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"keep_this.tmp",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_CommentsAndEmptyLines(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestFile(t, tmpDir, "ignore_me.mp4")
|
||||
|
||||
// Create .stashignore with comments and empty lines.
|
||||
stashignore := `# This is a comment
|
||||
ignore_me.mp4
|
||||
|
||||
# Another comment
|
||||
|
||||
`
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", stashignore)
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"video1.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_NestedStashIgnoreFiles(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "root_video.mp4")
|
||||
createTestFile(t, tmpDir, "root_ignore.tmp")
|
||||
createTestDir(t, tmpDir, "subdir")
|
||||
createTestFile(t, tmpDir, "subdir/sub_video.mp4")
|
||||
createTestFile(t, tmpDir, "subdir/sub_ignore.log")
|
||||
createTestFile(t, tmpDir, "subdir/also_tmp.tmp")
|
||||
|
||||
// Root .stashignore excludes *.tmp.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n")
|
||||
|
||||
// Subdir .stashignore excludes *.log.
|
||||
createTestFileWithContent(t, tmpDir, "subdir/.stashignore", "*.log\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
// *.tmp from root should apply everywhere.
|
||||
// *.log from subdir should only apply in subdir.
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"root_video.mp4",
|
||||
"subdir",
|
||||
"subdir/.stashignore",
|
||||
"subdir/sub_video.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_PathPattern(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestDir(t, tmpDir, "subdir")
|
||||
createTestFile(t, tmpDir, "subdir/video2.mp4")
|
||||
createTestFile(t, tmpDir, "subdir/skip_this.mp4")
|
||||
|
||||
// Create .stashignore that excludes a specific path.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "subdir/skip_this.mp4\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"subdir",
|
||||
"subdir/video2.mp4",
|
||||
"video1.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_DoubleStarPattern(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestDir(t, tmpDir, "a")
|
||||
createTestFile(t, tmpDir, "a/video2.mp4")
|
||||
createTestDir(t, tmpDir, "a/temp")
|
||||
createTestFile(t, tmpDir, "a/temp/video3.mp4")
|
||||
createTestDir(t, tmpDir, "a/b")
|
||||
createTestDir(t, tmpDir, "a/b/temp")
|
||||
createTestFile(t, tmpDir, "a/b/temp/video4.mp4")
|
||||
|
||||
// Create .stashignore that excludes temp directories at any level.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "**/temp/\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"a",
|
||||
"a/b",
|
||||
"a/video2.mp4",
|
||||
"video1.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_LeadingSlashPattern(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "ignore.mp4")
|
||||
createTestDir(t, tmpDir, "subdir")
|
||||
createTestFile(t, tmpDir, "subdir/ignore.mp4")
|
||||
|
||||
// Create .stashignore that excludes only at root level.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "/ignore.mp4\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
// Only root ignore.mp4 should be excluded.
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"subdir",
|
||||
"subdir/ignore.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_NoStashIgnoreFile(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files without any .stashignore.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestFile(t, tmpDir, "video2.mp4")
|
||||
createTestDir(t, tmpDir, "subdir")
|
||||
createTestFile(t, tmpDir, "subdir/video3.mp4")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
// All files should be accepted.
|
||||
expected := []string{
|
||||
"subdir",
|
||||
"subdir/video3.mp4",
|
||||
"video1.mp4",
|
||||
"video2.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_HiddenDirectories(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files including hidden directory.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestDir(t, tmpDir, ".hidden")
|
||||
createTestFile(t, tmpDir, ".hidden/video2.mp4")
|
||||
|
||||
// Create .stashignore that excludes hidden directories.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", ".*\n!.stashignore\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"video1.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_MultiplePatternsSameLine(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestFile(t, tmpDir, "file.tmp")
|
||||
createTestFile(t, tmpDir, "file.log")
|
||||
createTestFile(t, tmpDir, "file.bak")
|
||||
|
||||
// Each pattern should be on its own line.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "*.tmp\n*.log\n*.bak\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"video1.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_TrailingSpaces(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestFile(t, tmpDir, "ignore_me.mp4")
|
||||
|
||||
// Pattern with trailing spaces (should be trimmed).
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "ignore_me.mp4 \n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"video1.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_EscapedHash(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestFile(t, tmpDir, "#filename.mp4")
|
||||
|
||||
// Escaped hash should match literal # character.
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "\\#filename.mp4\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"video1.mp4",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_CaseSensitiveMatching(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create test files - use distinct names that work on all filesystems.
|
||||
createTestFile(t, tmpDir, "video_lower.mp4")
|
||||
createTestFile(t, tmpDir, "VIDEO_UPPER.mp4")
|
||||
createTestFile(t, tmpDir, "other.avi")
|
||||
|
||||
// Pattern should match exactly (case-sensitive).
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", "video_lower.mp4\n")
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
// Only exact match is excluded.
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"VIDEO_UPPER.mp4",
|
||||
"other.avi",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
||||
func TestStashIgnore_ComplexScenario(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Create a complex directory structure.
|
||||
createTestFile(t, tmpDir, "video1.mp4")
|
||||
createTestFile(t, tmpDir, "video2.avi")
|
||||
createTestFile(t, tmpDir, "thumbnail.jpg")
|
||||
createTestFile(t, tmpDir, "metadata.nfo")
|
||||
createTestDir(t, tmpDir, "movies")
|
||||
createTestFile(t, tmpDir, "movies/movie1.mp4")
|
||||
createTestFile(t, tmpDir, "movies/movie1.nfo")
|
||||
createTestDir(t, tmpDir, "movies/.thumbnails")
|
||||
createTestFile(t, tmpDir, "movies/.thumbnails/thumb1.jpg")
|
||||
createTestDir(t, tmpDir, "temp")
|
||||
createTestFile(t, tmpDir, "temp/processing.mp4")
|
||||
createTestDir(t, tmpDir, "backup")
|
||||
createTestFile(t, tmpDir, "backup/video1.mp4.bak")
|
||||
|
||||
// Complex .stashignore.
|
||||
stashignore := `# Ignore metadata files
|
||||
*.nfo
|
||||
|
||||
# Ignore hidden directories
|
||||
.*
|
||||
!.stashignore
|
||||
|
||||
# Ignore temp and backup directories
|
||||
temp/
|
||||
backup/
|
||||
|
||||
# But keep thumbnails in specific location
|
||||
!movies/.thumbnails/
|
||||
`
|
||||
createTestFileWithContent(t, tmpDir, ".stashignore", stashignore)
|
||||
|
||||
filter := NewStashIgnoreFilter()
|
||||
accepted := walkAndFilter(t, tmpDir, filter)
|
||||
|
||||
expected := []string{
|
||||
".stashignore",
|
||||
"movies",
|
||||
"movies/.thumbnails",
|
||||
"movies/.thumbnails/thumb1.jpg",
|
||||
"movies/movie1.mp4",
|
||||
"thumbnail.jpg",
|
||||
"video1.mp4",
|
||||
"video2.avi",
|
||||
}
|
||||
|
||||
assertPathsEqual(t, expected, accepted)
|
||||
}
|
||||
|
|
@ -10,6 +10,39 @@ Stash currently identifies files by performing a quick file hash. This means tha
|
|||
|
||||
Stash currently ignores duplicate files. If two files contain identical content, only the first one it comes across is used.
|
||||
|
||||
### Ignoring Files with .stashignore
|
||||
|
||||
You can create `.stashignore` files to exclude specific files or directories from being scanned. These files use gitignore-style pattern matching syntax.
|
||||
|
||||
Place a `.stashignore` file in any directory within your library. The patterns in that file will apply to all files and subdirectories within that directory. You can have multiple `.stashignore` files at different levels of your directory hierarchy - patterns from parent directories cascade down to child directories.
|
||||
|
||||
**Supported patterns:**
|
||||
|
||||
| Pattern | Description |
|
||||
|---------|-------------|
|
||||
| `filename.mp4` | Ignore a specific file. |
|
||||
| `*.tmp` | Ignore all files with a specific extension. |
|
||||
| `temp/` | Ignore a directory and all its contents. |
|
||||
| `**/cache/` | Ignore directories named "cache" at any level. |
|
||||
| `!important.mp4` | Negation - do not ignore this file even if it matches a previous pattern. |
|
||||
| `# comment` | Lines starting with # are comments. |
|
||||
| `\#filename` | Use backslash to match a literal # character. |
|
||||
|
||||
**Example .stashignore file:**
|
||||
|
||||
```
|
||||
# Ignore temporary files
|
||||
*.tmp
|
||||
*.log
|
||||
|
||||
# Ignore specific directories
|
||||
temp/
|
||||
.thumbnails/
|
||||
|
||||
# But keep this specific file
|
||||
!important.tmp
|
||||
```
|
||||
|
||||
The scan task accepts the following options:
|
||||
|
||||
| Option | Description |
|
||||
|
|
|
|||
Loading…
Reference in a new issue