From 52dd0197ee21aaf37ec294ae90c287e9fef3e481 Mon Sep 17 00:00:00 2001 From: bnkai <48220860+bnkai@users.noreply.github.com> Date: Tue, 24 Dec 2019 04:06:07 +0200 Subject: [PATCH] Added exclude patterns support for Clean Task (#274) * Added exclude patterns support for Clean Task * Added test file * Refactoring and cosmetic fixes * * Replace Match with MatchString --- pkg/manager/exclude_files.go | 89 ++++++++++++++++++ pkg/manager/exclude_files_test.go | 91 +++++++++++++++++++ pkg/manager/manager_tasks.go | 56 +----------- pkg/manager/manager_tasks_test.go | 66 -------------- pkg/manager/task_clean.go | 6 +- .../Settings/SettingsConfigurationPanel.tsx | 13 ++- 6 files changed, 202 insertions(+), 119 deletions(-) create mode 100644 pkg/manager/exclude_files.go create mode 100644 pkg/manager/exclude_files_test.go delete mode 100644 pkg/manager/manager_tasks_test.go diff --git a/pkg/manager/exclude_files.go b/pkg/manager/exclude_files.go new file mode 100644 index 000000000..818cb25f1 --- /dev/null +++ b/pkg/manager/exclude_files.go @@ -0,0 +1,89 @@ +package manager + +import ( + "github.com/stashapp/stash/pkg/logger" + "regexp" + "strings" +) + +func excludeFiles(files []string, patterns []string) ([]string, int) { + if patterns == nil { + logger.Infof("No exclude patterns in config") + return files, 0 + } else { + var results []string + var exclCount int + + fileRegexps := generateRegexps(patterns) + + if len(fileRegexps) == 0 { + logger.Infof("Excluded 0 files from scan") + return files, 0 + } + + for i := 0; i < len(files); i++ { + if matchFileSimple(files[i], fileRegexps) { + logger.Infof("File matched pattern. Excluding:\"%s\"", files[i]) + exclCount++ + } else { + + //if pattern doesn't match add file to list + results = append(results, files[i]) + } + } + logger.Infof("Excluded %d file(s) from scan", exclCount) + return results, exclCount + } +} + +func matchFile(file string, patterns []string) bool { + if patterns == nil { + logger.Infof("No exclude patterns in config.") + + } else { + fileRegexps := generateRegexps(patterns) + + if len(fileRegexps) == 0 { + return false + } + + for _, regPattern := range fileRegexps { + if regPattern.MatchString(strings.ToLower(file)) { + return true + } + + } + } + + return false +} + +func generateRegexps(patterns []string) []*regexp.Regexp { + + var fileRegexps []*regexp.Regexp + + for _, pattern := range patterns { + reg, err := regexp.Compile(strings.ToLower(pattern)) + if err != nil { + logger.Errorf("Exclude :%v", err) + } else { + fileRegexps = append(fileRegexps, reg) + } + } + + if len(fileRegexps) == 0 { + return nil + } else { + return fileRegexps + } + +} + +func matchFileSimple(file string, regExps []*regexp.Regexp) bool { + for _, regPattern := range regExps { + if regPattern.MatchString(strings.ToLower(file)) { + return true + } + } + return false +} diff --git a/pkg/manager/exclude_files_test.go b/pkg/manager/exclude_files_test.go new file mode 100644 index 000000000..978c9455e --- /dev/null +++ b/pkg/manager/exclude_files_test.go @@ -0,0 +1,91 @@ +package manager + +import ( + "fmt" + "github.com/stashapp/stash/pkg/logger" + "testing" +) + +var excludeTestFilenames = []string{ + "/stash/videos/filename.mp4", + "/stash/videos/new filename.mp4", + "filename sample.mp4", + "/stash/videos/exclude/not wanted.webm", + "/stash/videos/exclude/not wanted2.webm", + "/somewhere/trash/not wanted.wmv", + "/disk2/stash/videos/exclude/!!wanted!!.avi", + "/disk2/stash/videos/xcl/not wanted.avi", + "/stash/videos/partial.file.001.webm", + "/stash/videos/partial.file.002.webm", + "/stash/videos/partial.file.003.webm", + "/stash/videos/sample file sample.mkv", + "/stash/videos/.ckRVp1/.still_encoding.mp4", + "c:\\stash\\videos\\exclude\\filename windows.mp4", + "c:\\stash\\videos\\filename windows.mp4", + "\\\\network\\videos\\filename windows network.mp4", + "\\\\network\\share\\windows network wanted.mp4", + "\\\\network\\share\\windows network wanted sample.mp4", + "\\\\network\\private\\windows.network.skip.mp4"} + +var excludeTests = []struct { + testPattern []string + expected int +}{ + {[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic + {[]string{"no_match\\.mp4"}, 0}, //no match + {[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3}, //linux + {[]string{"/\\.[[:word:]]+/"}, 1}, //linux hidden dirs (handbrake unraid issue?) + {[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1}, //windows + {[]string{"\\/[/invalid"}, 0}, //invalid pattern + {[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3}, //invalid pattern but continue + {[]string{"^\\\\\\\\network"}, 4}, //windows net share + {[]string{"\\\\private\\\\"}, 1}, //windows net share + {[]string{"\\\\private\\\\", "sample\\.mp4"}, 3}, //windows net share +} + +func TestExcludeFiles(t *testing.T) { + for _, test := range excludeTests { + err := runExclude(excludeTestFilenames, test.testPattern, test.expected) + if err != nil { + t.Error(err) + } + } +} + +func runExclude(filenames []string, patterns []string, expCount int) error { + + files, count := excludeFiles(filenames, patterns) + + if count != expCount { + return fmt.Errorf("Was expecting %d, found %d", expCount, count) + } + if len(files) != len(filenames)-expCount { + return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files)) + } + + return nil +} + +func TestMatchFile(t *testing.T) { + for _, test := range excludeTests { + err := runMatch(excludeTestFilenames, test.testPattern, test.expected) + if err != nil { + t.Error(err) + } + } +} + +func runMatch(filenames []string, patterns []string, expCount int) error { + count := 0 + for _, file := range filenames { + if matchFile(file, patterns) { + logger.Infof("File \"%s\" matched pattern\n", file) + count++ + } + } + if count != expCount { + return fmt.Errorf("Was expecting %d, found %d", expCount, count) + } + + return nil +} diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index 8668b2b69..6f5272ad6 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -1,18 +1,15 @@ package manager import ( - "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "time" - "github.com/bmatcuk/doublestar" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" + "path/filepath" + "strconv" + "sync" + "time" ) type TaskStatus struct { @@ -414,7 +411,7 @@ func (s *singleton) Clean() { } if scene == nil { - logger.Errorf("nil scene, skipping generate") + logger.Errorf("nil scene, skipping Clean") continue } @@ -495,46 +492,3 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, sprites, previews, ma } return &totals } - -func excludeFiles(files []string, patterns []string) ([]string, int) { - if patterns == nil { - logger.Infof("No excludes in config.") - return files, 0 - } else { - var results []string - var exclCount int - var fileRegexps []*regexp.Regexp - - for _, pattern := range patterns { - reg, err := regexp.Compile(strings.ToLower(pattern)) - if err != nil { - logger.Errorf("Exclude :%v", err) - } else { - fileRegexps = append(fileRegexps, reg) - } - } - - if len(fileRegexps) == 0 { - return files, 0 - } - - for i := 0; i < len(files); i++ { - match := false - for _, regPattern := range fileRegexps { - if regPattern.Match([]byte(strings.ToLower(files[i]))) { - logger.Infof("File %s excluded from scan ", files[i]) - match = true - exclCount++ - break - } - - } - //if pattern doesn't match add file to list - if !match { - results = append(results, files[i]) - } - } - logger.Infof("Excluded %d file(s) from scan ", exclCount) - return results, exclCount - } -} diff --git a/pkg/manager/manager_tasks_test.go b/pkg/manager/manager_tasks_test.go deleted file mode 100644 index 93d2a07eb..000000000 --- a/pkg/manager/manager_tasks_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package manager - -import ( - "fmt" - "testing" -) - -func TestExcludeFiles(t *testing.T) { - - filenames := []string{ - "/stash/videos/filename.mp4", - "/stash/videos/new filename.mp4", - "filename sample.mp4", - "/stash/videos/exclude/not wanted.webm", - "/stash/videos/exclude/not wanted2.webm", - "/somewhere/trash/not wanted.wmv", - "/disk2/stash/videos/exclude/!!wanted!!.avi", - "/disk2/stash/videos/xcl/not wanted.avi", - "/stash/videos/partial.file.001.webm", - "/stash/videos/partial.file.002.webm", - "/stash/videos/partial.file.003.webm", - "/stash/videos/sample file sample.mkv", - "/stash/videos/.ckRVp1/.still_encoding.mp4", - "c:\\stash\\videos\\exclude\\filename windows.mp4", - "c:\\stash\\videos\\filename windows.mp4", - "\\\\network\\videos\\filename windows network.mp4", - "\\\\network\\share\\windows network wanted.mp4", - "\\\\network\\share\\windows network wanted sample.mp4", - "\\\\network\\private\\windows.network.skip.mp4"} - - var excludeTests = []struct { - testPattern []string - expected int - }{ - {[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic - {[]string{"no_match\\.mp4"}, 0}, //no match - {[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3}, //linux - {[]string{"/\\.[[:word:]]+/"}, 1}, //linux hidden dirs (handbrake unraid issue?) - {[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1}, //windows - {[]string{"\\/[/invalid"}, 0}, //invalid pattern - {[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3}, //invalid pattern but continue - {[]string{"^\\\\\\\\network"}, 4}, //windows net share - {[]string{"\\\\private\\\\"}, 1}, //windows net share - {[]string{"\\\\private\\\\", "sample\\.mp4"}, 3}, //windows net share - } - for _, test := range excludeTests { - err := runExclude(filenames, test.testPattern, test.expected) - if err != nil { - t.Error(err) - } - } -} - -func runExclude(filenames []string, patterns []string, expCount int) error { - - files, count := excludeFiles(filenames, patterns) - - if count != expCount { - return fmt.Errorf("Was expecting %d, found %d", expCount, count) - } - if len(files) != len(filenames)-expCount { - return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files)) - } - - return nil -} diff --git a/pkg/manager/task_clean.go b/pkg/manager/task_clean.go index bcc71b424..580d3631a 100644 --- a/pkg/manager/task_clean.go +++ b/pkg/manager/task_clean.go @@ -21,8 +21,12 @@ func (t *CleanTask) Start(wg *sync.WaitGroup) { if t.fileExists(t.Scene.Path) && t.pathInStash() { logger.Debugf("File Found: %s", t.Scene.Path) + if matchFile(t.Scene.Path, config.GetExcludes()) { + logger.Infof("File matched regex. Cleaning: \"%s\"", t.Scene.Path) + t.deleteScene(t.Scene.ID) + } } else { - logger.Infof("File not found. Cleaning: %s", t.Scene.Path) + logger.Infof("File not found. Cleaning: \"%s\"", t.Scene.Path) t.deleteScene(t.Scene.ID) } } diff --git a/ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx b/ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx index 300ec7f88..bf52604b3 100644 --- a/ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx +++ b/ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx @@ -1,4 +1,5 @@ import { + AnchorButton, Button, Divider, FormGroup, @@ -177,7 +178,6 @@ export const SettingsConfigurationPanel: FunctionComponent = (props: IPr { (excludes) ? excludes.map((regexp, i) => { @@ -192,6 +192,17 @@ export const SettingsConfigurationPanel: FunctionComponent = (props: IPr }