Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-07 17:02:38 +01:00
Code Issues Projects Releases Wiki Activity

Added exclude patterns support for Clean Task (#274)

Browse source 
* Added exclude patterns support for Clean Task

* Added test file

* Refactoring and cosmetic fixes

* * Replace Match with MatchString
This commit is contained in:
bnkai 2019-12-24 04:06:07 +02:00 committed by Leopere
parent f52db4f58b
commit 52dd0197ee
6 changed files with 202 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
pkg/manager/exclude_files.go Normal file
View file

@ -0,0 +1,89 @@
package manager
import (
"github.com/stashapp/stash/pkg/logger"
"regexp"
"strings"
)
func excludeFiles(files []string, patterns []string) ([]string, int) {
if patterns == nil {
logger.Infof("No exclude patterns in config")
return files, 0
} else {
var results []string
var exclCount int
fileRegexps := generateRegexps(patterns)
if len(fileRegexps) == 0 {
logger.Infof("Excluded 0 files from scan")
return files, 0
}
for i := 0; i < len(files); i++ {
if matchFileSimple(files[i], fileRegexps) {
logger.Infof("File matched pattern. Excluding:\"%s\"", files[i])
exclCount++
} else {
//if pattern doesn't match add file to list
results = append(results, files[i])
}
}
logger.Infof("Excluded %d file(s) from scan", exclCount)
return results, exclCount
}
}
func matchFile(file string, patterns []string) bool {
if patterns == nil {
logger.Infof("No exclude patterns in config.")
} else {
fileRegexps := generateRegexps(patterns)
if len(fileRegexps) == 0 {
return false
}
for _, regPattern := range fileRegexps {
if regPattern.MatchString(strings.ToLower(file)) {
return true
}
}
}
return false
}
func generateRegexps(patterns []string) []*regexp.Regexp {
var fileRegexps []*regexp.Regexp
for _, pattern := range patterns {
reg, err := regexp.Compile(strings.ToLower(pattern))
if err != nil {
logger.Errorf("Exclude :%v", err)
} else {
fileRegexps = append(fileRegexps, reg)
}
}
if len(fileRegexps) == 0 {
return nil
} else {
return fileRegexps
}
}
func matchFileSimple(file string, regExps []*regexp.Regexp) bool {
for _, regPattern := range regExps {
if regPattern.MatchString(strings.ToLower(file)) {
return true
}
}
return false
}

91
pkg/manager/exclude_files_test.go Normal file
View file

@ -0,0 +1,91 @@
package manager
import (
"fmt"
"github.com/stashapp/stash/pkg/logger"
"testing"
)
var excludeTestFilenames = []string{
"/stash/videos/filename.mp4",
"/stash/videos/new filename.mp4",
"filename sample.mp4",
"/stash/videos/exclude/not wanted.webm",
"/stash/videos/exclude/not wanted2.webm",
"/somewhere/trash/not wanted.wmv",
"/disk2/stash/videos/exclude/!!wanted!!.avi",
"/disk2/stash/videos/xcl/not wanted.avi",
"/stash/videos/partial.file.001.webm",
"/stash/videos/partial.file.002.webm",
"/stash/videos/partial.file.003.webm",
"/stash/videos/sample file sample.mkv",
"/stash/videos/.ckRVp1/.still_encoding.mp4",
"c:\\stash\\videos\\exclude\\filename windows.mp4",
"c:\\stash\\videos\\filename windows.mp4",
"\\\\network\\videos\\filename windows network.mp4",
"\\\\network\\share\\windows network wanted.mp4",
"\\\\network\\share\\windows network wanted sample.mp4",
"\\\\network\\private\\windows.network.skip.mp4"}
var excludeTests = []struct {
testPattern []string
expected int
}{
{[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic
{[]string{"no_match\\.mp4"}, 0}, //no match
{[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3}, //linux
{[]string{"/\\.[[:word:]]+/"}, 1}, //linux hidden dirs (handbrake unraid issue?)
{[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1}, //windows
{[]string{"\\/[/invalid"}, 0}, //invalid pattern
{[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3}, //invalid pattern but continue
{[]string{"^\\\\\\\\network"}, 4}, //windows net share
{[]string{"\\\\private\\\\"}, 1}, //windows net share
{[]string{"\\\\private\\\\", "sample\\.mp4"}, 3}, //windows net share
}
func TestExcludeFiles(t *testing.T) {
for _, test := range excludeTests {
err := runExclude(excludeTestFilenames, test.testPattern, test.expected)
if err != nil {
t.Error(err)
}
}
}
func runExclude(filenames []string, patterns []string, expCount int) error {
files, count := excludeFiles(filenames, patterns)
if count != expCount {
return fmt.Errorf("Was expecting %d, found %d", expCount, count)
}
if len(files) != len(filenames)-expCount {
return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files))
}
return nil
}
func TestMatchFile(t *testing.T) {
for _, test := range excludeTests {
err := runMatch(excludeTestFilenames, test.testPattern, test.expected)
if err != nil {
t.Error(err)
}
}
}
func runMatch(filenames []string, patterns []string, expCount int) error {
count := 0
for _, file := range filenames {
if matchFile(file, patterns) {
logger.Infof("File \"%s\" matched pattern\n", file)
count++
}
}
if count != expCount {
return fmt.Errorf("Was expecting %d, found %d", expCount, count)
}
return nil
}

56
pkg/manager/manager_tasks.go
View file

@ -1,18 +1,15 @@
package manager
import (
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/bmatcuk/doublestar"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/manager/config"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/utils"
"path/filepath"
"strconv"
"sync"
"time"
)
type TaskStatus struct {
@ -414,7 +411,7 @@ func (s *singleton) Clean() {
}
if scene == nil {
logger.Errorf("nil scene, skipping generate")
logger.Errorf("nil scene, skipping Clean")
continue
}
@ -495,46 +492,3 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, sprites, previews, ma
}
return &totals
}
func excludeFiles(files []string, patterns []string) ([]string, int) {
if patterns == nil {
logger.Infof("No excludes in config.")
return files, 0
} else {
var results []string
var exclCount int
var fileRegexps []*regexp.Regexp
for _, pattern := range patterns {
reg, err := regexp.Compile(strings.ToLower(pattern))
if err != nil {
logger.Errorf("Exclude :%v", err)
} else {
fileRegexps = append(fileRegexps, reg)
}
}
if len(fileRegexps) == 0 {
return files, 0
}
for i := 0; i < len(files); i++ {
match := false
for _, regPattern := range fileRegexps {
if regPattern.Match([]byte(strings.ToLower(files[i]))) {
logger.Infof("File %s excluded from scan ", files[i])
match = true
exclCount++
break
}
}
//if pattern doesn't match add file to list
if !match {
results = append(results, files[i])
}
}
logger.Infof("Excluded %d file(s) from scan ", exclCount)
return results, exclCount
}
}

66
pkg/manager/manager_tasks_test.go
View file

@ -1,66 +0,0 @@
package manager
import (
"fmt"
"testing"
)
func TestExcludeFiles(t *testing.T) {
filenames := []string{
"/stash/videos/filename.mp4",
"/stash/videos/new filename.mp4",
"filename sample.mp4",
"/stash/videos/exclude/not wanted.webm",
"/stash/videos/exclude/not wanted2.webm",
"/somewhere/trash/not wanted.wmv",
"/disk2/stash/videos/exclude/!!wanted!!.avi",
"/disk2/stash/videos/xcl/not wanted.avi",
"/stash/videos/partial.file.001.webm",
"/stash/videos/partial.file.002.webm",
"/stash/videos/partial.file.003.webm",
"/stash/videos/sample file sample.mkv",
"/stash/videos/.ckRVp1/.still_encoding.mp4",
"c:\\stash\\videos\\exclude\\filename windows.mp4",
"c:\\stash\\videos\\filename windows.mp4",
"\\\\network\\videos\\filename windows network.mp4",
"\\\\network\\share\\windows network wanted.mp4",
"\\\\network\\share\\windows network wanted sample.mp4",
"\\\\network\\private\\windows.network.skip.mp4"}
var excludeTests = []struct {
testPattern []string
expected int
}{
{[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic
{[]string{"no_match\\.mp4"}, 0}, //no match
{[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3}, //linux
{[]string{"/\\.[[:word:]]+/"}, 1}, //linux hidden dirs (handbrake unraid issue?)
{[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1}, //windows
{[]string{"\\/[/invalid"}, 0}, //invalid pattern
{[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3}, //invalid pattern but continue
{[]string{"^\\\\\\\\network"}, 4}, //windows net share
{[]string{"\\\\private\\\\"}, 1}, //windows net share
{[]string{"\\\\private\\\\", "sample\\.mp4"}, 3}, //windows net share
}
for _, test := range excludeTests {
err := runExclude(filenames, test.testPattern, test.expected)
if err != nil {
t.Error(err)
}
}
}
func runExclude(filenames []string, patterns []string, expCount int) error {
files, count := excludeFiles(filenames, patterns)
if count != expCount {
return fmt.Errorf("Was expecting %d, found %d", expCount, count)
}
if len(files) != len(filenames)-expCount {
return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files))
}
return nil
}

6
pkg/manager/task_clean.go
View file

@ -21,8 +21,12 @@ func (t *CleanTask) Start(wg *sync.WaitGroup) {
if t.fileExists(t.Scene.Path) && t.pathInStash() {
logger.Debugf("File Found: %s", t.Scene.Path)
if matchFile(t.Scene.Path, config.GetExcludes()) {
logger.Infof("File matched regex. Cleaning: \"%s\"", t.Scene.Path)
t.deleteScene(t.Scene.ID)
}
} else {
logger.Infof("File not found. Cleaning: %s", t.Scene.Path)
logger.Infof("File not found. Cleaning: \"%s\"", t.Scene.Path)
t.deleteScene(t.Scene.ID)
}
}

13
ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx
View file

@ -1,4 +1,5 @@
import {
AnchorButton,
Button,
Divider,
FormGroup,
@ -177,7 +178,6 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
<FormGroup
label="Excluded Patterns"
helperText="Regexps of files/paths to exclude from Scan"
>
{ (excludes) ? excludes.map((regexp, i) => {
@ -192,6 +192,17 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
}
<Button icon="plus" minimal={true} onClick={(e: any) => excludeAddRegex()} />
<div>
<p>
<AnchorButton
href="https://github.com/stashapp/stash/wiki/Exclude-file-configuration"
rightIcon="help"
text="Regexps of files/paths to exclude from Scan and add to Clean"
minimal={true}
target="_blank"
/>
</p>
</div>
</FormGroup>
</FormGroup>