Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-06 16:34:02 +01:00
Code Issues Projects Releases Wiki Activity

String regex filter criteria and selective autotag (#1082)

Browse source 
* Add regex string filter criterion
* Use query interface for auto tagging
* Use Query interface for filename parser
* Remove query regex interfaces
* Add selective auto tag
* Use page size 0 as no limit
This commit is contained in:
WithoutPants 2021-02-02 07:57:56 +11:00 committed by GitHub
parent 4fd022a93b
commit e4d91a0226
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 354 additions and 204 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
graphql/schema/types/filters.graphql
View file

@ -6,6 +6,7 @@ enum SortDirectionEnum {
input FindFilterType {
q: String
page: Int
"""use per_page = 0 to indicate all results. Defaults to 25."""
per_page: Int
sort: String
direction: SortDirectionEnum
@ -192,6 +193,10 @@ enum CriterionModifier {
INCLUDES_ALL,
INCLUDES,
EXCLUDES,
"""MATCHES REGEX"""
MATCHES_REGEX,
"""NOT MATCHES REGEX"""
NOT_MATCHES_REGEX,
}
input StringCriterionInput {

2
graphql/schema/types/metadata.graphql
View file

@ -50,6 +50,8 @@ input CleanMetadataInput {
}
input AutoTagMetadataInput {
"""Paths to tag, null for all files"""
paths: [String!]
"""IDs of performers to tag files with, or "*" for all"""
performers: [String!]
"""IDs of studios to tag files with, or "*" for all"""

2
pkg/api/resolver_mutation_metadata.go
View file

@ -70,7 +70,7 @@ func (r *mutationResolver) MetadataGenerate(ctx context.Context, input models.Ge
}
func (r *mutationResolver) MetadataAutoTag(ctx context.Context, input models.AutoTagMetadataInput) (string, error) {
manager.GetInstance().AutoTag(input.Performers, input.Studios, input.Tags)
manager.GetInstance().AutoTag(input)
return "todo", nil
}

20
pkg/api/resolver_query_find_scene.go
View file

@ -80,7 +80,25 @@ func (r *queryResolver) FindScenes(ctx context.Context, sceneFilter *models.Scen
func (r *queryResolver) FindScenesByPathRegex(ctx context.Context, filter *models.FindFilterType) (ret *models.FindScenesResultType, err error) {
if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error {
scenes, total, err := repo.Scene().QueryByPathRegex(filter)
sceneFilter := &models.SceneFilterType{}
if filter != nil && filter.Q != nil {
sceneFilter.Path = &models.StringCriterionInput{
Modifier: models.CriterionModifierMatchesRegex,
Value: "(?i)" + *filter.Q,
}
}
// make a copy of the filter if provided, nilling out Q
var queryFilter *models.FindFilterType
if filter != nil {
f := *filter
queryFilter = &f
queryFilter.Q = nil
}
scenes, total, err := repo.Scene().Query(sceneFilter, queryFilter)
if err != nil {
return err
}

9
pkg/manager/filename_parser.go
View file

@ -476,9 +476,14 @@ func (p *SceneFilenameParser) Parse(repo models.ReaderRepository) ([]*models.Sce
return nil, 0, err
}
p.Filter.Q = &mapper.regexString
sceneFilter := &models.SceneFilterType{
Path: &models.StringCriterionInput{
Modifier: models.CriterionModifierMatchesRegex,
Value: "(?i)" + mapper.regexString,
},
}
scenes, total, err := repo.Scene().QueryByPathRegex(p.Filter)
scenes, total, err := repo.Scene().Query(sceneFilter, p.Filter)
if err != nil {
return nil, 0, err
}

39
pkg/manager/manager_tasks.go
View file

@ -589,7 +589,7 @@ func (s *singleton) generateScreenshot(sceneId string, at *float64) {
}()
}
func (s *singleton) AutoTag(performerIds []string, studioIds []string, tagIds []string) {
func (s *singleton) AutoTag(input models.AutoTagMetadataInput) {
if s.Status.Status != Idle {
return
}
@ -599,6 +599,10 @@ func (s *singleton) AutoTag(performerIds []string, studioIds []string, tagIds []
go func() {
defer s.returnToIdleState()
performerIds := input.Performers
studioIds := input.Studios
tagIds := input.Tags
// calculate work load
performerCount := len(performerIds)
studioCount := len(studioIds)
@ -639,13 +643,13 @@ func (s *singleton) AutoTag(performerIds []string, studioIds []string, tagIds []
total := performerCount + studioCount + tagCount
s.Status.setProgress(0, total)
s.autoTagPerformers(performerIds)
s.autoTagStudios(studioIds)
s.autoTagTags(tagIds)
s.autoTagPerformers(input.Paths, performerIds)
s.autoTagStudios(input.Paths, studioIds)
s.autoTagTags(input.Paths, tagIds)
}()
}
func (s *singleton) autoTagPerformers(performerIds []string) {
func (s *singleton) autoTagPerformers(paths []string, performerIds []string) {
var wg sync.WaitGroup
for _, performerId := range performerIds {
var performers []*models.Performer
@ -681,8 +685,11 @@ func (s *singleton) autoTagPerformers(performerIds []string) {
for _, performer := range performers {
wg.Add(1)
task := AutoTagPerformerTask{
txnManager: s.TxnManager,
performer: performer,
AutoTagTask: AutoTagTask{
txnManager: s.TxnManager,
paths: paths,
},
performer: performer,
}
go task.Start(&wg)
wg.Wait()
@ -692,7 +699,7 @@ func (s *singleton) autoTagPerformers(performerIds []string) {
}
}
func (s *singleton) autoTagStudios(studioIds []string) {
func (s *singleton) autoTagStudios(paths []string, studioIds []string) {
var wg sync.WaitGroup
for _, studioId := range studioIds {
var studios []*models.Studio
@ -727,8 +734,11 @@ func (s *singleton) autoTagStudios(studioIds []string) {
for _, studio := range studios {
wg.Add(1)
task := AutoTagStudioTask{
studio: studio,
txnManager: s.TxnManager,
AutoTagTask: AutoTagTask{
txnManager: s.TxnManager,
paths: paths,
},
studio: studio,
}
go task.Start(&wg)
wg.Wait()
@ -738,7 +748,7 @@ func (s *singleton) autoTagStudios(studioIds []string) {
}
}
func (s *singleton) autoTagTags(tagIds []string) {
func (s *singleton) autoTagTags(paths []string, tagIds []string) {
var wg sync.WaitGroup
for _, tagId := range tagIds {
var tags []*models.Tag
@ -772,8 +782,11 @@ func (s *singleton) autoTagTags(tagIds []string) {
for _, tag := range tags {
wg.Add(1)
task := AutoTagTagTask{
txnManager: s.TxnManager,
tag: tag,
AutoTagTask: AutoTagTask{
txnManager: s.TxnManager,
paths: paths,
},
tag: tag,
}
go task.Start(&wg)
wg.Wait()

43
pkg/manager/task_autotag.go
View file

@ -12,18 +12,23 @@ import (
"github.com/stashapp/stash/pkg/scene"
)
type AutoTagPerformerTask struct {
performer *models.Performer
type AutoTagTask struct {
paths []string
txnManager models.TransactionManager
}
type AutoTagPerformerTask struct {
AutoTagTask
performer *models.Performer
}
func (t *AutoTagPerformerTask) Start(wg *sync.WaitGroup) {
defer wg.Done()
t.autoTagPerformer()
}
func getQueryRegex(name string) string {
func (t *AutoTagTask) getQueryRegex(name string) string {
const separatorChars = `.\-_ `
// handle path separators
const separator = `[` + separatorChars + `]`
@ -34,12 +39,12 @@ func getQueryRegex(name string) string {
}
func (t *AutoTagPerformerTask) autoTagPerformer() {
regex := getQueryRegex(t.performer.Name.String)
regex := t.getQueryRegex(t.performer.Name.String)
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error {
qb := r.Scene()
const ignoreOrganized = true
scenes, err := qb.QueryAllByPathRegex(regex, ignoreOrganized)
scenes, err := qb.QueryForAutoTag(regex, t.paths)
if err != nil {
return fmt.Errorf("Error querying scenes with regex '%s': %s", regex, err.Error())
@ -64,8 +69,8 @@ func (t *AutoTagPerformerTask) autoTagPerformer() {
}
type AutoTagStudioTask struct {
studio *models.Studio
txnManager models.TransactionManager
AutoTagTask
studio *models.Studio
}
func (t *AutoTagStudioTask) Start(wg *sync.WaitGroup) {
@ -75,30 +80,29 @@ func (t *AutoTagStudioTask) Start(wg *sync.WaitGroup) {
}
func (t *AutoTagStudioTask) autoTagStudio() {
regex := getQueryRegex(t.studio.Name.String)
regex := t.getQueryRegex(t.studio.Name.String)
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error {
qb := r.Scene()
const ignoreOrganized = true
scenes, err := qb.QueryAllByPathRegex(regex, ignoreOrganized)
scenes, err := qb.QueryForAutoTag(regex, t.paths)
if err != nil {
return fmt.Errorf("Error querying scenes with regex '%s': %s", regex, err.Error())
}
for _, scene := range scenes {
for _, s := range scenes {
// #306 - don't overwrite studio if already present
if scene.StudioID.Valid {
if s.StudioID.Valid {
// don't modify
continue
}
logger.Infof("Adding studio '%s' to scene '%s'", t.studio.Name.String, scene.GetTitle())
logger.Infof("Adding studio '%s' to scene '%s'", t.studio.Name.String, s.GetTitle())
// set the studio id
studioID := sql.NullInt64{Int64: int64(t.studio.ID), Valid: true}
scenePartial := models.ScenePartial{
ID: scene.ID,
ID: s.ID,
StudioID: &studioID,
}
@ -114,8 +118,8 @@ func (t *AutoTagStudioTask) autoTagStudio() {
}
type AutoTagTagTask struct {
tag *models.Tag
txnManager models.TransactionManager
AutoTagTask
tag *models.Tag
}
func (t *AutoTagTagTask) Start(wg *sync.WaitGroup) {
@ -125,12 +129,11 @@ func (t *AutoTagTagTask) Start(wg *sync.WaitGroup) {
}
func (t *AutoTagTagTask) autoTagTag() {
regex := getQueryRegex(t.tag.Name)
regex := t.getQueryRegex(t.tag.Name)
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error {
qb := r.Scene()
const ignoreOrganized = true
scenes, err := qb.QueryAllByPathRegex(regex, ignoreOrganized)
scenes, err := qb.QueryForAutoTag(regex, t.paths)
if err != nil {
return fmt.Errorf("Error querying scenes with regex '%s': %s", regex, err.Error())

18
pkg/manager/task_autotag_test.go
View file

@ -279,8 +279,10 @@ func TestParsePerformers(t *testing.T) {
}
task := AutoTagPerformerTask{
performer: performers[0],
txnManager: sqlite.NewTransactionManager(),
AutoTagTask: AutoTagTask{
txnManager: sqlite.NewTransactionManager(),
},
performer: performers[0],
}
var wg sync.WaitGroup
@ -327,8 +329,10 @@ func TestParseStudios(t *testing.T) {
}
task := AutoTagStudioTask{
studio: studios[0],
txnManager: sqlite.NewTransactionManager(),
AutoTagTask: AutoTagTask{
txnManager: sqlite.NewTransactionManager(),
},
studio: studios[0],
}
var wg sync.WaitGroup
@ -374,8 +378,10 @@ func TestParseTags(t *testing.T) {
}
task := AutoTagTagTask{
tag: tags[0],
txnManager: sqlite.NewTransactionManager(),
AutoTagTask: AutoTagTask{
txnManager: sqlite.NewTransactionManager(),
},
tag: tags[0],
}
var wg sync.WaitGroup

20
pkg/manager/task_clean.go
View file

@ -4,13 +4,13 @@ import (
"context"
"os"
"path/filepath"
"strings"
"sync"
"github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/manager/config"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/utils"
)
type CleanTask struct {
@ -198,28 +198,18 @@ func (t *CleanTask) fileExists(filename string) (bool, error) {
func getStashFromPath(pathToCheck string) *models.StashConfig {
for _, s := range config.GetStashPaths() {
rel, error := filepath.Rel(s.Path, filepath.Dir(pathToCheck))
if error == nil {
if !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return s
}
if utils.IsPathInDir(s.Path, filepath.Dir(pathToCheck)) {
return s
}
}
return nil
}
func getStashFromDirPath(pathToCheck string) *models.StashConfig {
for _, s := range config.GetStashPaths() {
rel, error := filepath.Rel(s.Path, pathToCheck)
if error == nil {
if !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return s
}
if utils.IsPathInDir(s.Path, pathToCheck) {
return s
}
}
return nil
}

32
pkg/models/extension_find_filter.go
View file

@ -23,3 +23,35 @@ func (ff FindFilterType) GetDirection() string {
}
return direction
}
func (ff FindFilterType) GetPage() int {
const defaultPage = 1
if ff.Page == nil || *ff.Page < 1 {
return defaultPage
}
return *ff.Page
}
func (ff FindFilterType) GetPageSize() int {
const defaultPerPage = 25
const minPerPage = 1
const maxPerPage = 1000
if ff.PerPage == nil {
return defaultPerPage
}
if *ff.PerPage > 1000 {
return maxPerPage
} else if *ff.PerPage < 0 {
// PerPage == 0 -> no limit
return minPerPage
}
return *ff.PerPage
}
func (ff FindFilterType) IsGetAll() bool {
return ff.PerPage != nil && *ff.PerPage == 0
}

44
pkg/models/mocks/SceneReaderWriter.go
View file

@ -627,13 +627,13 @@ func (_m *SceneReaderWriter) Query(sceneFilter *models.SceneFilterType, findFilt
return r0, r1, r2
}
// QueryAllByPathRegex provides a mock function with given fields: regex, ignoreOrganized
func (_m *SceneReaderWriter) QueryAllByPathRegex(regex string, ignoreOrganized bool) ([]*models.Scene, error) {
ret := _m.Called(regex, ignoreOrganized)
// QueryForAutoTag provides a mock function with given fields: regex, pathPrefixes
func (_m *SceneReaderWriter) QueryForAutoTag(regex string, pathPrefixes []string) ([]*models.Scene, error) {
ret := _m.Called(regex, pathPrefixes)
var r0 []*models.Scene
if rf, ok := ret.Get(0).(func(string, bool) []*models.Scene); ok {
r0 = rf(regex, ignoreOrganized)
if rf, ok := ret.Get(0).(func(string, []string) []*models.Scene); ok {
r0 = rf(regex, pathPrefixes)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*models.Scene)
@ -641,8 +641,8 @@ func (_m *SceneReaderWriter) QueryAllByPathRegex(regex string, ignoreOrganized b
}
var r1 error
if rf, ok := ret.Get(1).(func(string, bool) error); ok {
r1 = rf(regex, ignoreOrganized)
if rf, ok := ret.Get(1).(func(string, []string) error); ok {
r1 = rf(regex, pathPrefixes)
} else {
r1 = ret.Error(1)
}
@ -650,36 +650,6 @@ func (_m *SceneReaderWriter) QueryAllByPathRegex(regex string, ignoreOrganized b
return r0, r1
}
// QueryByPathRegex provides a mock function with given fields: findFilter
func (_m *SceneReaderWriter) QueryByPathRegex(findFilter *models.FindFilterType) ([]*models.Scene, int, error) {
ret := _m.Called(findFilter)
var r0 []*models.Scene
if rf, ok := ret.Get(0).(func(*models.FindFilterType) []*models.Scene); ok {
r0 = rf(findFilter)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*models.Scene)
}
}
var r1 int
if rf, ok := ret.Get(1).(func(*models.FindFilterType) int); ok {
r1 = rf(findFilter)
} else {
r1 = ret.Get(1).(int)
}
var r2 error
if rf, ok := ret.Get(2).(func(*models.FindFilterType) error); ok {
r2 = rf(findFilter)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// ResetOCounter provides a mock function with given fields: id
func (_m *SceneReaderWriter) ResetOCounter(id int) (int, error) {
ret := _m.Called(id)

3
pkg/models/scene.go
View file

@ -21,9 +21,8 @@ type SceneReader interface {
CountMissingOSHash() (int, error)
Wall(q *string) ([]*Scene, error)
All() ([]*Scene, error)
QueryForAutoTag(regex string, pathPrefixes []string) ([]*Scene, error)
Query(sceneFilter *SceneFilterType, findFilter *FindFilterType) ([]*Scene, int, error)
QueryAllByPathRegex(regex string, ignoreOrganized bool) ([]*Scene, error)
QueryByPathRegex(findFilter *FindFilterType) ([]*Scene, int, error)
GetCover(sceneID int) ([]byte, error)
GetMovies(sceneID int) ([]MoviesScenes, error)
GetTagIDs(sceneID int) ([]int, error)

7
pkg/sqlite/gallery_test.go
View file

@ -166,6 +166,13 @@ func TestGalleryQueryPath(t *testing.T) {
pathCriterion.Modifier = models.CriterionModifierNotEquals
verifyGalleriesPath(t, r.Gallery(), pathCriterion)
pathCriterion.Modifier = models.CriterionModifierMatchesRegex
pathCriterion.Value = "gallery.*1_Path"
verifyGalleriesPath(t, r.Gallery(), pathCriterion)
pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex
verifyGalleriesPath(t, r.Gallery(), pathCriterion)
return nil
})
}

15
pkg/sqlite/image_test.go
View file

@ -114,13 +114,20 @@ func TestImageQueryPath(t *testing.T) {
Modifier: models.CriterionModifierEquals,
}
verifyImagePath(t, pathCriterion)
verifyImagePath(t, pathCriterion, 1)
pathCriterion.Modifier = models.CriterionModifierNotEquals
verifyImagePath(t, pathCriterion)
verifyImagePath(t, pathCriterion, totalImages-1)
pathCriterion.Modifier = models.CriterionModifierMatchesRegex
pathCriterion.Value = "image_.*1_Path"
verifyImagePath(t, pathCriterion, 1) // TODO - 2 if zip path is included
pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex
verifyImagePath(t, pathCriterion, totalImages-1) // TODO - -2 if zip path is included
}
func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput) {
func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput, expected int) {
withTxn(func(r models.Repository) error {
sqb := r.Image()
imageFilter := models.ImageFilterType{
@ -132,6 +139,8 @@ func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput) {
t.Errorf("Error querying image: %s", err.Error())
}
assert.Equal(t, expected, len(images), "number of returned images")
for _, image := range images {
verifyString(t, image.Path, pathCriterion)
}

26
pkg/sqlite/query.go
View file

@ -1,6 +1,10 @@
package sqlite
import "github.com/stashapp/stash/pkg/models"
import (
"regexp"
"github.com/stashapp/stash/pkg/models"
)
type queryBuilder struct {
repository *repository
@ -12,9 +16,15 @@ type queryBuilder struct {
args []interface{}
sortAndPagination string
err error
}
func (qb queryBuilder) executeFind() ([]int, int, error) {
if qb.err != nil {
return nil, 0, qb.err
}
return qb.repository.executeFindQuery(qb.body, qb.args, qb.sortAndPagination, qb.whereClauses, qb.havingClauses)
}
@ -66,6 +76,20 @@ func (qb *queryBuilder) handleStringCriterionInput(c *models.StringCriterionInpu
case models.CriterionModifierNotEquals:
qb.addWhere(column + " NOT LIKE ?")
qb.addArg(c.Value)
case models.CriterionModifierMatchesRegex:
if _, err := regexp.Compile(c.Value); err != nil {
qb.err = err
return
}
qb.addWhere(column + " regexp ?")
qb.addArg(c.Value)
case models.CriterionModifierNotMatchesRegex:
if _, err := regexp.Compile(c.Value); err != nil {
qb.err = err
return
}
qb.addWhere(column + " NOT regexp ?")
qb.addArg(c.Value)
default:
clause, count := getSimpleCriterionClause(modifier, "?")
qb.addWhere(column + " " + clause)

113
pkg/sqlite/scene.go
View file

@ -3,6 +3,7 @@ package sqlite
import (
"database/sql"
"fmt"
"path/filepath"
"strings"
"github.com/jmoiron/sqlx"
@ -289,6 +290,53 @@ func (qb *sceneQueryBuilder) All() ([]*models.Scene, error) {
return qb.queryScenes(selectAll(sceneTable)+qb.getSceneSort(nil), nil)
}
// QueryForAutoTag queries for scenes whose paths match the provided regex and
// are optionally within the provided path. Excludes organized scenes.
// TODO - this should be replaced with Query once it can perform multiple
// filters on the same field.
func (qb *sceneQueryBuilder) QueryForAutoTag(regex string, pathPrefixes []string) ([]*models.Scene, error) {
var args []interface{}
body := selectDistinctIDs("scenes") + ` WHERE
scenes.path regexp ? AND
scenes.organized = 0`
args = append(args, "(?i)"+regex)
var pathClauses []string
for _, p := range pathPrefixes {
pathClauses = append(pathClauses, "scenes.path like ?")
sep := string(filepath.Separator)
if !strings.HasSuffix(p, sep) {
p = p + sep
}
args = append(args, p+"%")
}
if len(pathClauses) > 0 {
body += " AND (" + strings.Join(pathClauses, " OR ") + ")"
}
idsResult, err := qb.runIdsQuery(body, args)
if err != nil {
return nil, err
}
var scenes []*models.Scene
for _, id := range idsResult {
scene, err := qb.Find(id)
if err != nil {
return nil, err
}
scenes = append(scenes, scene)
}
return scenes, nil
}
func (qb *sceneQueryBuilder) Query(sceneFilter *models.SceneFilterType, findFilter *models.FindFilterType) ([]*models.Scene, int, error) {
if sceneFilter == nil {
sceneFilter = &models.SceneFilterType{}
@ -448,6 +496,7 @@ func (qb *sceneQueryBuilder) Query(sceneFilter *models.SceneFilterType, findFilt
}
query.sortAndPagination = qb.getSceneSort(findFilter) + getPagination(findFilter)
idsResult, countResult, err := query.executeFind()
if err != nil {
return nil, 0, err
@ -501,70 +550,6 @@ func getDurationWhereClause(durationFilter models.IntCriterionInput) (string, []
return clause, args
}
func (qb *sceneQueryBuilder) QueryAllByPathRegex(regex string, ignoreOrganized bool) ([]*models.Scene, error) {
var args []interface{}
body := selectDistinctIDs("scenes") + " WHERE scenes.path regexp ?"
if ignoreOrganized {
body += " AND scenes.organized = 0"
}
args = append(args, "(?i)"+regex)
idsResult, err := qb.runIdsQuery(body, args)
if err != nil {
return nil, err
}
var scenes []*models.Scene
for _, id := range idsResult {
scene, err := qb.Find(id)
if err != nil {
return nil, err
}
scenes = append(scenes, scene)
}
return scenes, nil
}
func (qb *sceneQueryBuilder) QueryByPathRegex(findFilter *models.FindFilterType) ([]*models.Scene, int, error) {
if findFilter == nil {
findFilter = &models.FindFilterType{}
}
var whereClauses []string
var havingClauses []string
var args []interface{}
body := selectDistinctIDs("scenes")
if q := findFilter.Q; q != nil && *q != "" {
whereClauses = append(whereClauses, "scenes.path regexp ?")
args = append(args, "(?i)"+*q)
}
sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter)
idsResult, countResult, err := qb.executeFindQuery(body, args, sortAndPagination, whereClauses, havingClauses)
if err != nil {
return nil, 0, err
}
var scenes []*models.Scene
for _, id := range idsResult {
scene, err := qb.Find(id)
if err != nil {
return nil, 0, err
}
scenes = append(scenes, scene)
}
return scenes, countResult, nil
}
func (qb *sceneQueryBuilder) getSceneSort(findFilter *models.FindFilterType) string {
if findFilter == nil {
return " ORDER BY scenes.path, scenes.date ASC "

14
pkg/sqlite/scene_test.go
View file

@ -5,6 +5,7 @@ package sqlite_test
import (
"database/sql"
"fmt"
"regexp"
"strconv"
"testing"
@ -176,6 +177,13 @@ func TestSceneQueryPath(t *testing.T) {
pathCriterion.Modifier = models.CriterionModifierNotEquals
verifyScenesPath(t, pathCriterion)
pathCriterion.Modifier = models.CriterionModifierMatchesRegex
pathCriterion.Value = "scene_.*1_Path"
verifyScenesPath(t, pathCriterion)
pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex
verifyScenesPath(t, pathCriterion)
}
func verifyScenesPath(t *testing.T, pathCriterion models.StringCriterionInput) {
@ -221,6 +229,12 @@ func verifyString(t *testing.T, value string, criterion models.StringCriterionIn
if criterion.Modifier == models.CriterionModifierNotEquals {
assert.NotEqual(criterion.Value, value)
}
if criterion.Modifier == models.CriterionModifierMatchesRegex {
assert.Regexp(regexp.MustCompile(criterion.Value), value)
}
if criterion.Modifier == models.CriterionModifierNotMatchesRegex {
assert.NotRegexp(regexp.MustCompile(criterion.Value), value)
}
}
func TestSceneQueryRating(t *testing.T) {

15
pkg/sqlite/setup_test.go
View file

@ -21,7 +21,7 @@ import (
)
const totalScenes = 12
const totalImages = 6
const totalImages = 6 // TODO - add one for zip file
const performersNameCase = 6
const performersNameNoCase = 2
const moviesNameCase = 2
@ -61,6 +61,7 @@ const imageIdxWithTwoPerformers = 2
const imageIdxWithTag = 3
const imageIdxWithTwoTags = 4
const imageIdxWithStudio = 5
const imageIdxInZip = 6
const performerIdxWithScene = 0
const performerIdx1WithScene = 1
@ -110,6 +111,7 @@ const markerIdxWithScene = 0
const pathField = "Path"
const checksumField = "Checksum"
const titleField = "Title"
const zipPath = "zipPath.zip"
func TestMain(m *testing.M) {
ret := runTests(m)
@ -318,10 +320,19 @@ func getImageStringValue(index int, field string) string {
return fmt.Sprintf("image_%04d_%s", index, field)
}
func getImagePath(index int) string {
// TODO - currently not working
// if index == imageIdxInZip {
// return image.ZipFilename(zipPath, "image_0001_Path")
// }
return getImageStringValue(index, pathField)
}
func createImages(qb models.ImageReaderWriter, n int) error {
for i := 0; i < n; i++ {
image := models.Image{
Path: getImageStringValue(i, pathField),
Path: getImagePath(i),
Title: sql.NullString{String: getImageStringValue(i, titleField), Valid: true},
Checksum: getImageStringValue(i, checksumField),
Rating: getRating(i),

22
pkg/sqlite/sql.go
View file

@ -32,26 +32,14 @@ func getPagination(findFilter *models.FindFilterType) string {
panic("nil find filter for pagination")
}
var page int
if findFilter.Page == nil || *findFilter.Page < 1 {
page = 1
} else {
page = *findFilter.Page
if findFilter.IsGetAll() {
return " "
}
var perPage int
if findFilter.PerPage == nil {
perPage = 25
} else {
perPage = *findFilter.PerPage
}
if perPage > 1000 {
perPage = 1000
} else if perPage < 1 {
perPage = 1
}
return getPaginationSQL(findFilter.GetPage(), findFilter.GetPageSize())
}
func getPaginationSQL(page int, perPage int) string {
page = (page - 1) * perPage
return " LIMIT " + strconv.Itoa(perPage) + " OFFSET " + strconv.Itoa(page) + " "
}

14
pkg/utils/file.go
View file

@ -10,6 +10,7 @@ import (
"os/user"
"path/filepath"
"regexp"
"strings"
"github.com/h2non/filetype"
"github.com/h2non/filetype/types"
@ -269,3 +270,16 @@ func MatchEntries(dir, pattern string) ([]string, error) {
}
return res, err
}
// IsPathInDir returns true if pathToCheck is within dir.
func IsPathInDir(dir, pathToCheck string) bool {
rel, err := filepath.Rel(dir, pathToCheck)
if err == nil {
if !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return true
}
}
return false
}

2
ui/v2.5/src/components/Changelog/versions/v050.md
View file

@ -8,6 +8,8 @@
* Allow configuration of visible navbar items.
### 🎨 Improvements
* Add directory selection to auto-tag task.
* Add string matches/not matches regex filter criteria.
* Added configuration option for import file size limit and increased default to 1GB.
* Add dry-run option for Clean task.
* Refresh UI when changing custom CSS options.

10
ui/v2.5/src/components/Settings/SettingsTasksPanel/ScanDialog.tsx → ui/v2.5/src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog.tsx
View file

@ -4,12 +4,12 @@ import { useConfiguration } from "src/core/StashService";
import { Icon, Modal } from "src/components/Shared";
import { FolderSelect } from "src/components/Shared/FolderSelect/FolderSelect";
interface IScanDialogProps {
interface IDirectorySelectionDialogProps {
onClose: (paths?: string[]) => void;
}
export const ScanDialog: React.FC<IScanDialogProps> = (
props: IScanDialogProps
export const DirectorySelectionDialog: React.FC<IDirectorySelectionDialogProps> = (
props: IDirectorySelectionDialogProps
) => {
const { data } = useConfiguration();
@ -33,12 +33,12 @@ export const ScanDialog: React.FC<IScanDialogProps> = (
show
disabled={paths.length === 0}
icon="pencil-alt"
header="Select folders to scan"
header="Select folders"
accept={{
onClick: () => {
props.onClose(paths);
},
text: "Scan",
text: "Confirm",
}}
cancel={{
onClick: () => props.onClose(),

45
ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx
View file

@ -21,7 +21,7 @@ import { LoadingIndicator, Modal } from "src/components/Shared";
import { downloadFile } from "src/utils";
import { GenerateButton } from "./GenerateButton";
import { ImportDialog } from "./ImportDialog";
import { ScanDialog } from "./ScanDialog";
import { DirectorySelectionDialog } from "./DirectorySelectionDialog";
type Plugin = Pick<GQL.Plugin, "id">;
type PluginTask = Pick<GQL.PluginTask, "name" | "description">;
@ -32,6 +32,9 @@ export const SettingsTasksPanel: React.FC = () => {
const [isCleanAlertOpen, setIsCleanAlertOpen] = useState<boolean>(false);
const [isImportDialogOpen, setIsImportDialogOpen] = useState<boolean>(false);
const [isScanDialogOpen, setIsScanDialogOpen] = useState<boolean>(false);
const [isAutoTagDialogOpen, setIsAutoTagDialogOpen] = useState<boolean>(
false
);
const [isBackupRunning, setIsBackupRunning] = useState<boolean>(false);
const [useFileMetadata, setUseFileMetadata] = useState<boolean>(false);
const [stripFileExtension, setStripFileExtension] = useState<boolean>(false);
@ -183,7 +186,7 @@ export const SettingsTasksPanel: React.FC = () => {
return;
}
return <ScanDialog onClose={onScanDialogClosed} />;
return <DirectorySelectionDialog onClose={onScanDialogClosed} />;
}
function onScanDialogClosed(paths?: string[]) {
@ -211,18 +214,35 @@ export const SettingsTasksPanel: React.FC = () => {
}
}
function getAutoTagInput() {
function renderAutoTagDialog() {
if (!isAutoTagDialogOpen) {
return;
}
return <DirectorySelectionDialog onClose={onAutoTagDialogClosed} />;
}
function onAutoTagDialogClosed(paths?: string[]) {
if (paths) {
onAutoTag(paths);
}
setIsAutoTagDialogOpen(false);
}
function getAutoTagInput(paths?: string[]) {
const wildcard = ["*"];
return {
paths,
performers: autoTagPerformers ? wildcard : [],
studios: autoTagStudios ? wildcard : [],
tags: autoTagTags ? wildcard : [],
};
}
async function onAutoTag() {
async function onAutoTag(paths?: string[]) {
try {
await mutateMetadataAutoTag(getAutoTagInput());
await mutateMetadataAutoTag(getAutoTagInput(paths));
Toast.success({ content: "Started auto tagging" });
jobStatus.refetch();
} catch (e) {
@ -347,6 +367,7 @@ export const SettingsTasksPanel: React.FC = () => {
{renderCleanAlert()}
{renderImportDialog()}
{renderScanDialog()}
{renderAutoTagDialog()}
<h4>Running Jobs</h4>
@ -440,9 +461,21 @@ export const SettingsTasksPanel: React.FC = () => {
/>
</Form.Group>
<Form.Group>
<Button variant="secondary" type="submit" onClick={() => onAutoTag()}>
<Button
variant="secondary"
type="submit"
className="mr-2"
onClick={() => onAutoTag()}
>
Auto Tag
</Button>
<Button
variant="secondary"
type="submit"
onClick={() => setIsAutoTagDialogOpen(true)}
>
Selective Auto Tag
</Button>
<Form.Text className="text-muted">
Auto-tag content based on filenames.
</Form.Text>

38
ui/v2.5/src/models/list-filter/criteria/criterion.ts
View file

@ -148,6 +148,16 @@ export abstract class Criterion {
return { value: CriterionModifier.Includes, label: "Includes" };
case CriterionModifier.Excludes:
return { value: CriterionModifier.Excludes, label: "Excludes" };
case CriterionModifier.MatchesRegex:
return {
value: CriterionModifier.MatchesRegex,
label: "Matches Regex",
};
case CriterionModifier.NotMatchesRegex:
return {
value: CriterionModifier.NotMatchesRegex,
label: "Not Matches Regex",
};
}
}
@ -195,6 +205,12 @@ export abstract class Criterion {
case CriterionModifier.Excludes:
modifierString = "excludes";
break;
case CriterionModifier.MatchesRegex:
modifierString = "matches regex";
break;
case CriterionModifier.NotMatchesRegex:
modifierString = "not matches regex";
break;
default:
modifierString = "";
}
@ -215,18 +231,18 @@ export abstract class Criterion {
return `${this.parameterName}-${this.modifier.toString()}`; // TODO add values?
}
/*
public set(modifier: CriterionModifier, value: Value) {
this.modifier = modifier;
if (Array.isArray(this.value)) {
this.value.push(value);
} else {
this.value = value;
}
private static replaceSpecialCharacter(str: string, c: string) {
return str.replaceAll(c, encodeURIComponent(c));
}
*/
public encodeValue(): CriterionValue {
// replace certain characters
if (typeof this.value === "string") {
let ret = this.value;
ret = Criterion.replaceSpecialCharacter(ret, "&");
ret = Criterion.replaceSpecialCharacter(ret, "+");
return ret;
}
return this.value;
}
}
@ -257,6 +273,8 @@ export class StringCriterion extends Criterion {
StringCriterion.getModifierOption(CriterionModifier.Excludes),
StringCriterion.getModifierOption(CriterionModifier.IsNull),
StringCriterion.getModifierOption(CriterionModifier.NotNull),
StringCriterion.getModifierOption(CriterionModifier.MatchesRegex),
StringCriterion.getModifierOption(CriterionModifier.NotMatchesRegex),
];
public options: string[] | undefined;
public value: string = "";
@ -286,6 +304,8 @@ export class MandatoryStringCriterion extends StringCriterion {
StringCriterion.getModifierOption(CriterionModifier.NotEquals),
StringCriterion.getModifierOption(CriterionModifier.Includes),
StringCriterion.getModifierOption(CriterionModifier.Excludes),
StringCriterion.getModifierOption(CriterionModifier.MatchesRegex),
StringCriterion.getModifierOption(CriterionModifier.NotMatchesRegex),
];
}