From e4d91a0226d4987c466e48e47641eb02656c62b8 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 2 Feb 2021 07:57:56 +1100 Subject: [PATCH] String regex filter criteria and selective autotag (#1082) * Add regex string filter criterion * Use query interface for auto tagging * Use Query interface for filename parser * Remove query regex interfaces * Add selective auto tag * Use page size 0 as no limit --- graphql/schema/types/filters.graphql | 5 + graphql/schema/types/metadata.graphql | 2 + pkg/api/resolver_mutation_metadata.go | 2 +- pkg/api/resolver_query_find_scene.go | 20 +++- pkg/manager/filename_parser.go | 9 +- pkg/manager/manager_tasks.go | 39 ++++-- pkg/manager/task_autotag.go | 43 +++---- pkg/manager/task_autotag_test.go | 18 ++- pkg/manager/task_clean.go | 20 +--- pkg/models/extension_find_filter.go | 32 +++++ pkg/models/mocks/SceneReaderWriter.go | 44 ++----- pkg/models/scene.go | 3 +- pkg/sqlite/gallery_test.go | 7 ++ pkg/sqlite/image_test.go | 15 ++- pkg/sqlite/query.go | 26 +++- pkg/sqlite/scene.go | 113 ++++++++---------- pkg/sqlite/scene_test.go | 14 +++ pkg/sqlite/setup_test.go | 15 ++- pkg/sqlite/sql.go | 22 +--- pkg/utils/file.go | 14 +++ .../src/components/Changelog/versions/v050.md | 2 + ...ialog.tsx => DirectorySelectionDialog.tsx} | 10 +- .../SettingsTasksPanel/SettingsTasksPanel.tsx | 45 ++++++- .../models/list-filter/criteria/criterion.ts | 38 ++++-- 24 files changed, 354 insertions(+), 204 deletions(-) rename ui/v2.5/src/components/Settings/SettingsTasksPanel/{ScanDialog.tsx => DirectorySelectionDialog.tsx} (90%) diff --git a/graphql/schema/types/filters.graphql b/graphql/schema/types/filters.graphql index 7dba266b4..d03aa062b 100644 --- a/graphql/schema/types/filters.graphql +++ b/graphql/schema/types/filters.graphql @@ -6,6 +6,7 @@ enum SortDirectionEnum { input FindFilterType { q: String page: Int + """use per_page = 0 to indicate all results. Defaults to 25.""" per_page: Int sort: String direction: SortDirectionEnum @@ -192,6 +193,10 @@ enum CriterionModifier { INCLUDES_ALL, INCLUDES, EXCLUDES, + """MATCHES REGEX""" + MATCHES_REGEX, + """NOT MATCHES REGEX""" + NOT_MATCHES_REGEX, } input StringCriterionInput { diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 89156d49b..d00e2846c 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -50,6 +50,8 @@ input CleanMetadataInput { } input AutoTagMetadataInput { + """Paths to tag, null for all files""" + paths: [String!] """IDs of performers to tag files with, or "*" for all""" performers: [String!] """IDs of studios to tag files with, or "*" for all""" diff --git a/pkg/api/resolver_mutation_metadata.go b/pkg/api/resolver_mutation_metadata.go index 8844e995b..19ce4f279 100644 --- a/pkg/api/resolver_mutation_metadata.go +++ b/pkg/api/resolver_mutation_metadata.go @@ -70,7 +70,7 @@ func (r *mutationResolver) MetadataGenerate(ctx context.Context, input models.Ge } func (r *mutationResolver) MetadataAutoTag(ctx context.Context, input models.AutoTagMetadataInput) (string, error) { - manager.GetInstance().AutoTag(input.Performers, input.Studios, input.Tags) + manager.GetInstance().AutoTag(input) return "todo", nil } diff --git a/pkg/api/resolver_query_find_scene.go b/pkg/api/resolver_query_find_scene.go index dcb59842a..ae8eec249 100644 --- a/pkg/api/resolver_query_find_scene.go +++ b/pkg/api/resolver_query_find_scene.go @@ -80,7 +80,25 @@ func (r *queryResolver) FindScenes(ctx context.Context, sceneFilter *models.Scen func (r *queryResolver) FindScenesByPathRegex(ctx context.Context, filter *models.FindFilterType) (ret *models.FindScenesResultType, err error) { if err := r.withReadTxn(ctx, func(repo models.ReaderRepository) error { - scenes, total, err := repo.Scene().QueryByPathRegex(filter) + + sceneFilter := &models.SceneFilterType{} + + if filter != nil && filter.Q != nil { + sceneFilter.Path = &models.StringCriterionInput{ + Modifier: models.CriterionModifierMatchesRegex, + Value: "(?i)" + *filter.Q, + } + } + + // make a copy of the filter if provided, nilling out Q + var queryFilter *models.FindFilterType + if filter != nil { + f := *filter + queryFilter = &f + queryFilter.Q = nil + } + + scenes, total, err := repo.Scene().Query(sceneFilter, queryFilter) if err != nil { return err } diff --git a/pkg/manager/filename_parser.go b/pkg/manager/filename_parser.go index 35f8c6297..eb5289e97 100644 --- a/pkg/manager/filename_parser.go +++ b/pkg/manager/filename_parser.go @@ -476,9 +476,14 @@ func (p *SceneFilenameParser) Parse(repo models.ReaderRepository) ([]*models.Sce return nil, 0, err } - p.Filter.Q = &mapper.regexString + sceneFilter := &models.SceneFilterType{ + Path: &models.StringCriterionInput{ + Modifier: models.CriterionModifierMatchesRegex, + Value: "(?i)" + mapper.regexString, + }, + } - scenes, total, err := repo.Scene().QueryByPathRegex(p.Filter) + scenes, total, err := repo.Scene().Query(sceneFilter, p.Filter) if err != nil { return nil, 0, err } diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index 03fe4c499..cf36f4c67 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -589,7 +589,7 @@ func (s *singleton) generateScreenshot(sceneId string, at *float64) { }() } -func (s *singleton) AutoTag(performerIds []string, studioIds []string, tagIds []string) { +func (s *singleton) AutoTag(input models.AutoTagMetadataInput) { if s.Status.Status != Idle { return } @@ -599,6 +599,10 @@ func (s *singleton) AutoTag(performerIds []string, studioIds []string, tagIds [] go func() { defer s.returnToIdleState() + performerIds := input.Performers + studioIds := input.Studios + tagIds := input.Tags + // calculate work load performerCount := len(performerIds) studioCount := len(studioIds) @@ -639,13 +643,13 @@ func (s *singleton) AutoTag(performerIds []string, studioIds []string, tagIds [] total := performerCount + studioCount + tagCount s.Status.setProgress(0, total) - s.autoTagPerformers(performerIds) - s.autoTagStudios(studioIds) - s.autoTagTags(tagIds) + s.autoTagPerformers(input.Paths, performerIds) + s.autoTagStudios(input.Paths, studioIds) + s.autoTagTags(input.Paths, tagIds) }() } -func (s *singleton) autoTagPerformers(performerIds []string) { +func (s *singleton) autoTagPerformers(paths []string, performerIds []string) { var wg sync.WaitGroup for _, performerId := range performerIds { var performers []*models.Performer @@ -681,8 +685,11 @@ func (s *singleton) autoTagPerformers(performerIds []string) { for _, performer := range performers { wg.Add(1) task := AutoTagPerformerTask{ - txnManager: s.TxnManager, - performer: performer, + AutoTagTask: AutoTagTask{ + txnManager: s.TxnManager, + paths: paths, + }, + performer: performer, } go task.Start(&wg) wg.Wait() @@ -692,7 +699,7 @@ func (s *singleton) autoTagPerformers(performerIds []string) { } } -func (s *singleton) autoTagStudios(studioIds []string) { +func (s *singleton) autoTagStudios(paths []string, studioIds []string) { var wg sync.WaitGroup for _, studioId := range studioIds { var studios []*models.Studio @@ -727,8 +734,11 @@ func (s *singleton) autoTagStudios(studioIds []string) { for _, studio := range studios { wg.Add(1) task := AutoTagStudioTask{ - studio: studio, - txnManager: s.TxnManager, + AutoTagTask: AutoTagTask{ + txnManager: s.TxnManager, + paths: paths, + }, + studio: studio, } go task.Start(&wg) wg.Wait() @@ -738,7 +748,7 @@ func (s *singleton) autoTagStudios(studioIds []string) { } } -func (s *singleton) autoTagTags(tagIds []string) { +func (s *singleton) autoTagTags(paths []string, tagIds []string) { var wg sync.WaitGroup for _, tagId := range tagIds { var tags []*models.Tag @@ -772,8 +782,11 @@ func (s *singleton) autoTagTags(tagIds []string) { for _, tag := range tags { wg.Add(1) task := AutoTagTagTask{ - txnManager: s.TxnManager, - tag: tag, + AutoTagTask: AutoTagTask{ + txnManager: s.TxnManager, + paths: paths, + }, + tag: tag, } go task.Start(&wg) wg.Wait() diff --git a/pkg/manager/task_autotag.go b/pkg/manager/task_autotag.go index ac47c404e..c9077b50d 100644 --- a/pkg/manager/task_autotag.go +++ b/pkg/manager/task_autotag.go @@ -12,18 +12,23 @@ import ( "github.com/stashapp/stash/pkg/scene" ) -type AutoTagPerformerTask struct { - performer *models.Performer +type AutoTagTask struct { + paths []string txnManager models.TransactionManager } +type AutoTagPerformerTask struct { + AutoTagTask + performer *models.Performer +} + func (t *AutoTagPerformerTask) Start(wg *sync.WaitGroup) { defer wg.Done() t.autoTagPerformer() } -func getQueryRegex(name string) string { +func (t *AutoTagTask) getQueryRegex(name string) string { const separatorChars = `.\-_ ` // handle path separators const separator = `[` + separatorChars + `]` @@ -34,12 +39,12 @@ func getQueryRegex(name string) string { } func (t *AutoTagPerformerTask) autoTagPerformer() { - regex := getQueryRegex(t.performer.Name.String) + regex := t.getQueryRegex(t.performer.Name.String) if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { qb := r.Scene() - const ignoreOrganized = true - scenes, err := qb.QueryAllByPathRegex(regex, ignoreOrganized) + + scenes, err := qb.QueryForAutoTag(regex, t.paths) if err != nil { return fmt.Errorf("Error querying scenes with regex '%s': %s", regex, err.Error()) @@ -64,8 +69,8 @@ func (t *AutoTagPerformerTask) autoTagPerformer() { } type AutoTagStudioTask struct { - studio *models.Studio - txnManager models.TransactionManager + AutoTagTask + studio *models.Studio } func (t *AutoTagStudioTask) Start(wg *sync.WaitGroup) { @@ -75,30 +80,29 @@ func (t *AutoTagStudioTask) Start(wg *sync.WaitGroup) { } func (t *AutoTagStudioTask) autoTagStudio() { - regex := getQueryRegex(t.studio.Name.String) + regex := t.getQueryRegex(t.studio.Name.String) if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { qb := r.Scene() - const ignoreOrganized = true - scenes, err := qb.QueryAllByPathRegex(regex, ignoreOrganized) + scenes, err := qb.QueryForAutoTag(regex, t.paths) if err != nil { return fmt.Errorf("Error querying scenes with regex '%s': %s", regex, err.Error()) } - for _, scene := range scenes { + for _, s := range scenes { // #306 - don't overwrite studio if already present - if scene.StudioID.Valid { + if s.StudioID.Valid { // don't modify continue } - logger.Infof("Adding studio '%s' to scene '%s'", t.studio.Name.String, scene.GetTitle()) + logger.Infof("Adding studio '%s' to scene '%s'", t.studio.Name.String, s.GetTitle()) // set the studio id studioID := sql.NullInt64{Int64: int64(t.studio.ID), Valid: true} scenePartial := models.ScenePartial{ - ID: scene.ID, + ID: s.ID, StudioID: &studioID, } @@ -114,8 +118,8 @@ func (t *AutoTagStudioTask) autoTagStudio() { } type AutoTagTagTask struct { - tag *models.Tag - txnManager models.TransactionManager + AutoTagTask + tag *models.Tag } func (t *AutoTagTagTask) Start(wg *sync.WaitGroup) { @@ -125,12 +129,11 @@ func (t *AutoTagTagTask) Start(wg *sync.WaitGroup) { } func (t *AutoTagTagTask) autoTagTag() { - regex := getQueryRegex(t.tag.Name) + regex := t.getQueryRegex(t.tag.Name) if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { qb := r.Scene() - const ignoreOrganized = true - scenes, err := qb.QueryAllByPathRegex(regex, ignoreOrganized) + scenes, err := qb.QueryForAutoTag(regex, t.paths) if err != nil { return fmt.Errorf("Error querying scenes with regex '%s': %s", regex, err.Error()) diff --git a/pkg/manager/task_autotag_test.go b/pkg/manager/task_autotag_test.go index feaeb43c3..0eb755c4d 100644 --- a/pkg/manager/task_autotag_test.go +++ b/pkg/manager/task_autotag_test.go @@ -279,8 +279,10 @@ func TestParsePerformers(t *testing.T) { } task := AutoTagPerformerTask{ - performer: performers[0], - txnManager: sqlite.NewTransactionManager(), + AutoTagTask: AutoTagTask{ + txnManager: sqlite.NewTransactionManager(), + }, + performer: performers[0], } var wg sync.WaitGroup @@ -327,8 +329,10 @@ func TestParseStudios(t *testing.T) { } task := AutoTagStudioTask{ - studio: studios[0], - txnManager: sqlite.NewTransactionManager(), + AutoTagTask: AutoTagTask{ + txnManager: sqlite.NewTransactionManager(), + }, + studio: studios[0], } var wg sync.WaitGroup @@ -374,8 +378,10 @@ func TestParseTags(t *testing.T) { } task := AutoTagTagTask{ - tag: tags[0], - txnManager: sqlite.NewTransactionManager(), + AutoTagTask: AutoTagTask{ + txnManager: sqlite.NewTransactionManager(), + }, + tag: tags[0], } var wg sync.WaitGroup diff --git a/pkg/manager/task_clean.go b/pkg/manager/task_clean.go index d4b4ada56..f696ef495 100644 --- a/pkg/manager/task_clean.go +++ b/pkg/manager/task_clean.go @@ -4,13 +4,13 @@ import ( "context" "os" "path/filepath" - "strings" "sync" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" ) type CleanTask struct { @@ -198,28 +198,18 @@ func (t *CleanTask) fileExists(filename string) (bool, error) { func getStashFromPath(pathToCheck string) *models.StashConfig { for _, s := range config.GetStashPaths() { - rel, error := filepath.Rel(s.Path, filepath.Dir(pathToCheck)) - - if error == nil { - if !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return s - } + if utils.IsPathInDir(s.Path, filepath.Dir(pathToCheck)) { + return s } - } return nil } func getStashFromDirPath(pathToCheck string) *models.StashConfig { for _, s := range config.GetStashPaths() { - rel, error := filepath.Rel(s.Path, pathToCheck) - - if error == nil { - if !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return s - } + if utils.IsPathInDir(s.Path, pathToCheck) { + return s } - } return nil } diff --git a/pkg/models/extension_find_filter.go b/pkg/models/extension_find_filter.go index c3bb24ffb..e2d4f8d7c 100644 --- a/pkg/models/extension_find_filter.go +++ b/pkg/models/extension_find_filter.go @@ -23,3 +23,35 @@ func (ff FindFilterType) GetDirection() string { } return direction } + +func (ff FindFilterType) GetPage() int { + const defaultPage = 1 + if ff.Page == nil || *ff.Page < 1 { + return defaultPage + } + + return *ff.Page +} + +func (ff FindFilterType) GetPageSize() int { + const defaultPerPage = 25 + const minPerPage = 1 + const maxPerPage = 1000 + + if ff.PerPage == nil { + return defaultPerPage + } + + if *ff.PerPage > 1000 { + return maxPerPage + } else if *ff.PerPage < 0 { + // PerPage == 0 -> no limit + return minPerPage + } + + return *ff.PerPage +} + +func (ff FindFilterType) IsGetAll() bool { + return ff.PerPage != nil && *ff.PerPage == 0 +} diff --git a/pkg/models/mocks/SceneReaderWriter.go b/pkg/models/mocks/SceneReaderWriter.go index 2a67e8891..386d93130 100644 --- a/pkg/models/mocks/SceneReaderWriter.go +++ b/pkg/models/mocks/SceneReaderWriter.go @@ -627,13 +627,13 @@ func (_m *SceneReaderWriter) Query(sceneFilter *models.SceneFilterType, findFilt return r0, r1, r2 } -// QueryAllByPathRegex provides a mock function with given fields: regex, ignoreOrganized -func (_m *SceneReaderWriter) QueryAllByPathRegex(regex string, ignoreOrganized bool) ([]*models.Scene, error) { - ret := _m.Called(regex, ignoreOrganized) +// QueryForAutoTag provides a mock function with given fields: regex, pathPrefixes +func (_m *SceneReaderWriter) QueryForAutoTag(regex string, pathPrefixes []string) ([]*models.Scene, error) { + ret := _m.Called(regex, pathPrefixes) var r0 []*models.Scene - if rf, ok := ret.Get(0).(func(string, bool) []*models.Scene); ok { - r0 = rf(regex, ignoreOrganized) + if rf, ok := ret.Get(0).(func(string, []string) []*models.Scene); ok { + r0 = rf(regex, pathPrefixes) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*models.Scene) @@ -641,8 +641,8 @@ func (_m *SceneReaderWriter) QueryAllByPathRegex(regex string, ignoreOrganized b } var r1 error - if rf, ok := ret.Get(1).(func(string, bool) error); ok { - r1 = rf(regex, ignoreOrganized) + if rf, ok := ret.Get(1).(func(string, []string) error); ok { + r1 = rf(regex, pathPrefixes) } else { r1 = ret.Error(1) } @@ -650,36 +650,6 @@ func (_m *SceneReaderWriter) QueryAllByPathRegex(regex string, ignoreOrganized b return r0, r1 } -// QueryByPathRegex provides a mock function with given fields: findFilter -func (_m *SceneReaderWriter) QueryByPathRegex(findFilter *models.FindFilterType) ([]*models.Scene, int, error) { - ret := _m.Called(findFilter) - - var r0 []*models.Scene - if rf, ok := ret.Get(0).(func(*models.FindFilterType) []*models.Scene); ok { - r0 = rf(findFilter) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*models.Scene) - } - } - - var r1 int - if rf, ok := ret.Get(1).(func(*models.FindFilterType) int); ok { - r1 = rf(findFilter) - } else { - r1 = ret.Get(1).(int) - } - - var r2 error - if rf, ok := ret.Get(2).(func(*models.FindFilterType) error); ok { - r2 = rf(findFilter) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - // ResetOCounter provides a mock function with given fields: id func (_m *SceneReaderWriter) ResetOCounter(id int) (int, error) { ret := _m.Called(id) diff --git a/pkg/models/scene.go b/pkg/models/scene.go index cefc12461..6bd5e78f8 100644 --- a/pkg/models/scene.go +++ b/pkg/models/scene.go @@ -21,9 +21,8 @@ type SceneReader interface { CountMissingOSHash() (int, error) Wall(q *string) ([]*Scene, error) All() ([]*Scene, error) + QueryForAutoTag(regex string, pathPrefixes []string) ([]*Scene, error) Query(sceneFilter *SceneFilterType, findFilter *FindFilterType) ([]*Scene, int, error) - QueryAllByPathRegex(regex string, ignoreOrganized bool) ([]*Scene, error) - QueryByPathRegex(findFilter *FindFilterType) ([]*Scene, int, error) GetCover(sceneID int) ([]byte, error) GetMovies(sceneID int) ([]MoviesScenes, error) GetTagIDs(sceneID int) ([]int, error) diff --git a/pkg/sqlite/gallery_test.go b/pkg/sqlite/gallery_test.go index 9149c69d9..c06ff19a2 100644 --- a/pkg/sqlite/gallery_test.go +++ b/pkg/sqlite/gallery_test.go @@ -166,6 +166,13 @@ func TestGalleryQueryPath(t *testing.T) { pathCriterion.Modifier = models.CriterionModifierNotEquals verifyGalleriesPath(t, r.Gallery(), pathCriterion) + pathCriterion.Modifier = models.CriterionModifierMatchesRegex + pathCriterion.Value = "gallery.*1_Path" + verifyGalleriesPath(t, r.Gallery(), pathCriterion) + + pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex + verifyGalleriesPath(t, r.Gallery(), pathCriterion) + return nil }) } diff --git a/pkg/sqlite/image_test.go b/pkg/sqlite/image_test.go index a56f9b888..40f4c2a9f 100644 --- a/pkg/sqlite/image_test.go +++ b/pkg/sqlite/image_test.go @@ -114,13 +114,20 @@ func TestImageQueryPath(t *testing.T) { Modifier: models.CriterionModifierEquals, } - verifyImagePath(t, pathCriterion) + verifyImagePath(t, pathCriterion, 1) pathCriterion.Modifier = models.CriterionModifierNotEquals - verifyImagePath(t, pathCriterion) + verifyImagePath(t, pathCriterion, totalImages-1) + + pathCriterion.Modifier = models.CriterionModifierMatchesRegex + pathCriterion.Value = "image_.*1_Path" + verifyImagePath(t, pathCriterion, 1) // TODO - 2 if zip path is included + + pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex + verifyImagePath(t, pathCriterion, totalImages-1) // TODO - -2 if zip path is included } -func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput) { +func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput, expected int) { withTxn(func(r models.Repository) error { sqb := r.Image() imageFilter := models.ImageFilterType{ @@ -132,6 +139,8 @@ func verifyImagePath(t *testing.T, pathCriterion models.StringCriterionInput) { t.Errorf("Error querying image: %s", err.Error()) } + assert.Equal(t, expected, len(images), "number of returned images") + for _, image := range images { verifyString(t, image.Path, pathCriterion) } diff --git a/pkg/sqlite/query.go b/pkg/sqlite/query.go index 58e5420de..378fe0efe 100644 --- a/pkg/sqlite/query.go +++ b/pkg/sqlite/query.go @@ -1,6 +1,10 @@ package sqlite -import "github.com/stashapp/stash/pkg/models" +import ( + "regexp" + + "github.com/stashapp/stash/pkg/models" +) type queryBuilder struct { repository *repository @@ -12,9 +16,15 @@ type queryBuilder struct { args []interface{} sortAndPagination string + + err error } func (qb queryBuilder) executeFind() ([]int, int, error) { + if qb.err != nil { + return nil, 0, qb.err + } + return qb.repository.executeFindQuery(qb.body, qb.args, qb.sortAndPagination, qb.whereClauses, qb.havingClauses) } @@ -66,6 +76,20 @@ func (qb *queryBuilder) handleStringCriterionInput(c *models.StringCriterionInpu case models.CriterionModifierNotEquals: qb.addWhere(column + " NOT LIKE ?") qb.addArg(c.Value) + case models.CriterionModifierMatchesRegex: + if _, err := regexp.Compile(c.Value); err != nil { + qb.err = err + return + } + qb.addWhere(column + " regexp ?") + qb.addArg(c.Value) + case models.CriterionModifierNotMatchesRegex: + if _, err := regexp.Compile(c.Value); err != nil { + qb.err = err + return + } + qb.addWhere(column + " NOT regexp ?") + qb.addArg(c.Value) default: clause, count := getSimpleCriterionClause(modifier, "?") qb.addWhere(column + " " + clause) diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index 71d6492ba..83c983725 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -3,6 +3,7 @@ package sqlite import ( "database/sql" "fmt" + "path/filepath" "strings" "github.com/jmoiron/sqlx" @@ -289,6 +290,53 @@ func (qb *sceneQueryBuilder) All() ([]*models.Scene, error) { return qb.queryScenes(selectAll(sceneTable)+qb.getSceneSort(nil), nil) } +// QueryForAutoTag queries for scenes whose paths match the provided regex and +// are optionally within the provided path. Excludes organized scenes. +// TODO - this should be replaced with Query once it can perform multiple +// filters on the same field. +func (qb *sceneQueryBuilder) QueryForAutoTag(regex string, pathPrefixes []string) ([]*models.Scene, error) { + var args []interface{} + body := selectDistinctIDs("scenes") + ` WHERE + scenes.path regexp ? AND + scenes.organized = 0` + + args = append(args, "(?i)"+regex) + + var pathClauses []string + for _, p := range pathPrefixes { + pathClauses = append(pathClauses, "scenes.path like ?") + + sep := string(filepath.Separator) + if !strings.HasSuffix(p, sep) { + p = p + sep + } + args = append(args, p+"%") + } + + if len(pathClauses) > 0 { + body += " AND (" + strings.Join(pathClauses, " OR ") + ")" + } + + idsResult, err := qb.runIdsQuery(body, args) + + if err != nil { + return nil, err + } + + var scenes []*models.Scene + for _, id := range idsResult { + scene, err := qb.Find(id) + + if err != nil { + return nil, err + } + + scenes = append(scenes, scene) + } + + return scenes, nil +} + func (qb *sceneQueryBuilder) Query(sceneFilter *models.SceneFilterType, findFilter *models.FindFilterType) ([]*models.Scene, int, error) { if sceneFilter == nil { sceneFilter = &models.SceneFilterType{} @@ -448,6 +496,7 @@ func (qb *sceneQueryBuilder) Query(sceneFilter *models.SceneFilterType, findFilt } query.sortAndPagination = qb.getSceneSort(findFilter) + getPagination(findFilter) + idsResult, countResult, err := query.executeFind() if err != nil { return nil, 0, err @@ -501,70 +550,6 @@ func getDurationWhereClause(durationFilter models.IntCriterionInput) (string, [] return clause, args } -func (qb *sceneQueryBuilder) QueryAllByPathRegex(regex string, ignoreOrganized bool) ([]*models.Scene, error) { - var args []interface{} - body := selectDistinctIDs("scenes") + " WHERE scenes.path regexp ?" - - if ignoreOrganized { - body += " AND scenes.organized = 0" - } - - args = append(args, "(?i)"+regex) - - idsResult, err := qb.runIdsQuery(body, args) - - if err != nil { - return nil, err - } - - var scenes []*models.Scene - for _, id := range idsResult { - scene, err := qb.Find(id) - - if err != nil { - return nil, err - } - - scenes = append(scenes, scene) - } - - return scenes, nil -} - -func (qb *sceneQueryBuilder) QueryByPathRegex(findFilter *models.FindFilterType) ([]*models.Scene, int, error) { - if findFilter == nil { - findFilter = &models.FindFilterType{} - } - - var whereClauses []string - var havingClauses []string - var args []interface{} - body := selectDistinctIDs("scenes") - - if q := findFilter.Q; q != nil && *q != "" { - whereClauses = append(whereClauses, "scenes.path regexp ?") - args = append(args, "(?i)"+*q) - } - - sortAndPagination := qb.getSceneSort(findFilter) + getPagination(findFilter) - idsResult, countResult, err := qb.executeFindQuery(body, args, sortAndPagination, whereClauses, havingClauses) - if err != nil { - return nil, 0, err - } - - var scenes []*models.Scene - for _, id := range idsResult { - scene, err := qb.Find(id) - if err != nil { - return nil, 0, err - } - - scenes = append(scenes, scene) - } - - return scenes, countResult, nil -} - func (qb *sceneQueryBuilder) getSceneSort(findFilter *models.FindFilterType) string { if findFilter == nil { return " ORDER BY scenes.path, scenes.date ASC " diff --git a/pkg/sqlite/scene_test.go b/pkg/sqlite/scene_test.go index f8858bd08..a4aef4089 100644 --- a/pkg/sqlite/scene_test.go +++ b/pkg/sqlite/scene_test.go @@ -5,6 +5,7 @@ package sqlite_test import ( "database/sql" "fmt" + "regexp" "strconv" "testing" @@ -176,6 +177,13 @@ func TestSceneQueryPath(t *testing.T) { pathCriterion.Modifier = models.CriterionModifierNotEquals verifyScenesPath(t, pathCriterion) + + pathCriterion.Modifier = models.CriterionModifierMatchesRegex + pathCriterion.Value = "scene_.*1_Path" + verifyScenesPath(t, pathCriterion) + + pathCriterion.Modifier = models.CriterionModifierNotMatchesRegex + verifyScenesPath(t, pathCriterion) } func verifyScenesPath(t *testing.T, pathCriterion models.StringCriterionInput) { @@ -221,6 +229,12 @@ func verifyString(t *testing.T, value string, criterion models.StringCriterionIn if criterion.Modifier == models.CriterionModifierNotEquals { assert.NotEqual(criterion.Value, value) } + if criterion.Modifier == models.CriterionModifierMatchesRegex { + assert.Regexp(regexp.MustCompile(criterion.Value), value) + } + if criterion.Modifier == models.CriterionModifierNotMatchesRegex { + assert.NotRegexp(regexp.MustCompile(criterion.Value), value) + } } func TestSceneQueryRating(t *testing.T) { diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index 5200dacc4..81019ce9d 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -21,7 +21,7 @@ import ( ) const totalScenes = 12 -const totalImages = 6 +const totalImages = 6 // TODO - add one for zip file const performersNameCase = 6 const performersNameNoCase = 2 const moviesNameCase = 2 @@ -61,6 +61,7 @@ const imageIdxWithTwoPerformers = 2 const imageIdxWithTag = 3 const imageIdxWithTwoTags = 4 const imageIdxWithStudio = 5 +const imageIdxInZip = 6 const performerIdxWithScene = 0 const performerIdx1WithScene = 1 @@ -110,6 +111,7 @@ const markerIdxWithScene = 0 const pathField = "Path" const checksumField = "Checksum" const titleField = "Title" +const zipPath = "zipPath.zip" func TestMain(m *testing.M) { ret := runTests(m) @@ -318,10 +320,19 @@ func getImageStringValue(index int, field string) string { return fmt.Sprintf("image_%04d_%s", index, field) } +func getImagePath(index int) string { + // TODO - currently not working + // if index == imageIdxInZip { + // return image.ZipFilename(zipPath, "image_0001_Path") + // } + + return getImageStringValue(index, pathField) +} + func createImages(qb models.ImageReaderWriter, n int) error { for i := 0; i < n; i++ { image := models.Image{ - Path: getImageStringValue(i, pathField), + Path: getImagePath(i), Title: sql.NullString{String: getImageStringValue(i, titleField), Valid: true}, Checksum: getImageStringValue(i, checksumField), Rating: getRating(i), diff --git a/pkg/sqlite/sql.go b/pkg/sqlite/sql.go index 27b4db64d..7736559d2 100644 --- a/pkg/sqlite/sql.go +++ b/pkg/sqlite/sql.go @@ -32,26 +32,14 @@ func getPagination(findFilter *models.FindFilterType) string { panic("nil find filter for pagination") } - var page int - if findFilter.Page == nil || *findFilter.Page < 1 { - page = 1 - } else { - page = *findFilter.Page + if findFilter.IsGetAll() { + return " " } - var perPage int - if findFilter.PerPage == nil { - perPage = 25 - } else { - perPage = *findFilter.PerPage - } - - if perPage > 1000 { - perPage = 1000 - } else if perPage < 1 { - perPage = 1 - } + return getPaginationSQL(findFilter.GetPage(), findFilter.GetPageSize()) +} +func getPaginationSQL(page int, perPage int) string { page = (page - 1) * perPage return " LIMIT " + strconv.Itoa(perPage) + " OFFSET " + strconv.Itoa(page) + " " } diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 072bdbc82..9e0368db7 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -10,6 +10,7 @@ import ( "os/user" "path/filepath" "regexp" + "strings" "github.com/h2non/filetype" "github.com/h2non/filetype/types" @@ -269,3 +270,16 @@ func MatchEntries(dir, pattern string) ([]string, error) { } return res, err } + +// IsPathInDir returns true if pathToCheck is within dir. +func IsPathInDir(dir, pathToCheck string) bool { + rel, err := filepath.Rel(dir, pathToCheck) + + if err == nil { + if !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return true + } + } + + return false +} diff --git a/ui/v2.5/src/components/Changelog/versions/v050.md b/ui/v2.5/src/components/Changelog/versions/v050.md index f30269d08..d2d6c7a99 100644 --- a/ui/v2.5/src/components/Changelog/versions/v050.md +++ b/ui/v2.5/src/components/Changelog/versions/v050.md @@ -8,6 +8,8 @@ * Allow configuration of visible navbar items. ### 🎨 Improvements +* Add directory selection to auto-tag task. +* Add string matches/not matches regex filter criteria. * Added configuration option for import file size limit and increased default to 1GB. * Add dry-run option for Clean task. * Refresh UI when changing custom CSS options. diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/ScanDialog.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog.tsx similarity index 90% rename from ui/v2.5/src/components/Settings/SettingsTasksPanel/ScanDialog.tsx rename to ui/v2.5/src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog.tsx index 40cad0738..54f4c1a43 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/ScanDialog.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/DirectorySelectionDialog.tsx @@ -4,12 +4,12 @@ import { useConfiguration } from "src/core/StashService"; import { Icon, Modal } from "src/components/Shared"; import { FolderSelect } from "src/components/Shared/FolderSelect/FolderSelect"; -interface IScanDialogProps { +interface IDirectorySelectionDialogProps { onClose: (paths?: string[]) => void; } -export const ScanDialog: React.FC = ( - props: IScanDialogProps +export const DirectorySelectionDialog: React.FC = ( + props: IDirectorySelectionDialogProps ) => { const { data } = useConfiguration(); @@ -33,12 +33,12 @@ export const ScanDialog: React.FC = ( show disabled={paths.length === 0} icon="pencil-alt" - header="Select folders to scan" + header="Select folders" accept={{ onClick: () => { props.onClose(paths); }, - text: "Scan", + text: "Confirm", }} cancel={{ onClick: () => props.onClose(), diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx index ef2d2cf3e..5144eafb1 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx @@ -21,7 +21,7 @@ import { LoadingIndicator, Modal } from "src/components/Shared"; import { downloadFile } from "src/utils"; import { GenerateButton } from "./GenerateButton"; import { ImportDialog } from "./ImportDialog"; -import { ScanDialog } from "./ScanDialog"; +import { DirectorySelectionDialog } from "./DirectorySelectionDialog"; type Plugin = Pick; type PluginTask = Pick; @@ -32,6 +32,9 @@ export const SettingsTasksPanel: React.FC = () => { const [isCleanAlertOpen, setIsCleanAlertOpen] = useState(false); const [isImportDialogOpen, setIsImportDialogOpen] = useState(false); const [isScanDialogOpen, setIsScanDialogOpen] = useState(false); + const [isAutoTagDialogOpen, setIsAutoTagDialogOpen] = useState( + false + ); const [isBackupRunning, setIsBackupRunning] = useState(false); const [useFileMetadata, setUseFileMetadata] = useState(false); const [stripFileExtension, setStripFileExtension] = useState(false); @@ -183,7 +186,7 @@ export const SettingsTasksPanel: React.FC = () => { return; } - return ; + return ; } function onScanDialogClosed(paths?: string[]) { @@ -211,18 +214,35 @@ export const SettingsTasksPanel: React.FC = () => { } } - function getAutoTagInput() { + function renderAutoTagDialog() { + if (!isAutoTagDialogOpen) { + return; + } + + return ; + } + + function onAutoTagDialogClosed(paths?: string[]) { + if (paths) { + onAutoTag(paths); + } + + setIsAutoTagDialogOpen(false); + } + + function getAutoTagInput(paths?: string[]) { const wildcard = ["*"]; return { + paths, performers: autoTagPerformers ? wildcard : [], studios: autoTagStudios ? wildcard : [], tags: autoTagTags ? wildcard : [], }; } - async function onAutoTag() { + async function onAutoTag(paths?: string[]) { try { - await mutateMetadataAutoTag(getAutoTagInput()); + await mutateMetadataAutoTag(getAutoTagInput(paths)); Toast.success({ content: "Started auto tagging" }); jobStatus.refetch(); } catch (e) { @@ -347,6 +367,7 @@ export const SettingsTasksPanel: React.FC = () => { {renderCleanAlert()} {renderImportDialog()} {renderScanDialog()} + {renderAutoTagDialog()}

Running Jobs

@@ -440,9 +461,21 @@ export const SettingsTasksPanel: React.FC = () => { /> - + Auto-tag content based on filenames. diff --git a/ui/v2.5/src/models/list-filter/criteria/criterion.ts b/ui/v2.5/src/models/list-filter/criteria/criterion.ts index c3d710316..2e4d6e2fd 100644 --- a/ui/v2.5/src/models/list-filter/criteria/criterion.ts +++ b/ui/v2.5/src/models/list-filter/criteria/criterion.ts @@ -148,6 +148,16 @@ export abstract class Criterion { return { value: CriterionModifier.Includes, label: "Includes" }; case CriterionModifier.Excludes: return { value: CriterionModifier.Excludes, label: "Excludes" }; + case CriterionModifier.MatchesRegex: + return { + value: CriterionModifier.MatchesRegex, + label: "Matches Regex", + }; + case CriterionModifier.NotMatchesRegex: + return { + value: CriterionModifier.NotMatchesRegex, + label: "Not Matches Regex", + }; } } @@ -195,6 +205,12 @@ export abstract class Criterion { case CriterionModifier.Excludes: modifierString = "excludes"; break; + case CriterionModifier.MatchesRegex: + modifierString = "matches regex"; + break; + case CriterionModifier.NotMatchesRegex: + modifierString = "not matches regex"; + break; default: modifierString = ""; } @@ -215,18 +231,18 @@ export abstract class Criterion { return `${this.parameterName}-${this.modifier.toString()}`; // TODO add values? } - /* - public set(modifier: CriterionModifier, value: Value) { - this.modifier = modifier; - if (Array.isArray(this.value)) { - this.value.push(value); - } else { - this.value = value; - } + private static replaceSpecialCharacter(str: string, c: string) { + return str.replaceAll(c, encodeURIComponent(c)); } - */ public encodeValue(): CriterionValue { + // replace certain characters + if (typeof this.value === "string") { + let ret = this.value; + ret = Criterion.replaceSpecialCharacter(ret, "&"); + ret = Criterion.replaceSpecialCharacter(ret, "+"); + return ret; + } return this.value; } } @@ -257,6 +273,8 @@ export class StringCriterion extends Criterion { StringCriterion.getModifierOption(CriterionModifier.Excludes), StringCriterion.getModifierOption(CriterionModifier.IsNull), StringCriterion.getModifierOption(CriterionModifier.NotNull), + StringCriterion.getModifierOption(CriterionModifier.MatchesRegex), + StringCriterion.getModifierOption(CriterionModifier.NotMatchesRegex), ]; public options: string[] | undefined; public value: string = ""; @@ -286,6 +304,8 @@ export class MandatoryStringCriterion extends StringCriterion { StringCriterion.getModifierOption(CriterionModifier.NotEquals), StringCriterion.getModifierOption(CriterionModifier.Includes), StringCriterion.getModifierOption(CriterionModifier.Excludes), + StringCriterion.getModifierOption(CriterionModifier.MatchesRegex), + StringCriterion.getModifierOption(CriterionModifier.NotMatchesRegex), ]; }