diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index 330b7b02f..048bb4815 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -124,6 +124,10 @@ fragment IdentifyMetadataOptionsData on IdentifyMetadataOptions { setCoverImage setOrganized includeMalePerformers + skipMultipleMatches + skipMultipleMatchTag + skipSingleNamePerformers + skipSingleNamePerformerTag } fragment ScraperSourceData on ScraperSource { diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 8e575b3ec..7cd89202b 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -185,6 +185,14 @@ input IdentifyMetadataOptionsInput { setOrganized: Boolean """defaults to true if not provided""" includeMalePerformers: Boolean + """defaults to true if not provided""" + skipMultipleMatches: Boolean + """tag to tag skipped multiple matches with""" + skipMultipleMatchTag: String + """defaults to true if not provided""" + skipSingleNamePerformers: Boolean + """tag to tag skipped single name performers with""" + skipSingleNamePerformerTag: String } input IdentifySourceInput { @@ -222,6 +230,14 @@ type IdentifyMetadataOptions { setOrganized: Boolean """defaults to true if not provided""" includeMalePerformers: Boolean + """defaults to true if not provided""" + skipMultipleMatches: Boolean + """tag to tag skipped multiple matches with""" + skipMultipleMatchTag: String + """defaults to true if not provided""" + skipSingleNamePerformers: Boolean + """tag to tag skipped single name performers with""" + skipSingleNamePerformerTag: String } type IdentifySource { diff --git a/internal/identify/identify.go b/internal/identify/identify.go index 04eccb7b0..40b26a87b 100644 --- a/internal/identify/identify.go +++ b/internal/identify/identify.go @@ -2,18 +2,33 @@ package identify import ( "context" + "errors" "fmt" + "strconv" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scraper" + "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/txn" "github.com/stashapp/stash/pkg/utils" ) +var ( + ErrSkipSingleNamePerformer = errors.New("a performer was skipped because they only had a single name and no disambiguation") +) + +type MultipleMatchesFoundError struct { + Source ScraperSource +} + +func (e *MultipleMatchesFoundError) Error() string { + return fmt.Sprintf("multiple matches found for %s", e.Source.Name) +} + type SceneScraper interface { - ScrapeScene(ctx context.Context, sceneID int) (*scraper.ScrapedScene, error) + ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) } type SceneUpdatePostHookExecutor interface { @@ -31,7 +46,7 @@ type SceneIdentifier struct { SceneReaderUpdater SceneReaderUpdater StudioCreator StudioCreator PerformerCreator PerformerCreator - TagCreator TagCreator + TagCreatorFinder TagCreatorFinder DefaultOptions *MetadataOptions Sources []ScraperSource @@ -39,13 +54,31 @@ type SceneIdentifier struct { } func (t *SceneIdentifier) Identify(ctx context.Context, txnManager txn.Manager, scene *models.Scene) error { - result, err := t.scrapeScene(ctx, scene) + result, err := t.scrapeScene(ctx, txnManager, scene) + var multipleMatchErr *MultipleMatchesFoundError if err != nil { - return err + if !errors.As(err, &multipleMatchErr) { + return err + } } if result == nil { - logger.Debugf("Unable to identify %s", scene.Path) + if multipleMatchErr != nil { + logger.Debugf("Identify skipped because multiple results returned for %s", scene.Path) + + // find if the scene should be tagged for multiple results + options := t.getOptions(multipleMatchErr.Source) + if options.SkipMultipleMatchTag != nil && len(*options.SkipMultipleMatchTag) > 0 { + // Tag it with the multiple results tag + err := t.addTagToScene(ctx, txnManager, scene, *options.SkipMultipleMatchTag) + if err != nil { + return err + } + return nil + } + } else { + logger.Debugf("Unable to identify %s", scene.Path) + } return nil } @@ -62,63 +95,98 @@ type scrapeResult struct { source ScraperSource } -func (t *SceneIdentifier) scrapeScene(ctx context.Context, scene *models.Scene) (*scrapeResult, error) { +func (t *SceneIdentifier) scrapeScene(ctx context.Context, txnManager txn.Manager, scene *models.Scene) (*scrapeResult, error) { // iterate through the input sources for _, source := range t.Sources { // scrape using the source - scraped, err := source.Scraper.ScrapeScene(ctx, scene.ID) + results, err := source.Scraper.ScrapeScenes(ctx, scene.ID) if err != nil { logger.Errorf("error scraping from %v: %v", source.Scraper, err) continue } - // if results were found then return - if scraped != nil { - return &scrapeResult{ - result: scraped, - source: source, - }, nil + if len(results) > 0 { + options := t.getOptions(source) + if len(results) > 1 && utils.IsTrue(options.SkipMultipleMatches) { + return nil, &MultipleMatchesFoundError{ + Source: source, + } + } else { + // if results were found then return + return &scrapeResult{ + result: results[0], + source: source, + }, nil + } } } return nil, nil } +// Returns a MetadataOptions object with any default options overwritten by source specific options +func (t *SceneIdentifier) getOptions(source ScraperSource) MetadataOptions { + options := *t.DefaultOptions + if source.Options == nil { + return options + } + if source.Options.SetCoverImage != nil { + options.SetCoverImage = source.Options.SetCoverImage + } + if source.Options.SetOrganized != nil { + options.SetOrganized = source.Options.SetOrganized + } + if source.Options.IncludeMalePerformers != nil { + options.IncludeMalePerformers = source.Options.IncludeMalePerformers + } + if source.Options.SkipMultipleMatches != nil { + options.SkipMultipleMatches = source.Options.SkipMultipleMatches + } + if source.Options.SkipMultipleMatchTag != nil && len(*source.Options.SkipMultipleMatchTag) > 0 { + options.SkipMultipleMatchTag = source.Options.SkipMultipleMatchTag + } + if source.Options.SkipSingleNamePerformers != nil { + options.SkipSingleNamePerformers = source.Options.SkipSingleNamePerformers + } + if source.Options.SkipSingleNamePerformerTag != nil && len(*source.Options.SkipSingleNamePerformerTag) > 0 { + options.SkipSingleNamePerformerTag = source.Options.SkipSingleNamePerformerTag + } + return options +} + func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene, result *scrapeResult) (*scene.UpdateSet, error) { ret := &scene.UpdateSet{ ID: s.ID, } - options := []MetadataOptions{} + allOptions := []MetadataOptions{} if result.source.Options != nil { - options = append(options, *result.source.Options) + allOptions = append(allOptions, *result.source.Options) } if t.DefaultOptions != nil { - options = append(options, *t.DefaultOptions) + allOptions = append(allOptions, *t.DefaultOptions) } - fieldOptions := getFieldOptions(options) - - setOrganized := false - for _, o := range options { - if o.SetOrganized != nil { - setOrganized = *o.SetOrganized - break - } - } + fieldOptions := getFieldOptions(allOptions) + options := t.getOptions(result.source) scraped := result.result rel := sceneRelationships{ - sceneReader: t.SceneReaderUpdater, - studioCreator: t.StudioCreator, - performerCreator: t.PerformerCreator, - tagCreator: t.TagCreator, - scene: s, - result: result, - fieldOptions: fieldOptions, + sceneReader: t.SceneReaderUpdater, + studioCreator: t.StudioCreator, + performerCreator: t.PerformerCreator, + tagCreatorFinder: t.TagCreatorFinder, + scene: s, + result: result, + fieldOptions: fieldOptions, + skipSingleNamePerformers: *options.SkipSingleNamePerformers, } + setOrganized := false + if options.SetOrganized != nil { + setOrganized = *options.SetOrganized + } ret.Partial = getScenePartial(s, scraped, fieldOptions, setOrganized) studioID, err := rel.studio(ctx) @@ -130,17 +198,19 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene, ret.Partial.StudioID = models.NewOptionalInt(*studioID) } - ignoreMale := false - for _, o := range options { - if o.IncludeMalePerformers != nil { - ignoreMale = !*o.IncludeMalePerformers - break - } + includeMalePerformers := true + if options.IncludeMalePerformers != nil { + includeMalePerformers = *options.IncludeMalePerformers } - performerIDs, err := rel.performers(ctx, ignoreMale) + addSkipSingleNamePerformerTag := false + performerIDs, err := rel.performers(ctx, !includeMalePerformers) if err != nil { - return nil, err + if errors.Is(err, ErrSkipSingleNamePerformer) { + addSkipSingleNamePerformerTag = true + } else { + return nil, err + } } if performerIDs != nil { ret.Partial.PerformerIDs = &models.UpdateIDs{ @@ -153,6 +223,14 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene, if err != nil { return nil, err } + if addSkipSingleNamePerformerTag { + tagID, err := strconv.ParseInt(*options.SkipSingleNamePerformerTag, 10, 64) + if err != nil { + return nil, fmt.Errorf("error converting tag ID %s: %w", *options.SkipSingleNamePerformerTag, err) + } + + tagIDs = intslice.IntAppendUnique(tagIDs, int(tagID)) + } if tagIDs != nil { ret.Partial.TagIDs = &models.UpdateIDs{ IDs: tagIDs, @@ -171,15 +249,7 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene, } } - setCoverImage := false - for _, o := range options { - if o.SetCoverImage != nil { - setCoverImage = *o.SetCoverImage - break - } - } - - if setCoverImage { + if options.SetCoverImage != nil && *options.SetCoverImage { ret.CoverImage, err = rel.cover(ctx) if err != nil { return nil, err @@ -241,6 +311,41 @@ func (t *SceneIdentifier) modifyScene(ctx context.Context, txnManager txn.Manage return nil } +func (t *SceneIdentifier) addTagToScene(ctx context.Context, txnManager txn.Manager, s *models.Scene, tagToAdd string) error { + if err := txn.WithTxn(ctx, txnManager, func(ctx context.Context) error { + tagID, err := strconv.Atoi(tagToAdd) + if err != nil { + return fmt.Errorf("error converting tag ID %s: %w", tagToAdd, err) + } + + if err := s.LoadTagIDs(ctx, t.SceneReaderUpdater); err != nil { + return err + } + existing := s.TagIDs.List() + + if intslice.IntInclude(existing, tagID) { + // skip if the scene was already tagged + return nil + } + + if err := scene.AddTag(ctx, t.SceneReaderUpdater, s, tagID); err != nil { + return err + } + + ret, err := t.TagCreatorFinder.Find(ctx, tagID) + if err != nil { + logger.Infof("Added tag id %s to skipped scene %s", tagToAdd, s.Path) + } else { + logger.Infof("Added tag %s to skipped scene %s", ret.Name, s.Path) + } + + return nil + }); err != nil { + return err + } + return nil +} + func getFieldOptions(options []MetadataOptions) map[string]*FieldOptions { // prefer source-specific field strategies, then the defaults ret := make(map[string]*FieldOptions) @@ -291,8 +396,7 @@ func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOp } if setOrganized && !scene.Organized { - // just reuse the boolean since we know it's true - partial.Organized = models.NewOptionalBool(setOrganized) + partial.Organized = models.NewOptionalBool(true) } return partial diff --git a/internal/identify/identify_test.go b/internal/identify/identify_test.go index 751f9bf4c..0b59a5ad5 100644 --- a/internal/identify/identify_test.go +++ b/internal/identify/identify_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "reflect" + "strconv" "testing" "github.com/stashapp/stash/pkg/models" @@ -17,10 +18,10 @@ var testCtx = context.Background() type mockSceneScraper struct { errIDs []int - results map[int]*scraper.ScrapedScene + results map[int][]*scraper.ScrapedScene } -func (s mockSceneScraper) ScrapeScene(ctx context.Context, sceneID int) (*scraper.ScrapedScene, error) { +func (s mockSceneScraper) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) { if intslice.IntInclude(s.errIDs, sceneID) { return nil, errors.New("scrape scene error") } @@ -40,32 +41,66 @@ func TestSceneIdentifier_Identify(t *testing.T) { missingID found1ID found2ID + multiFoundID + multiFound2ID errUpdateID ) - var scrapedTitle = "scrapedTitle" + var ( + skipMultipleTagID = 1 + skipMultipleTagIDStr = strconv.Itoa(skipMultipleTagID) + ) - defaultOptions := &MetadataOptions{} + var ( + scrapedTitle = "scrapedTitle" + scrapedTitle2 = "scrapedTitle2" + + boolFalse = false + boolTrue = true + ) + + defaultOptions := &MetadataOptions{ + SetOrganized: &boolFalse, + SetCoverImage: &boolFalse, + IncludeMalePerformers: &boolFalse, + SkipSingleNamePerformers: &boolFalse, + } sources := []ScraperSource{ { Scraper: mockSceneScraper{ errIDs: []int{errID1}, - results: map[int]*scraper.ScrapedScene{ - found1ID: { + results: map[int][]*scraper.ScrapedScene{ + found1ID: {{ Title: &scrapedTitle, - }, + }}, }, }, }, { Scraper: mockSceneScraper{ errIDs: []int{errID2}, - results: map[int]*scraper.ScrapedScene{ - found2ID: { + results: map[int][]*scraper.ScrapedScene{ + found2ID: {{ Title: &scrapedTitle, + }}, + errUpdateID: {{ + Title: &scrapedTitle, + }}, + multiFoundID: { + { + Title: &scrapedTitle, + }, + { + Title: &scrapedTitle2, + }, }, - errUpdateID: { - Title: &scrapedTitle, + multiFound2ID: { + { + Title: &scrapedTitle, + }, + { + Title: &scrapedTitle2, + }, }, }, }, @@ -73,6 +108,7 @@ func TestSceneIdentifier_Identify(t *testing.T) { } mockSceneReaderWriter := &mocks.SceneReaderWriter{} + mockTagFinderCreator := &mocks.TagReaderWriter{} mockSceneReaderWriter.On("UpdatePartial", mock.Anything, mock.MatchedBy(func(id int) bool { return id == errUpdateID @@ -81,52 +117,84 @@ func TestSceneIdentifier_Identify(t *testing.T) { return id != errUpdateID }), mock.Anything).Return(nil, nil) + mockTagFinderCreator.On("Find", mock.Anything, skipMultipleTagID).Return(&models.Tag{ + ID: skipMultipleTagID, + Name: skipMultipleTagIDStr, + }, nil) + tests := []struct { name string sceneID int + options *MetadataOptions wantErr bool }{ { "error scraping", errID1, + nil, false, }, { "error scraping from second", errID2, + nil, false, }, { "found in first scraper", found1ID, + nil, false, }, { "found in second scraper", found2ID, + nil, false, }, { "not found", missingID, + nil, false, }, { "error modifying", errUpdateID, + nil, true, }, - } - - identifier := SceneIdentifier{ - SceneReaderUpdater: mockSceneReaderWriter, - DefaultOptions: defaultOptions, - Sources: sources, - SceneUpdatePostHookExecutor: mockHookExecutor{}, + { + "multiple found", + multiFoundID, + nil, + false, + }, + { + "multiple found - set tag", + multiFound2ID, + &MetadataOptions{ + SkipMultipleMatches: &boolTrue, + SkipMultipleMatchTag: &skipMultipleTagIDStr, + }, + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + identifier := SceneIdentifier{ + SceneReaderUpdater: mockSceneReaderWriter, + TagCreatorFinder: mockTagFinderCreator, + DefaultOptions: defaultOptions, + Sources: sources, + SceneUpdatePostHookExecutor: mockHookExecutor{}, + } + + if tt.options != nil { + identifier.DefaultOptions = tt.options + } + scene := &models.Scene{ ID: tt.sceneID, PerformerIDs: models.NewRelatedIDs([]int{}), @@ -144,7 +212,16 @@ func TestSceneIdentifier_modifyScene(t *testing.T) { repo := models.Repository{ TxnManager: &mocks.TxnManager{}, } - tr := &SceneIdentifier{} + boolFalse := false + defaultOptions := &MetadataOptions{ + SetOrganized: &boolFalse, + SetCoverImage: &boolFalse, + IncludeMalePerformers: &boolFalse, + SkipSingleNamePerformers: &boolFalse, + } + tr := &SceneIdentifier{ + DefaultOptions: defaultOptions, + } type args struct { scene *models.Scene @@ -165,6 +242,9 @@ func TestSceneIdentifier_modifyScene(t *testing.T) { }, &scrapeResult{ result: &scraper.ScrapedScene{}, + source: ScraperSource{ + Options: defaultOptions, + }, }, }, false, diff --git a/internal/identify/options.go b/internal/identify/options.go index 84530e5fc..b4954a1f1 100644 --- a/internal/identify/options.go +++ b/internal/identify/options.go @@ -33,6 +33,14 @@ type MetadataOptions struct { SetOrganized *bool `json:"setOrganized"` // defaults to true if not provided IncludeMalePerformers *bool `json:"includeMalePerformers"` + // defaults to true if not provided + SkipMultipleMatches *bool `json:"skipMultipleMatches"` + // ID of tag to tag skipped multiple matches with + SkipMultipleMatchTag *string `json:"skipMultipleMatchTag"` + // defaults to true if not provided + SkipSingleNamePerformers *bool `json:"skipSingleNamePerformers"` + // ID of tag to tag skipped single name performers with + SkipSingleNamePerformerTag *string `json:"skipSingleNamePerformerTag"` } type FieldOptions struct { diff --git a/internal/identify/performer.go b/internal/identify/performer.go index cb16f2a83..a75bfb024 100644 --- a/internal/identify/performer.go +++ b/internal/identify/performer.go @@ -4,17 +4,20 @@ import ( "context" "fmt" "strconv" + "strings" "time" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/sliceutil/stringslice" + "github.com/stashapp/stash/pkg/utils" ) type PerformerCreator interface { Create(ctx context.Context, newPerformer *models.Performer) error + UpdateImage(ctx context.Context, performerID int, image []byte) error } -func getPerformerID(ctx context.Context, endpoint string, w PerformerCreator, p *models.ScrapedPerformer, createMissing bool) (*int, error) { +func getPerformerID(ctx context.Context, endpoint string, w PerformerCreator, p *models.ScrapedPerformer, createMissing bool, skipSingleNamePerformers bool) (*int, error) { if p.StoredID != nil { // existing performer, just add it performerID, err := strconv.Atoi(*p.StoredID) @@ -24,6 +27,10 @@ func getPerformerID(ctx context.Context, endpoint string, w PerformerCreator, p return &performerID, nil } else if createMissing && p.Name != nil { // name is mandatory + // skip single name performers with no disambiguation + if skipSingleNamePerformers && !strings.Contains(*p.Name, " ") && (p.Disambiguation == nil || len(*p.Disambiguation) == 0) { + return nil, ErrSkipSingleNamePerformer + } return createMissingPerformer(ctx, endpoint, w, p) } @@ -46,6 +53,19 @@ func createMissingPerformer(ctx context.Context, endpoint string, w PerformerCre return nil, fmt.Errorf("error creating performer: %w", err) } + // update image table + if p.Image != nil && len(*p.Image) > 0 { + imageData, err := utils.ReadImageFromURL(ctx, *p.Image) + if err != nil { + return nil, err + } + + err = w.UpdateImage(ctx, performerInput.ID, imageData) + if err != nil { + return nil, err + } + } + return &performerInput.ID, nil } @@ -56,6 +76,9 @@ func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performe CreatedAt: currentTime, UpdatedAt: currentTime, } + if performer.Disambiguation != nil { + ret.Disambiguation = *performer.Disambiguation + } if performer.Birthdate != nil { d := models.NewDate(*performer.Birthdate) ret.Birthdate = &d @@ -126,6 +149,12 @@ func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performe if performer.Instagram != nil { ret.Instagram = *performer.Instagram } + if performer.URL != nil { + ret.URL = *performer.URL + } + if performer.Details != nil { + ret.Details = *performer.Details + } return ret } diff --git a/internal/identify/performer_test.go b/internal/identify/performer_test.go index 9ba1018c7..2e22837c4 100644 --- a/internal/identify/performer_test.go +++ b/internal/identify/performer_test.go @@ -31,9 +31,10 @@ func Test_getPerformerID(t *testing.T) { }).Return(nil) type args struct { - endpoint string - p *models.ScrapedPerformer - createMissing bool + endpoint string + p *models.ScrapedPerformer + createMissing bool + skipSingleName bool } tests := []struct { name string @@ -47,6 +48,7 @@ func Test_getPerformerID(t *testing.T) { emptyEndpoint, &models.ScrapedPerformer{}, false, + false, }, nil, false, @@ -59,6 +61,7 @@ func Test_getPerformerID(t *testing.T) { StoredID: &invalidStoredID, }, false, + false, }, nil, true, @@ -71,6 +74,7 @@ func Test_getPerformerID(t *testing.T) { StoredID: &validStoredIDStr, }, false, + false, }, &validStoredID, false, @@ -83,6 +87,7 @@ func Test_getPerformerID(t *testing.T) { Name: &name, }, false, + false, }, nil, false, @@ -93,10 +98,24 @@ func Test_getPerformerID(t *testing.T) { emptyEndpoint, &models.ScrapedPerformer{}, true, + false, }, nil, false, }, + { + "single name no disambig creating", + args{ + emptyEndpoint, + &models.ScrapedPerformer{ + Name: &name, + }, + true, + true, + }, + nil, + true, + }, { "valid name creating", args{ @@ -105,6 +124,7 @@ func Test_getPerformerID(t *testing.T) { Name: &name, }, true, + false, }, &validStoredID, false, @@ -112,7 +132,7 @@ func Test_getPerformerID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := getPerformerID(testCtx, tt.args.endpoint, &mockPerformerReaderWriter, tt.args.p, tt.args.createMissing) + got, err := getPerformerID(testCtx, tt.args.endpoint, &mockPerformerReaderWriter, tt.args.p, tt.args.createMissing, tt.args.skipSingleName) if (err != nil) != tt.wantErr { t.Errorf("getPerformerID() error = %v, wantErr %v", err, tt.wantErr) return @@ -207,7 +227,7 @@ func Test_scrapedToPerformerInput(t *testing.T) { name := "name" var stringValues []string - for i := 0; i < 17; i++ { + for i := 0; i < 20; i++ { stringValues = append(stringValues, strconv.Itoa(i)) } @@ -240,44 +260,50 @@ func Test_scrapedToPerformerInput(t *testing.T) { { "set all", &models.ScrapedPerformer{ - Name: &name, - Birthdate: nextVal(), - DeathDate: nextVal(), - Gender: nextVal(), - Ethnicity: nextVal(), - Country: nextVal(), - EyeColor: nextVal(), - HairColor: nextVal(), - Height: nextVal(), - Weight: nextVal(), - Measurements: nextVal(), - FakeTits: nextVal(), - CareerLength: nextVal(), - Tattoos: nextVal(), - Piercings: nextVal(), - Aliases: nextVal(), - Twitter: nextVal(), - Instagram: nextVal(), + Name: &name, + Disambiguation: nextVal(), + Birthdate: nextVal(), + DeathDate: nextVal(), + Gender: nextVal(), + Ethnicity: nextVal(), + Country: nextVal(), + EyeColor: nextVal(), + HairColor: nextVal(), + Height: nextVal(), + Weight: nextVal(), + Measurements: nextVal(), + FakeTits: nextVal(), + CareerLength: nextVal(), + Tattoos: nextVal(), + Piercings: nextVal(), + Aliases: nextVal(), + Twitter: nextVal(), + Instagram: nextVal(), + URL: nextVal(), + Details: nextVal(), }, models.Performer{ - Name: name, - Birthdate: dateToDatePtr(models.NewDate(*nextVal())), - DeathDate: dateToDatePtr(models.NewDate(*nextVal())), - Gender: genderPtr(models.GenderEnum(*nextVal())), - Ethnicity: *nextVal(), - Country: *nextVal(), - EyeColor: *nextVal(), - HairColor: *nextVal(), - Height: nextIntVal(), - Weight: nextIntVal(), - Measurements: *nextVal(), - FakeTits: *nextVal(), - CareerLength: *nextVal(), - Tattoos: *nextVal(), - Piercings: *nextVal(), - Aliases: models.NewRelatedStrings([]string{*nextVal()}), - Twitter: *nextVal(), - Instagram: *nextVal(), + Name: name, + Disambiguation: *nextVal(), + Birthdate: dateToDatePtr(models.NewDate(*nextVal())), + DeathDate: dateToDatePtr(models.NewDate(*nextVal())), + Gender: genderPtr(models.GenderEnum(*nextVal())), + Ethnicity: *nextVal(), + Country: *nextVal(), + EyeColor: *nextVal(), + HairColor: *nextVal(), + Height: nextIntVal(), + Weight: nextIntVal(), + Measurements: *nextVal(), + FakeTits: *nextVal(), + CareerLength: *nextVal(), + Tattoos: *nextVal(), + Piercings: *nextVal(), + Aliases: models.NewRelatedStrings([]string{*nextVal()}), + Twitter: *nextVal(), + Instagram: *nextVal(), + URL: *nextVal(), + Details: *nextVal(), }, }, { diff --git a/internal/identify/scene.go b/internal/identify/scene.go index 9f99f67dc..86256cebb 100644 --- a/internal/identify/scene.go +++ b/internal/identify/scene.go @@ -3,6 +3,7 @@ package identify import ( "bytes" "context" + "errors" "fmt" "strconv" "strings" @@ -13,6 +14,7 @@ import ( "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/sliceutil" "github.com/stashapp/stash/pkg/sliceutil/intslice" + "github.com/stashapp/stash/pkg/tag" "github.com/stashapp/stash/pkg/utils" ) @@ -24,18 +26,20 @@ type SceneReaderUpdater interface { models.StashIDLoader } -type TagCreator interface { +type TagCreatorFinder interface { Create(ctx context.Context, newTag *models.Tag) error + tag.Finder } type sceneRelationships struct { - sceneReader SceneReaderUpdater - studioCreator StudioCreator - performerCreator PerformerCreator - tagCreator TagCreator - scene *models.Scene - result *scrapeResult - fieldOptions map[string]*FieldOptions + sceneReader SceneReaderUpdater + studioCreator StudioCreator + performerCreator PerformerCreator + tagCreatorFinder TagCreatorFinder + scene *models.Scene + result *scrapeResult + fieldOptions map[string]*FieldOptions + skipSingleNamePerformers bool } func (g sceneRelationships) studio(ctx context.Context) (*int, error) { @@ -93,13 +97,19 @@ func (g sceneRelationships) performers(ctx context.Context, ignoreMale bool) ([] performerIDs = originalPerformerIDs } + singleNamePerformerSkipped := false + for _, p := range scraped { if ignoreMale && p.Gender != nil && strings.EqualFold(*p.Gender, models.GenderEnumMale.String()) { continue } - performerID, err := getPerformerID(ctx, endpoint, g.performerCreator, p, createMissing) + performerID, err := getPerformerID(ctx, endpoint, g.performerCreator, p, createMissing, g.skipSingleNamePerformers) if err != nil { + if errors.Is(err, ErrSkipSingleNamePerformer) { + singleNamePerformerSkipped = true + continue + } return nil, err } @@ -110,9 +120,15 @@ func (g sceneRelationships) performers(ctx context.Context, ignoreMale bool) ([] // don't return if nothing was added if sliceutil.SliceSame(originalPerformerIDs, performerIDs) { + if singleNamePerformerSkipped { + return nil, ErrSkipSingleNamePerformer + } return nil, nil } + if singleNamePerformerSkipped { + return performerIDs, ErrSkipSingleNamePerformer + } return performerIDs, nil } @@ -156,7 +172,7 @@ func (g sceneRelationships) tags(ctx context.Context) ([]int, error) { CreatedAt: now, UpdatedAt: now, } - err := g.tagCreator.Create(ctx, &newTag) + err := g.tagCreatorFinder.Create(ctx, &newTag) if err != nil { return nil, fmt.Errorf("error creating tag: %w", err) } diff --git a/internal/identify/scene_test.go b/internal/identify/scene_test.go index b91220f9f..714b559ce 100644 --- a/internal/identify/scene_test.go +++ b/internal/identify/scene_test.go @@ -374,9 +374,9 @@ func Test_sceneRelationships_tags(t *testing.T) { })).Return(errors.New("error creating tag")) tr := sceneRelationships{ - sceneReader: mockSceneReaderWriter, - tagCreator: mockTagReaderWriter, - fieldOptions: make(map[string]*FieldOptions), + sceneReader: mockSceneReaderWriter, + tagCreatorFinder: mockTagReaderWriter, + fieldOptions: make(map[string]*FieldOptions), } tests := []struct { diff --git a/internal/identify/studio.go b/internal/identify/studio.go index e90864b11..682245d5b 100644 --- a/internal/identify/studio.go +++ b/internal/identify/studio.go @@ -7,11 +7,13 @@ import ( "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" ) type StudioCreator interface { Create(ctx context.Context, newStudio *models.Studio) error UpdateStashIDs(ctx context.Context, studioID int, stashIDs []models.StashID) error + UpdateImage(ctx context.Context, studioID int, image []byte) error } func createMissingStudio(ctx context.Context, endpoint string, w StudioCreator, studio *models.ScrapedStudio) (*int, error) { @@ -21,6 +23,19 @@ func createMissingStudio(ctx context.Context, endpoint string, w StudioCreator, return nil, fmt.Errorf("error creating studio: %w", err) } + // update image table + if studio.Image != nil && len(*studio.Image) > 0 { + imageData, err := utils.ReadImageFromURL(ctx, *studio.Image) + if err != nil { + return nil, err + } + + err = w.UpdateImage(ctx, studioInput.ID, imageData) + if err != nil { + return nil, err + } + } + if endpoint != "" && studio.RemoteSiteID != nil { if err := w.UpdateStashIDs(ctx, studioInput.ID, []models.StashID{ { diff --git a/internal/manager/task_identify.go b/internal/manager/task_identify.go index 4cbacde2b..2a0c942f2 100644 --- a/internal/manager/task_identify.go +++ b/internal/manager/task_identify.go @@ -136,7 +136,7 @@ func (j *IdentifyJob) identifyScene(ctx context.Context, s *models.Scene, source SceneReaderUpdater: instance.Repository.Scene, StudioCreator: instance.Repository.Studio, PerformerCreator: instance.Repository.Performer, - TagCreator: instance.Repository.Tag, + TagCreatorFinder: instance.Repository.Tag, DefaultOptions: j.input.Options, Sources: sources, @@ -248,14 +248,14 @@ type stashboxSource struct { endpoint string } -func (s stashboxSource) ScrapeScene(ctx context.Context, sceneID int) (*scraper.ScrapedScene, error) { +func (s stashboxSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) { results, err := s.FindStashBoxSceneByFingerprints(ctx, sceneID) if err != nil { return nil, fmt.Errorf("error querying stash-box using scene ID %d: %w", sceneID, err) } if len(results) > 0 { - return results[0], nil + return results, nil } return nil, nil @@ -270,7 +270,7 @@ type scraperSource struct { scraperID string } -func (s scraperSource) ScrapeScene(ctx context.Context, sceneID int) (*scraper.ScrapedScene, error) { +func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) { content, err := s.cache.ScrapeID(ctx, s.scraperID, sceneID, scraper.ScrapeContentTypeScene) if err != nil { return nil, err @@ -282,7 +282,7 @@ func (s scraperSource) ScrapeScene(ctx context.Context, sceneID int) (*scraper.S } if scene, ok := content.(scraper.ScrapedScene); ok { - return &scene, nil + return []*scraper.ScrapedScene{&scene}, nil } return nil, errors.New("could not convert content to scene") diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 65176bbea..73cf5b030 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -722,6 +722,9 @@ func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.Scen URL: findURL(s.Studio.Urls, "HOME"), RemoteSiteID: &studioID, } + if s.Studio.Images != nil && len(s.Studio.Images) > 0 { + ss.Studio.Image = &s.Studio.Images[0].URL + } err := match.ScrapedStudio(ctx, c.repository.Studio, ss.Studio, &c.box.Endpoint) if err != nil { diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/FieldOptions.tsx b/ui/v2.5/src/components/Dialogs/IdentifyDialog/FieldOptions.tsx index 68a31fe6b..79fe2b6c1 100644 --- a/ui/v2.5/src/components/Dialogs/IdentifyDialog/FieldOptions.tsx +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/FieldOptions.tsx @@ -311,7 +311,7 @@ export const FieldOptionsList: React.FC = ({ } return ( - +
diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/IdentifyDialog.tsx b/ui/v2.5/src/components/Dialogs/IdentifyDialog/IdentifyDialog.tsx index 7c5207f44..ece7589dc 100644 --- a/ui/v2.5/src/components/Dialogs/IdentifyDialog/IdentifyDialog.tsx +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/IdentifyDialog.tsx @@ -50,6 +50,10 @@ export const IdentifyDialog: React.FC = ({ includeMalePerformers: true, setCoverImage: true, setOrganized: false, + skipMultipleMatches: true, + skipMultipleMatchTag: undefined, + skipSingleNamePerformers: true, + skipSingleNamePerformerTag: undefined, }; } @@ -240,6 +244,8 @@ export const IdentifyDialog: React.FC = ({ const autoTagCopy = { ...autoTag }; autoTagCopy.options = { setOrganized: false, + skipMultipleMatches: true, + skipSingleNamePerformers: true, }; newSources.push(autoTagCopy); } diff --git a/ui/v2.5/src/components/Dialogs/IdentifyDialog/Options.tsx b/ui/v2.5/src/components/Dialogs/IdentifyDialog/Options.tsx index 88655c860..0bc31e6ae 100644 --- a/ui/v2.5/src/components/Dialogs/IdentifyDialog/Options.tsx +++ b/ui/v2.5/src/components/Dialogs/IdentifyDialog/Options.tsx @@ -1,10 +1,11 @@ import React from "react"; -import { Form } from "react-bootstrap"; +import { Col, Form, Row } from "react-bootstrap"; import * as GQL from "src/core/generated-graphql"; import { FormattedMessage, useIntl } from "react-intl"; import { IScraperSource } from "./constants"; import { FieldOptionsList } from "./FieldOptions"; import { ThreeStateBoolean } from "./ThreeStateBoolean"; +import { TagSelect } from "src/components/Shared/Select"; interface IOptionsEditor { options: GQL.IdentifyMetadataOptionsInput; @@ -35,8 +36,76 @@ export const OptionsEditor: React.FC = ({ indeterminateClassname: "text-muted", }; + function maybeRenderMultipleMatchesTag() { + if (!options.skipMultipleMatches) { + return; + } + + return ( + + + + + + + setOptions({ + skipMultipleMatchTag: tags[0]?.id, + }) + } + ids={ + options.skipMultipleMatchTag ? [options.skipMultipleMatchTag] : [] + } + noSelectionString="Select/create tag..." + /> + + + ); + } + + function maybeRenderPerformersTag() { + if (!options.skipSingleNamePerformers) { + return; + } + + return ( + + + + + + + setOptions({ + skipSingleNamePerformerTag: tags[0]?.id, + }) + } + ids={ + options.skipSingleNamePerformerTag + ? [options.skipSingleNamePerformerTag] + : [] + } + noSelectionString="Select/create tag..." + /> + + + ); + } + return ( - +
= ({ )} - + = ({ {...checkboxProps} /> + + setOptions({ + skipMultipleMatches: v, + }) + } + label={intl.formatMessage({ + id: "config.tasks.identify.skip_multiple_matches", + })} + defaultValue={defaultOptions?.skipMultipleMatches ?? undefined} + tooltip={intl.formatMessage({ + id: "config.tasks.identify.skip_multiple_matches_tooltip", + })} + {...checkboxProps} + /> + {maybeRenderMultipleMatchesTag()} + + setOptions({ + skipSingleNamePerformers: v, + }) + } + label={intl.formatMessage({ + id: "config.tasks.identify.skip_single_name_performers", + })} + defaultValue={defaultOptions?.skipSingleNamePerformers ?? undefined} + tooltip={intl.formatMessage({ + id: "config.tasks.identify.skip_single_name_performers_tooltip", + })} + {...checkboxProps} + /> + {maybeRenderPerformersTag()} = ({ @@ -20,6 +21,7 @@ export const ThreeStateBoolean: React.FC = ({ label, disabled, defaultValue, + tooltip, }) => { const intl = useIntl(); @@ -31,6 +33,7 @@ export const ThreeStateBoolean: React.FC = ({ checked={value} label={label} onChange={() => setValue(!value)} + title={tooltip} /> ); } @@ -79,7 +82,7 @@ export const ThreeStateBoolean: React.FC = ({ return ( -
{label}
+
{label}
{renderModeButton(undefined)} {renderModeButton(false)} diff --git a/ui/v2.5/src/components/Dialogs/styles.scss b/ui/v2.5/src/components/Dialogs/styles.scss index 36c350c2d..dfd8b555f 100644 --- a/ui/v2.5/src/components/Dialogs/styles.scss +++ b/ui/v2.5/src/components/Dialogs/styles.scss @@ -6,3 +6,13 @@ justify-content: space-between; } } + +.form-group { + h6, + label { + &[title]:not([title=""]) { + cursor: help; + text-decoration: underline dotted; + } + } +} diff --git a/ui/v2.5/src/docs/en/Manual/Identify.md b/ui/v2.5/src/docs/en/Manual/Identify.md index dceb8c1dc..8d0b3af2e 100644 --- a/ui/v2.5/src/docs/en/Manual/Identify.md +++ b/ui/v2.5/src/docs/en/Manual/Identify.md @@ -15,6 +15,10 @@ The following options can be set: | Include male performers | If false, then male performers will not be created or set on scenes. | | Set cover images | If false, then scene cover images will not be modified. | | Set organised flag | If true, the organised flag is set to true when a scene is organised. | +| Skip matches that have more than one result | If this is not enabled and more than one result is returned, one will be randomly chosen to match | +| Tag skipped matches with | If the above option is set and a scene is skipped, this will add the tag so that you can filter for it in the Scene Tagger view and choose the correct match by hand | +| Skip single name performers with no disambiguation | If this is not enabled, performers that are often generic like Samantha or Olga will be matched | +| Tag skipped performers with | If the above options is set and a performer is skipped, this will add the tag so that you can filter for in it the Scene Tagger view and choose how you want to handle those performers | Field specific options may be set as well. Each field may have a Strategy. The behaviour for each strategy value is as follows: diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index fbec7ddb2..67748505a 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -455,10 +455,18 @@ "include_male_performers": "Include male performers", "set_cover_images": "Set cover images", "set_organized": "Set organised flag", + "skip_multiple_matches": "Skip matches that have more than one result", + "skip_multiple_matches_tooltip": "If this is not enabled and more than one result is returned, one will be randomly chosen to match", + "skip_single_name_performers": "Skip single name performers with no disambiguation", + "skip_single_name_performers_tooltip": "If this is not enabled, performers that are often generic like Samantha or Olga will be matched", "source": "Source", "source_options": "{source} Options", "sources": "Sources", - "strategy": "Strategy" + "strategy": "Strategy", + "tag_skipped_matches": "Tag skipped matches with", + "tag_skipped_matches_tooltip": "Create a tag like 'Identify: Multiple Matches' that you can filter for in the Scene Tagger view and choose the correct match by hand", + "tag_skipped_performers": "Tag skipped performers with", + "tag_skipped_performer_tooltip": "Create a tag like 'Identify: Single Name Performer' that you can filter for in the Scene Tagger view and choose how you want to handle these performers" }, "import_from_exported_json": "Import from exported JSON in the metadata directory. Wipes the existing database.", "incremental_import": "Incremental import from a supplied export zip file.",