Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-06 08:26:00 +01:00
Code Issues Projects Releases Wiki Activity

Identify: Options to skip multiple results and single name performers (#3707)

Browse source 
Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
This commit is contained in:
Flashy78 2023-07-10 21:37:00 -07:00 committed by GitHub
parent ff22577ce0
commit cbdd4d3cbf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 581 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
graphql/documents/data/config.graphql
View file

@ -124,6 +124,10 @@ fragment IdentifyMetadataOptionsData on IdentifyMetadataOptions {
setCoverImage
setOrganized
includeMalePerformers
skipMultipleMatches
skipMultipleMatchTag
skipSingleNamePerformers
skipSingleNamePerformerTag
}
fragment ScraperSourceData on ScraperSource {

16
graphql/schema/types/metadata.graphql
View file

@ -185,6 +185,14 @@ input IdentifyMetadataOptionsInput {
setOrganized: Boolean
"""defaults to true if not provided"""
includeMalePerformers: Boolean
"""defaults to true if not provided"""
skipMultipleMatches: Boolean
"""tag to tag skipped multiple matches with"""
skipMultipleMatchTag: String
"""defaults to true if not provided"""
skipSingleNamePerformers: Boolean
"""tag to tag skipped single name performers with"""
skipSingleNamePerformerTag: String
}
input IdentifySourceInput {
@ -222,6 +230,14 @@ type IdentifyMetadataOptions {
setOrganized: Boolean
"""defaults to true if not provided"""
includeMalePerformers: Boolean
"""defaults to true if not provided"""
skipMultipleMatches: Boolean
"""tag to tag skipped multiple matches with"""
skipMultipleMatchTag: String
"""defaults to true if not provided"""
skipSingleNamePerformers: Boolean
"""tag to tag skipped single name performers with"""
skipSingleNamePerformerTag: String
}
type IdentifySource {

206
internal/identify/identify.go
View file

@ -2,18 +2,33 @@ package identify
import (
"context"
"errors"
"fmt"
"strconv"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stashapp/stash/pkg/txn"
"github.com/stashapp/stash/pkg/utils"
)
var (
ErrSkipSingleNamePerformer = errors.New("a performer was skipped because they only had a single name and no disambiguation")
)
type MultipleMatchesFoundError struct {
Source ScraperSource
}
func (e *MultipleMatchesFoundError) Error() string {
return fmt.Sprintf("multiple matches found for %s", e.Source.Name)
}
type SceneScraper interface {
ScrapeScene(ctx context.Context, sceneID int) (*scraper.ScrapedScene, error)
ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error)
}
type SceneUpdatePostHookExecutor interface {
@ -31,7 +46,7 @@ type SceneIdentifier struct {
SceneReaderUpdater SceneReaderUpdater
StudioCreator StudioCreator
PerformerCreator PerformerCreator
TagCreator TagCreator
TagCreatorFinder TagCreatorFinder
DefaultOptions *MetadataOptions
Sources []ScraperSource
@ -39,13 +54,31 @@ type SceneIdentifier struct {
}
func (t *SceneIdentifier) Identify(ctx context.Context, txnManager txn.Manager, scene *models.Scene) error {
result, err := t.scrapeScene(ctx, scene)
result, err := t.scrapeScene(ctx, txnManager, scene)
var multipleMatchErr *MultipleMatchesFoundError
if err != nil {
return err
if !errors.As(err, &multipleMatchErr) {
return err
}
}
if result == nil {
logger.Debugf("Unable to identify %s", scene.Path)
if multipleMatchErr != nil {
logger.Debugf("Identify skipped because multiple results returned for %s", scene.Path)
// find if the scene should be tagged for multiple results
options := t.getOptions(multipleMatchErr.Source)
if options.SkipMultipleMatchTag != nil && len(*options.SkipMultipleMatchTag) > 0 {
// Tag it with the multiple results tag
err := t.addTagToScene(ctx, txnManager, scene, *options.SkipMultipleMatchTag)
if err != nil {
return err
}
return nil
}
} else {
logger.Debugf("Unable to identify %s", scene.Path)
}
return nil
}
@ -62,63 +95,98 @@ type scrapeResult struct {
source ScraperSource
}
func (t *SceneIdentifier) scrapeScene(ctx context.Context, scene *models.Scene) (*scrapeResult, error) {
func (t *SceneIdentifier) scrapeScene(ctx context.Context, txnManager txn.Manager, scene *models.Scene) (*scrapeResult, error) {
// iterate through the input sources
for _, source := range t.Sources {
// scrape using the source
scraped, err := source.Scraper.ScrapeScene(ctx, scene.ID)
results, err := source.Scraper.ScrapeScenes(ctx, scene.ID)
if err != nil {
logger.Errorf("error scraping from %v: %v", source.Scraper, err)
continue
}
// if results were found then return
if scraped != nil {
return &scrapeResult{
result: scraped,
source: source,
}, nil
if len(results) > 0 {
options := t.getOptions(source)
if len(results) > 1 && utils.IsTrue(options.SkipMultipleMatches) {
return nil, &MultipleMatchesFoundError{
Source: source,
}
} else {
// if results were found then return
return &scrapeResult{
result: results[0],
source: source,
}, nil
}
}
}
return nil, nil
}
// Returns a MetadataOptions object with any default options overwritten by source specific options
func (t *SceneIdentifier) getOptions(source ScraperSource) MetadataOptions {
options := *t.DefaultOptions
if source.Options == nil {
return options
}
if source.Options.SetCoverImage != nil {
options.SetCoverImage = source.Options.SetCoverImage
}
if source.Options.SetOrganized != nil {
options.SetOrganized = source.Options.SetOrganized
}
if source.Options.IncludeMalePerformers != nil {
options.IncludeMalePerformers = source.Options.IncludeMalePerformers
}
if source.Options.SkipMultipleMatches != nil {
options.SkipMultipleMatches = source.Options.SkipMultipleMatches
}
if source.Options.SkipMultipleMatchTag != nil && len(*source.Options.SkipMultipleMatchTag) > 0 {
options.SkipMultipleMatchTag = source.Options.SkipMultipleMatchTag
}
if source.Options.SkipSingleNamePerformers != nil {
options.SkipSingleNamePerformers = source.Options.SkipSingleNamePerformers
}
if source.Options.SkipSingleNamePerformerTag != nil && len(*source.Options.SkipSingleNamePerformerTag) > 0 {
options.SkipSingleNamePerformerTag = source.Options.SkipSingleNamePerformerTag
}
return options
}
func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene, result *scrapeResult) (*scene.UpdateSet, error) {
ret := &scene.UpdateSet{
ID: s.ID,
}
options := []MetadataOptions{}
allOptions := []MetadataOptions{}
if result.source.Options != nil {
options = append(options, *result.source.Options)
allOptions = append(allOptions, *result.source.Options)
}
if t.DefaultOptions != nil {
options = append(options, *t.DefaultOptions)
allOptions = append(allOptions, *t.DefaultOptions)
}
fieldOptions := getFieldOptions(options)
setOrganized := false
for _, o := range options {
if o.SetOrganized != nil {
setOrganized = *o.SetOrganized
break
}
}
fieldOptions := getFieldOptions(allOptions)
options := t.getOptions(result.source)
scraped := result.result
rel := sceneRelationships{
sceneReader: t.SceneReaderUpdater,
studioCreator: t.StudioCreator,
performerCreator: t.PerformerCreator,
tagCreator: t.TagCreator,
scene: s,
result: result,
fieldOptions: fieldOptions,
sceneReader: t.SceneReaderUpdater,
studioCreator: t.StudioCreator,
performerCreator: t.PerformerCreator,
tagCreatorFinder: t.TagCreatorFinder,
scene: s,
result: result,
fieldOptions: fieldOptions,
skipSingleNamePerformers: *options.SkipSingleNamePerformers,
}
setOrganized := false
if options.SetOrganized != nil {
setOrganized = *options.SetOrganized
}
ret.Partial = getScenePartial(s, scraped, fieldOptions, setOrganized)
studioID, err := rel.studio(ctx)
@ -130,17 +198,19 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene,
ret.Partial.StudioID = models.NewOptionalInt(*studioID)
}
ignoreMale := false
for _, o := range options {
if o.IncludeMalePerformers != nil {
ignoreMale = !*o.IncludeMalePerformers
break
}
includeMalePerformers := true
if options.IncludeMalePerformers != nil {
includeMalePerformers = *options.IncludeMalePerformers
}
performerIDs, err := rel.performers(ctx, ignoreMale)
addSkipSingleNamePerformerTag := false
performerIDs, err := rel.performers(ctx, !includeMalePerformers)
if err != nil {
return nil, err
if errors.Is(err, ErrSkipSingleNamePerformer) {
addSkipSingleNamePerformerTag = true
} else {
return nil, err
}
}
if performerIDs != nil {
ret.Partial.PerformerIDs = &models.UpdateIDs{
@ -153,6 +223,14 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene,
if err != nil {
return nil, err
}
if addSkipSingleNamePerformerTag {
tagID, err := strconv.ParseInt(*options.SkipSingleNamePerformerTag, 10, 64)
if err != nil {
return nil, fmt.Errorf("error converting tag ID %s: %w", *options.SkipSingleNamePerformerTag, err)
}
tagIDs = intslice.IntAppendUnique(tagIDs, int(tagID))
}
if tagIDs != nil {
ret.Partial.TagIDs = &models.UpdateIDs{
IDs: tagIDs,
@ -171,15 +249,7 @@ func (t *SceneIdentifier) getSceneUpdater(ctx context.Context, s *models.Scene,
}
}
setCoverImage := false
for _, o := range options {
if o.SetCoverImage != nil {
setCoverImage = *o.SetCoverImage
break
}
}
if setCoverImage {
if options.SetCoverImage != nil && *options.SetCoverImage {
ret.CoverImage, err = rel.cover(ctx)
if err != nil {
return nil, err
@ -241,6 +311,41 @@ func (t *SceneIdentifier) modifyScene(ctx context.Context, txnManager txn.Manage
return nil
}
func (t *SceneIdentifier) addTagToScene(ctx context.Context, txnManager txn.Manager, s *models.Scene, tagToAdd string) error {
if err := txn.WithTxn(ctx, txnManager, func(ctx context.Context) error {
tagID, err := strconv.Atoi(tagToAdd)
if err != nil {
return fmt.Errorf("error converting tag ID %s: %w", tagToAdd, err)
}
if err := s.LoadTagIDs(ctx, t.SceneReaderUpdater); err != nil {
return err
}
existing := s.TagIDs.List()
if intslice.IntInclude(existing, tagID) {
// skip if the scene was already tagged
return nil
}
if err := scene.AddTag(ctx, t.SceneReaderUpdater, s, tagID); err != nil {
return err
}
ret, err := t.TagCreatorFinder.Find(ctx, tagID)
if err != nil {
logger.Infof("Added tag id %s to skipped scene %s", tagToAdd, s.Path)
} else {
logger.Infof("Added tag %s to skipped scene %s", ret.Name, s.Path)
}
return nil
}); err != nil {
return err
}
return nil
}
func getFieldOptions(options []MetadataOptions) map[string]*FieldOptions {
// prefer source-specific field strategies, then the defaults
ret := make(map[string]*FieldOptions)
@ -291,8 +396,7 @@ func getScenePartial(scene *models.Scene, scraped *scraper.ScrapedScene, fieldOp
}
if setOrganized && !scene.Organized {
// just reuse the boolean since we know it's true
partial.Organized = models.NewOptionalBool(setOrganized)
partial.Organized = models.NewOptionalBool(true)
}
return partial

118
internal/identify/identify_test.go
View file

@ -4,6 +4,7 @@ import (
"context"
"errors"
"reflect"
"strconv"
"testing"
"github.com/stashapp/stash/pkg/models"
@ -17,10 +18,10 @@ var testCtx = context.Background()
type mockSceneScraper struct {
errIDs []int
results map[int]*scraper.ScrapedScene
results map[int][]*scraper.ScrapedScene
}
func (s mockSceneScraper) ScrapeScene(ctx context.Context, sceneID int) (*scraper.ScrapedScene, error) {
func (s mockSceneScraper) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
if intslice.IntInclude(s.errIDs, sceneID) {
return nil, errors.New("scrape scene error")
}
@ -40,32 +41,66 @@ func TestSceneIdentifier_Identify(t *testing.T) {
missingID
found1ID
found2ID
multiFoundID
multiFound2ID
errUpdateID
)
var scrapedTitle = "scrapedTitle"
var (
skipMultipleTagID = 1
skipMultipleTagIDStr = strconv.Itoa(skipMultipleTagID)
)
defaultOptions := &MetadataOptions{}
var (
scrapedTitle = "scrapedTitle"
scrapedTitle2 = "scrapedTitle2"
boolFalse = false
boolTrue = true
)
defaultOptions := &MetadataOptions{
SetOrganized: &boolFalse,
SetCoverImage: &boolFalse,
IncludeMalePerformers: &boolFalse,
SkipSingleNamePerformers: &boolFalse,
}
sources := []ScraperSource{
{
Scraper: mockSceneScraper{
errIDs: []int{errID1},
results: map[int]*scraper.ScrapedScene{
found1ID: {
results: map[int][]*scraper.ScrapedScene{
found1ID: {{
Title: &scrapedTitle,
},
}},
},
},
},
{
Scraper: mockSceneScraper{
errIDs: []int{errID2},
results: map[int]*scraper.ScrapedScene{
found2ID: {
results: map[int][]*scraper.ScrapedScene{
found2ID: {{
Title: &scrapedTitle,
}},
errUpdateID: {{
Title: &scrapedTitle,
}},
multiFoundID: {
{
Title: &scrapedTitle,
},
{
Title: &scrapedTitle2,
},
},
errUpdateID: {
Title: &scrapedTitle,
multiFound2ID: {
{
Title: &scrapedTitle,
},
{
Title: &scrapedTitle2,
},
},
},
},
@ -73,6 +108,7 @@ func TestSceneIdentifier_Identify(t *testing.T) {
}
mockSceneReaderWriter := &mocks.SceneReaderWriter{}
mockTagFinderCreator := &mocks.TagReaderWriter{}
mockSceneReaderWriter.On("UpdatePartial", mock.Anything, mock.MatchedBy(func(id int) bool {
return id == errUpdateID
@ -81,52 +117,84 @@ func TestSceneIdentifier_Identify(t *testing.T) {
return id != errUpdateID
}), mock.Anything).Return(nil, nil)
mockTagFinderCreator.On("Find", mock.Anything, skipMultipleTagID).Return(&models.Tag{
ID: skipMultipleTagID,
Name: skipMultipleTagIDStr,
}, nil)
tests := []struct {
name string
sceneID int
options *MetadataOptions
wantErr bool
}{
{
"error scraping",
errID1,
nil,
false,
},
{
"error scraping from second",
errID2,
nil,
false,
},
{
"found in first scraper",
found1ID,
nil,
false,
},
{
"found in second scraper",
found2ID,
nil,
false,
},
{
"not found",
missingID,
nil,
false,
},
{
"error modifying",
errUpdateID,
nil,
true,
},
}
identifier := SceneIdentifier{
SceneReaderUpdater: mockSceneReaderWriter,
DefaultOptions: defaultOptions,
Sources: sources,
SceneUpdatePostHookExecutor: mockHookExecutor{},
{
"multiple found",
multiFoundID,
nil,
false,
},
{
"multiple found - set tag",
multiFound2ID,
&MetadataOptions{
SkipMultipleMatches: &boolTrue,
SkipMultipleMatchTag: &skipMultipleTagIDStr,
},
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
identifier := SceneIdentifier{
SceneReaderUpdater: mockSceneReaderWriter,
TagCreatorFinder: mockTagFinderCreator,
DefaultOptions: defaultOptions,
Sources: sources,
SceneUpdatePostHookExecutor: mockHookExecutor{},
}
if tt.options != nil {
identifier.DefaultOptions = tt.options
}
scene := &models.Scene{
ID: tt.sceneID,
PerformerIDs: models.NewRelatedIDs([]int{}),
@ -144,7 +212,16 @@ func TestSceneIdentifier_modifyScene(t *testing.T) {
repo := models.Repository{
TxnManager: &mocks.TxnManager{},
}
tr := &SceneIdentifier{}
boolFalse := false
defaultOptions := &MetadataOptions{
SetOrganized: &boolFalse,
SetCoverImage: &boolFalse,
IncludeMalePerformers: &boolFalse,
SkipSingleNamePerformers: &boolFalse,
}
tr := &SceneIdentifier{
DefaultOptions: defaultOptions,
}
type args struct {
scene *models.Scene
@ -165,6 +242,9 @@ func TestSceneIdentifier_modifyScene(t *testing.T) {
},
&scrapeResult{
result: &scraper.ScrapedScene{},
source: ScraperSource{
Options: defaultOptions,
},
},
},
false,

8
internal/identify/options.go
View file

@ -33,6 +33,14 @@ type MetadataOptions struct {
SetOrganized *bool `json:"setOrganized"`
// defaults to true if not provided
IncludeMalePerformers *bool `json:"includeMalePerformers"`
// defaults to true if not provided
SkipMultipleMatches *bool `json:"skipMultipleMatches"`
// ID of tag to tag skipped multiple matches with
SkipMultipleMatchTag *string `json:"skipMultipleMatchTag"`
// defaults to true if not provided
SkipSingleNamePerformers *bool `json:"skipSingleNamePerformers"`
// ID of tag to tag skipped single name performers with
SkipSingleNamePerformerTag *string `json:"skipSingleNamePerformerTag"`
}
type FieldOptions struct {

31
internal/identify/performer.go
View file

@ -4,17 +4,20 @@ import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
"github.com/stashapp/stash/pkg/utils"
)
type PerformerCreator interface {
Create(ctx context.Context, newPerformer *models.Performer) error
UpdateImage(ctx context.Context, performerID int, image []byte) error
}
func getPerformerID(ctx context.Context, endpoint string, w PerformerCreator, p *models.ScrapedPerformer, createMissing bool) (*int, error) {
func getPerformerID(ctx context.Context, endpoint string, w PerformerCreator, p *models.ScrapedPerformer, createMissing bool, skipSingleNamePerformers bool) (*int, error) {
if p.StoredID != nil {
// existing performer, just add it
performerID, err := strconv.Atoi(*p.StoredID)
@ -24,6 +27,10 @@ func getPerformerID(ctx context.Context, endpoint string, w PerformerCreator, p
return &performerID, nil
} else if createMissing && p.Name != nil { // name is mandatory
// skip single name performers with no disambiguation
if skipSingleNamePerformers && !strings.Contains(*p.Name, " ") && (p.Disambiguation == nil || len(*p.Disambiguation) == 0) {
return nil, ErrSkipSingleNamePerformer
}
return createMissingPerformer(ctx, endpoint, w, p)
}
@ -46,6 +53,19 @@ func createMissingPerformer(ctx context.Context, endpoint string, w PerformerCre
return nil, fmt.Errorf("error creating performer: %w", err)
}
// update image table
if p.Image != nil && len(*p.Image) > 0 {
imageData, err := utils.ReadImageFromURL(ctx, *p.Image)
if err != nil {
return nil, err
}
err = w.UpdateImage(ctx, performerInput.ID, imageData)
if err != nil {
return nil, err
}
}
return &performerInput.ID, nil
}
@ -56,6 +76,9 @@ func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performe
CreatedAt: currentTime,
UpdatedAt: currentTime,
}
if performer.Disambiguation != nil {
ret.Disambiguation = *performer.Disambiguation
}
if performer.Birthdate != nil {
d := models.NewDate(*performer.Birthdate)
ret.Birthdate = &d
@ -126,6 +149,12 @@ func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performe
if performer.Instagram != nil {
ret.Instagram = *performer.Instagram
}
if performer.URL != nil {
ret.URL = *performer.URL
}
if performer.Details != nil {
ret.Details = *performer.Details
}
return ret
}

108
internal/identify/performer_test.go
View file

@ -31,9 +31,10 @@ func Test_getPerformerID(t *testing.T) {
}).Return(nil)
type args struct {
endpoint string
p *models.ScrapedPerformer
createMissing bool
endpoint string
p *models.ScrapedPerformer
createMissing bool
skipSingleName bool
}
tests := []struct {
name string
@ -47,6 +48,7 @@ func Test_getPerformerID(t *testing.T) {
emptyEndpoint,
&models.ScrapedPerformer{},
false,
false,
},
nil,
false,
@ -59,6 +61,7 @@ func Test_getPerformerID(t *testing.T) {
StoredID: &invalidStoredID,
},
false,
false,
},
nil,
true,
@ -71,6 +74,7 @@ func Test_getPerformerID(t *testing.T) {
StoredID: &validStoredIDStr,
},
false,
false,
},
&validStoredID,
false,
@ -83,6 +87,7 @@ func Test_getPerformerID(t *testing.T) {
Name: &name,
},
false,
false,
},
nil,
false,
@ -93,10 +98,24 @@ func Test_getPerformerID(t *testing.T) {
emptyEndpoint,
&models.ScrapedPerformer{},
true,
false,
},
nil,
false,
},
{
"single name no disambig creating",
args{
emptyEndpoint,
&models.ScrapedPerformer{
Name: &name,
},
true,
true,
},
nil,
true,
},
{
"valid name creating",
args{
@ -105,6 +124,7 @@ func Test_getPerformerID(t *testing.T) {
Name: &name,
},
true,
false,
},
&validStoredID,
false,
@ -112,7 +132,7 @@ func Test_getPerformerID(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := getPerformerID(testCtx, tt.args.endpoint, &mockPerformerReaderWriter, tt.args.p, tt.args.createMissing)
got, err := getPerformerID(testCtx, tt.args.endpoint, &mockPerformerReaderWriter, tt.args.p, tt.args.createMissing, tt.args.skipSingleName)
if (err != nil) != tt.wantErr {
t.Errorf("getPerformerID() error = %v, wantErr %v", err, tt.wantErr)
return
@ -207,7 +227,7 @@ func Test_scrapedToPerformerInput(t *testing.T) {
name := "name"
var stringValues []string
for i := 0; i < 17; i++ {
for i := 0; i < 20; i++ {
stringValues = append(stringValues, strconv.Itoa(i))
}
@ -240,44 +260,50 @@ func Test_scrapedToPerformerInput(t *testing.T) {
{
"set all",
&models.ScrapedPerformer{
Name: &name,
Birthdate: nextVal(),
DeathDate: nextVal(),
Gender: nextVal(),
Ethnicity: nextVal(),
Country: nextVal(),
EyeColor: nextVal(),
HairColor: nextVal(),
Height: nextVal(),
Weight: nextVal(),
Measurements: nextVal(),
FakeTits: nextVal(),
CareerLength: nextVal(),
Tattoos: nextVal(),
Piercings: nextVal(),
Aliases: nextVal(),
Twitter: nextVal(),
Instagram: nextVal(),
Name: &name,
Disambiguation: nextVal(),
Birthdate: nextVal(),
DeathDate: nextVal(),
Gender: nextVal(),
Ethnicity: nextVal(),
Country: nextVal(),
EyeColor: nextVal(),
HairColor: nextVal(),
Height: nextVal(),
Weight: nextVal(),
Measurements: nextVal(),
FakeTits: nextVal(),
CareerLength: nextVal(),
Tattoos: nextVal(),
Piercings: nextVal(),
Aliases: nextVal(),
Twitter: nextVal(),
Instagram: nextVal(),
URL: nextVal(),
Details: nextVal(),
},
models.Performer{
Name: name,
Birthdate: dateToDatePtr(models.NewDate(*nextVal())),
DeathDate: dateToDatePtr(models.NewDate(*nextVal())),
Gender: genderPtr(models.GenderEnum(*nextVal())),
Ethnicity: *nextVal(),
Country: *nextVal(),
EyeColor: *nextVal(),
HairColor: *nextVal(),
Height: nextIntVal(),
Weight: nextIntVal(),
Measurements: *nextVal(),
FakeTits: *nextVal(),
CareerLength: *nextVal(),
Tattoos: *nextVal(),
Piercings: *nextVal(),
Aliases: models.NewRelatedStrings([]string{*nextVal()}),
Twitter: *nextVal(),
Instagram: *nextVal(),
Name: name,
Disambiguation: *nextVal(),
Birthdate: dateToDatePtr(models.NewDate(*nextVal())),
DeathDate: dateToDatePtr(models.NewDate(*nextVal())),
Gender: genderPtr(models.GenderEnum(*nextVal())),
Ethnicity: *nextVal(),
Country: *nextVal(),
EyeColor: *nextVal(),
HairColor: *nextVal(),
Height: nextIntVal(),
Weight: nextIntVal(),
Measurements: *nextVal(),
FakeTits: *nextVal(),
CareerLength: *nextVal(),
Tattoos: *nextVal(),
Piercings: *nextVal(),
Aliases: models.NewRelatedStrings([]string{*nextVal()}),
Twitter: *nextVal(),
Instagram: *nextVal(),
URL: *nextVal(),
Details: *nextVal(),
},
},
{

36
internal/identify/scene.go
View file

@ -3,6 +3,7 @@ package identify
import (
"bytes"
"context"
"errors"
"fmt"
"strconv"
"strings"
@ -13,6 +14,7 @@ import (
"github.com/stashapp/stash/pkg/scene"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stashapp/stash/pkg/tag"
"github.com/stashapp/stash/pkg/utils"
)
@ -24,18 +26,20 @@ type SceneReaderUpdater interface {
models.StashIDLoader
}
type TagCreator interface {
type TagCreatorFinder interface {
Create(ctx context.Context, newTag *models.Tag) error
tag.Finder
}
type sceneRelationships struct {
sceneReader SceneReaderUpdater
studioCreator StudioCreator
performerCreator PerformerCreator
tagCreator TagCreator
scene *models.Scene
result *scrapeResult
fieldOptions map[string]*FieldOptions
sceneReader SceneReaderUpdater
studioCreator StudioCreator
performerCreator PerformerCreator
tagCreatorFinder TagCreatorFinder
scene *models.Scene
result *scrapeResult
fieldOptions map[string]*FieldOptions
skipSingleNamePerformers bool
}
func (g sceneRelationships) studio(ctx context.Context) (*int, error) {
@ -93,13 +97,19 @@ func (g sceneRelationships) performers(ctx context.Context, ignoreMale bool) ([]
performerIDs = originalPerformerIDs
}
singleNamePerformerSkipped := false
for _, p := range scraped {
if ignoreMale && p.Gender != nil && strings.EqualFold(*p.Gender, models.GenderEnumMale.String()) {
continue
}
performerID, err := getPerformerID(ctx, endpoint, g.performerCreator, p, createMissing)
performerID, err := getPerformerID(ctx, endpoint, g.performerCreator, p, createMissing, g.skipSingleNamePerformers)
if err != nil {
if errors.Is(err, ErrSkipSingleNamePerformer) {
singleNamePerformerSkipped = true
continue
}
return nil, err
}
@ -110,9 +120,15 @@ func (g sceneRelationships) performers(ctx context.Context, ignoreMale bool) ([]
// don't return if nothing was added
if sliceutil.SliceSame(originalPerformerIDs, performerIDs) {
if singleNamePerformerSkipped {
return nil, ErrSkipSingleNamePerformer
}
return nil, nil
}
if singleNamePerformerSkipped {
return performerIDs, ErrSkipSingleNamePerformer
}
return performerIDs, nil
}
@ -156,7 +172,7 @@ func (g sceneRelationships) tags(ctx context.Context) ([]int, error) {
CreatedAt: now,
UpdatedAt: now,
}
err := g.tagCreator.Create(ctx, &newTag)
err := g.tagCreatorFinder.Create(ctx, &newTag)
if err != nil {
return nil, fmt.Errorf("error creating tag: %w", err)
}

6
internal/identify/scene_test.go
View file

@ -374,9 +374,9 @@ func Test_sceneRelationships_tags(t *testing.T) {
})).Return(errors.New("error creating tag"))
tr := sceneRelationships{
sceneReader: mockSceneReaderWriter,
tagCreator: mockTagReaderWriter,
fieldOptions: make(map[string]*FieldOptions),
sceneReader: mockSceneReaderWriter,
tagCreatorFinder: mockTagReaderWriter,
fieldOptions: make(map[string]*FieldOptions),
}
tests := []struct {

15
internal/identify/studio.go
View file

@ -7,11 +7,13 @@ import (
"github.com/stashapp/stash/pkg/hash/md5"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/utils"
)
type StudioCreator interface {
Create(ctx context.Context, newStudio *models.Studio) error
UpdateStashIDs(ctx context.Context, studioID int, stashIDs []models.StashID) error
UpdateImage(ctx context.Context, studioID int, image []byte) error
}
func createMissingStudio(ctx context.Context, endpoint string, w StudioCreator, studio *models.ScrapedStudio) (*int, error) {
@ -21,6 +23,19 @@ func createMissingStudio(ctx context.Context, endpoint string, w StudioCreator,
return nil, fmt.Errorf("error creating studio: %w", err)
}
// update image table
if studio.Image != nil && len(*studio.Image) > 0 {
imageData, err := utils.ReadImageFromURL(ctx, *studio.Image)
if err != nil {
return nil, err
}
err = w.UpdateImage(ctx, studioInput.ID, imageData)
if err != nil {
return nil, err
}
}
if endpoint != "" && studio.RemoteSiteID != nil {
if err := w.UpdateStashIDs(ctx, studioInput.ID, []models.StashID{
{

10
internal/manager/task_identify.go
View file

@ -136,7 +136,7 @@ func (j *IdentifyJob) identifyScene(ctx context.Context, s *models.Scene, source
SceneReaderUpdater: instance.Repository.Scene,
StudioCreator: instance.Repository.Studio,
PerformerCreator: instance.Repository.Performer,
TagCreator: instance.Repository.Tag,
TagCreatorFinder: instance.Repository.Tag,
DefaultOptions: j.input.Options,
Sources: sources,
@ -248,14 +248,14 @@ type stashboxSource struct {
endpoint string
}
func (s stashboxSource) ScrapeScene(ctx context.Context, sceneID int) (*scraper.ScrapedScene, error) {
func (s stashboxSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
results, err := s.FindStashBoxSceneByFingerprints(ctx, sceneID)
if err != nil {
return nil, fmt.Errorf("error querying stash-box using scene ID %d: %w", sceneID, err)
}
if len(results) > 0 {
return results[0], nil
return results, nil
}
return nil, nil
@ -270,7 +270,7 @@ type scraperSource struct {
scraperID string
}
func (s scraperSource) ScrapeScene(ctx context.Context, sceneID int) (*scraper.ScrapedScene, error) {
func (s scraperSource) ScrapeScenes(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
content, err := s.cache.ScrapeID(ctx, s.scraperID, sceneID, scraper.ScrapeContentTypeScene)
if err != nil {
return nil, err
@ -282,7 +282,7 @@ func (s scraperSource) ScrapeScene(ctx context.Context, sceneID int) (*scraper.S
}
if scene, ok := content.(scraper.ScrapedScene); ok {
return &scene, nil
return []*scraper.ScrapedScene{&scene}, nil
}
return nil, errors.New("could not convert content to scene")

3
pkg/scraper/stashbox/stash_box.go
View file

@ -722,6 +722,9 @@ func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.Scen
URL: findURL(s.Studio.Urls, "HOME"),
RemoteSiteID: &studioID,
}
if s.Studio.Images != nil && len(s.Studio.Images) > 0 {
ss.Studio.Image = &s.Studio.Images[0].URL
}
err := match.ScrapedStudio(ctx, c.repository.Studio, ss.Studio, &c.box.Endpoint)
if err != nil {

2
ui/v2.5/src/components/Dialogs/IdentifyDialog/FieldOptions.tsx
View file

@ -311,7 +311,7 @@ export const FieldOptionsList: React.FC<IFieldOptionsList> = ({
}
return (
<Form.Group className="scraper-sources">
<Form.Group className="scraper-sources mt-3">
<h5>
<FormattedMessage id="config.tasks.identify.field_options" />
</h5>

6
ui/v2.5/src/components/Dialogs/IdentifyDialog/IdentifyDialog.tsx
View file

@ -50,6 +50,10 @@ export const IdentifyDialog: React.FC<IIdentifyDialogProps> = ({
includeMalePerformers: true,
setCoverImage: true,
setOrganized: false,
skipMultipleMatches: true,
skipMultipleMatchTag: undefined,
skipSingleNamePerformers: true,
skipSingleNamePerformerTag: undefined,
};
}
@ -240,6 +244,8 @@ export const IdentifyDialog: React.FC<IIdentifyDialogProps> = ({
const autoTagCopy = { ...autoTag };
autoTagCopy.options = {
setOrganized: false,
skipMultipleMatches: true,
skipSingleNamePerformers: true,
};
newSources.push(autoTagCopy);
}

119
ui/v2.5/src/components/Dialogs/IdentifyDialog/Options.tsx
View file

@ -1,10 +1,11 @@
import React from "react";
import { Form } from "react-bootstrap";
import { Col, Form, Row } from "react-bootstrap";
import * as GQL from "src/core/generated-graphql";
import { FormattedMessage, useIntl } from "react-intl";
import { IScraperSource } from "./constants";
import { FieldOptionsList } from "./FieldOptions";
import { ThreeStateBoolean } from "./ThreeStateBoolean";
import { TagSelect } from "src/components/Shared/Select";
interface IOptionsEditor {
options: GQL.IdentifyMetadataOptionsInput;
@ -35,8 +36,76 @@ export const OptionsEditor: React.FC<IOptionsEditor> = ({
indeterminateClassname: "text-muted",
};
function maybeRenderMultipleMatchesTag() {
if (!options.skipMultipleMatches) {
return;
}
return (
<Form.Group controlId="match_tags" className="ml-3 mt-1 mb-0" as={Row}>
<Form.Label
column
sm={{ span: 4, offset: 1 }}
title={intl.formatMessage({
id: "config.tasks.identify.tag_skipped_matches_tooltip",
})}
>
<FormattedMessage id="config.tasks.identify.tag_skipped_matches" />
</Form.Label>
<Col sm>
<TagSelect
onSelect={(tags) =>
setOptions({
skipMultipleMatchTag: tags[0]?.id,
})
}
ids={
options.skipMultipleMatchTag ? [options.skipMultipleMatchTag] : []
}
noSelectionString="Select/create tag..."
/>
</Col>
</Form.Group>
);
}
function maybeRenderPerformersTag() {
if (!options.skipSingleNamePerformers) {
return;
}
return (
<Form.Group controlId="match_tags" className="ml-3 mt-1 mb-0" as={Row}>
<Form.Label
column
sm={{ span: 4, offset: 1 }}
title={intl.formatMessage({
id: "config.tasks.identify.tag_skipped_performer_tooltip",
})}
>
<FormattedMessage id="config.tasks.identify.tag_skipped_performers" />
</Form.Label>
<Col sm>
<TagSelect
onSelect={(tags) =>
setOptions({
skipSingleNamePerformerTag: tags[0]?.id,
})
}
ids={
options.skipSingleNamePerformerTag
? [options.skipSingleNamePerformerTag]
: []
}
noSelectionString="Select/create tag..."
/>
</Col>
</Form.Group>
);
}
return (
<Form.Group>
<Form.Group className="mb-0">
<Form.Group>
<h5>
<FormattedMessage
@ -52,7 +121,7 @@ export const OptionsEditor: React.FC<IOptionsEditor> = ({
</Form.Text>
)}
</Form.Group>
<Form.Group>
<Form.Group className="mb-0">
<ThreeStateBoolean
id="include-male-performers"
value={
@ -104,6 +173,50 @@ export const OptionsEditor: React.FC<IOptionsEditor> = ({
{...checkboxProps}
/>
</Form.Group>
<ThreeStateBoolean
id="skip-multiple-match"
value={
options.skipMultipleMatches === null
? undefined
: options.skipMultipleMatches
}
setValue={(v) =>
setOptions({
skipMultipleMatches: v,
})
}
label={intl.formatMessage({
id: "config.tasks.identify.skip_multiple_matches",
})}
defaultValue={defaultOptions?.skipMultipleMatches ?? undefined}
tooltip={intl.formatMessage({
id: "config.tasks.identify.skip_multiple_matches_tooltip",
})}
{...checkboxProps}
/>
{maybeRenderMultipleMatchesTag()}
<ThreeStateBoolean
id="skip-single-name-performers"
value={
options.skipSingleNamePerformers === null
? undefined
: options.skipSingleNamePerformers
}
setValue={(v) =>
setOptions({
skipSingleNamePerformers: v,
})
}
label={intl.formatMessage({
id: "config.tasks.identify.skip_single_name_performers",
})}
defaultValue={defaultOptions?.skipSingleNamePerformers ?? undefined}
tooltip={intl.formatMessage({
id: "config.tasks.identify.skip_single_name_performers_tooltip",
})}
{...checkboxProps}
/>
{maybeRenderPerformersTag()}
<FieldOptionsList
fieldOptions={options.fieldOptions ?? undefined}

5
ui/v2.5/src/components/Dialogs/IdentifyDialog/ThreeStateBoolean.tsx
View file

@ -10,6 +10,7 @@ interface IThreeStateBoolean {
label?: React.ReactNode;
disabled?: boolean;
defaultValue?: boolean;
tooltip?: string | undefined;
}
export const ThreeStateBoolean: React.FC<IThreeStateBoolean> = ({
@ -20,6 +21,7 @@ export const ThreeStateBoolean: React.FC<IThreeStateBoolean> = ({
label,
disabled,
defaultValue,
tooltip,
}) => {
const intl = useIntl();
@ -31,6 +33,7 @@ export const ThreeStateBoolean: React.FC<IThreeStateBoolean> = ({
checked={value}
label={label}
onChange={() => setValue(!value)}
title={tooltip}
/>
);
}
@ -79,7 +82,7 @@ export const ThreeStateBoolean: React.FC<IThreeStateBoolean> = ({
return (
<Form.Group>
<h6>{label}</h6>
<h6 title={tooltip}>{label}</h6>
<Form.Group>
{renderModeButton(undefined)}
{renderModeButton(false)}

10
ui/v2.5/src/components/Dialogs/styles.scss
View file

@ -6,3 +6,13 @@
justify-content: space-between;
}
}
.form-group {
h6,
label {
&[title]:not([title=""]) {
cursor: help;
text-decoration: underline dotted;
}
}
}

4
ui/v2.5/src/docs/en/Manual/Identify.md
View file

@ -15,6 +15,10 @@ The following options can be set:
| Include male performers | If false, then male performers will not be created or set on scenes. |
| Set cover images | If false, then scene cover images will not be modified. |
| Set organised flag | If true, the organised flag is set to true when a scene is organised. |
| Skip matches that have more than one result | If this is not enabled and more than one result is returned, one will be randomly chosen to match |
| Tag skipped matches with | If the above option is set and a scene is skipped, this will add the tag so that you can filter for it in the Scene Tagger view and choose the correct match by hand |
| Skip single name performers with no disambiguation | If this is not enabled, performers that are often generic like Samantha or Olga will be matched |
| Tag skipped performers with | If the above options is set and a performer is skipped, this will add the tag so that you can filter for in it the Scene Tagger view and choose how you want to handle those performers |
Field specific options may be set as well. Each field may have a Strategy. The behaviour for each strategy value is as follows:

10
ui/v2.5/src/locales/en-GB.json
View file

@ -455,10 +455,18 @@
"include_male_performers": "Include male performers",
"set_cover_images": "Set cover images",
"set_organized": "Set organised flag",
"skip_multiple_matches": "Skip matches that have more than one result",
"skip_multiple_matches_tooltip": "If this is not enabled and more than one result is returned, one will be randomly chosen to match",
"skip_single_name_performers": "Skip single name performers with no disambiguation",
"skip_single_name_performers_tooltip": "If this is not enabled, performers that are often generic like Samantha or Olga will be matched",
"source": "Source",
"source_options": "{source} Options",
"sources": "Sources",
"strategy": "Strategy"
"strategy": "Strategy",
"tag_skipped_matches": "Tag skipped matches with",
"tag_skipped_matches_tooltip": "Create a tag like 'Identify: Multiple Matches' that you can filter for in the Scene Tagger view and choose the correct match by hand",
"tag_skipped_performers": "Tag skipped performers with",
"tag_skipped_performer_tooltip": "Create a tag like 'Identify: Single Name Performer' that you can filter for in the Scene Tagger view and choose how you want to handle these performers"
},
"import_from_exported_json": "Import from exported JSON in the metadata directory. Wipes the existing database.",
"incremental_import": "Incremental import from a supplied export zip file.",