Backup_Repos/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-06 08:26:00 +01:00
Code Issues Projects Releases Wiki Activity

Fix bulk performer tagger (#4024)

Browse source 
* Fix tagger modal checkboxes
* Fix UNIQUE constraint detection
* Performer tagger cache invalidation
* Fix batch performer tagger
* Use ToPerformer in identify
* Add missing excluded fields
* Internationalize excluded fields
* Replace deprecated substr()
* Check RemoteSiteID nil
This commit is contained in:
DingDongSoLong4 2023-08-17 02:21:24 +02:00 committed by GitHub
parent 2bb04a623f
commit 1591180070
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 729 additions and 661 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

123
internal/identify/performer.go
View file

@ -5,11 +5,8 @@ import (
"fmt"
"strconv"
"strings"
"time"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
"github.com/stashapp/stash/pkg/utils"
)
type PerformerCreator interface {
@ -38,127 +35,23 @@ func getPerformerID(ctx context.Context, endpoint string, w PerformerCreator, p
}
func createMissingPerformer(ctx context.Context, endpoint string, w PerformerCreator, p *models.ScrapedPerformer) (*int, error) {
performerInput := scrapedToPerformerInput(p)
if endpoint != "" && p.RemoteSiteID != nil {
performerInput.StashIDs = models.NewRelatedStashIDs([]models.StashID{
{
Endpoint: endpoint,
StashID: *p.RemoteSiteID,
},
})
newPerformer := p.ToPerformer(endpoint, nil)
performerImage, err := p.GetImage(ctx, nil)
if err != nil {
return nil, err
}
err := w.Create(ctx, &performerInput)
err = w.Create(ctx, newPerformer)
if err != nil {
return nil, fmt.Errorf("error creating performer: %w", err)
}
// update image table
if p.Image != nil && len(*p.Image) > 0 {
imageData, err := utils.ReadImageFromURL(ctx, *p.Image)
if err != nil {
return nil, err
}
err = w.UpdateImage(ctx, performerInput.ID, imageData)
if err != nil {
if len(performerImage) > 0 {
if err := w.UpdateImage(ctx, newPerformer.ID, performerImage); err != nil {
return nil, err
}
}
return &performerInput.ID, nil
}
func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performer {
currentTime := time.Now()
ret := models.Performer{
Name: *performer.Name,
CreatedAt: currentTime,
UpdatedAt: currentTime,
}
if performer.Disambiguation != nil {
ret.Disambiguation = *performer.Disambiguation
}
if performer.Birthdate != nil {
d, err := models.ParseDate(*performer.Birthdate)
if err == nil {
ret.Birthdate = &d
}
}
if performer.DeathDate != nil {
d, err := models.ParseDate(*performer.DeathDate)
if err == nil {
ret.DeathDate = &d
}
}
if performer.Gender != nil {
v := models.GenderEnum(*performer.Gender)
ret.Gender = &v
}
if performer.Ethnicity != nil {
ret.Ethnicity = *performer.Ethnicity
}
if performer.Country != nil {
ret.Country = *performer.Country
}
if performer.EyeColor != nil {
ret.EyeColor = *performer.EyeColor
}
if performer.HairColor != nil {
ret.HairColor = *performer.HairColor
}
if performer.Height != nil {
h, err := strconv.Atoi(*performer.Height) // height is stored as an int
if err == nil {
ret.Height = &h
}
}
if performer.Weight != nil {
h, err := strconv.Atoi(*performer.Weight)
if err == nil {
ret.Weight = &h
}
}
if performer.Measurements != nil {
ret.Measurements = *performer.Measurements
}
if performer.FakeTits != nil {
ret.FakeTits = *performer.FakeTits
}
if performer.PenisLength != nil {
h, err := strconv.ParseFloat(*performer.PenisLength, 64)
if err == nil {
ret.PenisLength = &h
}
}
if performer.Circumcised != nil {
v := models.CircumisedEnum(*performer.Circumcised)
ret.Circumcised = &v
}
if performer.CareerLength != nil {
ret.CareerLength = *performer.CareerLength
}
if performer.Tattoos != nil {
ret.Tattoos = *performer.Tattoos
}
if performer.Piercings != nil {
ret.Piercings = *performer.Piercings
}
if performer.Aliases != nil {
ret.Aliases = models.NewRelatedStrings(stringslice.FromString(*performer.Aliases, ","))
}
if performer.Twitter != nil {
ret.Twitter = *performer.Twitter
}
if performer.Instagram != nil {
ret.Instagram = *performer.Instagram
}
if performer.URL != nil {
ret.URL = *performer.URL
}
if performer.Details != nil {
ret.Details = *performer.Details
}
return ret
return &newPerformer.ID, nil
}

130
internal/identify/performer_test.go
View file

@ -3,14 +3,11 @@ package identify
import (
"errors"
"reflect"
"strconv"
"testing"
"time"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/models/mocks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
@ -22,6 +19,7 @@ func Test_getPerformerID(t *testing.T) {
invalidStoredID := "invalidStoredID"
validStoredIDStr := "1"
validStoredID := 1
remoteSiteID := "2"
name := "name"
mockPerformerReaderWriter := mocks.PerformerReaderWriter{}
@ -121,7 +119,8 @@ func Test_getPerformerID(t *testing.T) {
args{
emptyEndpoint,
&models.ScrapedPerformer{
Name: &name,
Name: &name,
RemoteSiteID: &remoteSiteID,
},
true,
false,
@ -179,7 +178,8 @@ func Test_createMissingPerformer(t *testing.T) {
args{
emptyEndpoint,
&models.ScrapedPerformer{
Name: &validName,
Name: &validName,
RemoteSiteID: &remoteSiteID,
},
},
&performerID,
@ -190,7 +190,8 @@ func Test_createMissingPerformer(t *testing.T) {
args{
emptyEndpoint,
&models.ScrapedPerformer{
Name: &invalidName,
Name: &invalidName,
RemoteSiteID: &remoteSiteID,
},
},
nil,
@ -222,120 +223,3 @@ func Test_createMissingPerformer(t *testing.T) {
})
}
}
func Test_scrapedToPerformerInput(t *testing.T) {
name := "name"
var stringValues []string
for i := 0; i < 20; i++ {
stringValues = append(stringValues, strconv.Itoa(i))
}
upTo := 0
nextVal := func() *string {
ret := stringValues[upTo]
upTo = (upTo + 1) % len(stringValues)
return &ret
}
nextIntVal := func() *int {
ret := upTo
upTo = (upTo + 1) % len(stringValues)
return &ret
}
dateFromInt := func(i int) *models.Date {
t := time.Date(2001, 1, i, 0, 0, 0, 0, time.UTC)
d := models.Date{Time: t}
return &d
}
dateStrFromInt := func(i int) *string {
s := dateFromInt(i).String()
return &s
}
genderFromInt := func(i int) *models.GenderEnum {
g := models.AllGenderEnum[i%len(models.AllGenderEnum)]
return &g
}
genderStrFromInt := func(i int) *string {
s := genderFromInt(i).String()
return &s
}
tests := []struct {
name string
performer *models.ScrapedPerformer
want models.Performer
}{
{
"set all",
&models.ScrapedPerformer{
Name: &name,
Disambiguation: nextVal(),
Birthdate: dateStrFromInt(*nextIntVal()),
DeathDate: dateStrFromInt(*nextIntVal()),
Gender: genderStrFromInt(*nextIntVal()),
Ethnicity: nextVal(),
Country: nextVal(),
EyeColor: nextVal(),
HairColor: nextVal(),
Height: nextVal(),
Weight: nextVal(),
Measurements: nextVal(),
FakeTits: nextVal(),
CareerLength: nextVal(),
Tattoos: nextVal(),
Piercings: nextVal(),
Aliases: nextVal(),
Twitter: nextVal(),
Instagram: nextVal(),
URL: nextVal(),
Details: nextVal(),
},
models.Performer{
Name: name,
Disambiguation: *nextVal(),
Birthdate: dateFromInt(*nextIntVal()),
DeathDate: dateFromInt(*nextIntVal()),
Gender: genderFromInt(*nextIntVal()),
Ethnicity: *nextVal(),
Country: *nextVal(),
EyeColor: *nextVal(),
HairColor: *nextVal(),
Height: nextIntVal(),
Weight: nextIntVal(),
Measurements: *nextVal(),
FakeTits: *nextVal(),
CareerLength: *nextVal(),
Tattoos: *nextVal(),
Piercings: *nextVal(),
Aliases: models.NewRelatedStrings([]string{*nextVal()}),
Twitter: *nextVal(),
Instagram: *nextVal(),
URL: *nextVal(),
Details: *nextVal(),
},
},
{
"set none",
&models.ScrapedPerformer{
Name: &name,
},
models.Performer{
Name: name,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := scrapedToPerformerInput(tt.performer)
// clear created/updated dates
got.CreatedAt = time.Time{}
got.UpdatedAt = got.CreatedAt
assert.Equal(t, tt.want, got)
})
}
}

21
internal/manager/manager_tasks.go
View file

@ -410,13 +410,10 @@ func (s *Manager) StashBoxBatchPerformerTag(ctx context.Context, input StashBoxB
}
for i := range namesToUse {
if len(namesToUse[i]) > 0 {
performer := models.Performer{
Name: namesToUse[i],
}
name := namesToUse[i]
if len(name) > 0 {
tasks = append(tasks, StashBoxBatchTagTask{
performer: &performer,
name: &name,
refresh: false,
box: box,
excludedFields: input.ExcludeFields,
@ -435,6 +432,7 @@ func (s *Manager) StashBoxBatchPerformerTag(ctx context.Context, input StashBoxB
performerQuery := s.Repository.Performer
var performers []*models.Performer
var err error
if input.Refresh {
performers, err = performerQuery.FindByStashIDStatus(ctx, true, box.Endpoint)
} else {
@ -473,12 +471,9 @@ func (s *Manager) StashBoxBatchPerformerTag(ctx context.Context, input StashBoxB
logger.Infof("Starting stash-box batch operation for %d performers", len(tasks))
var wg sync.WaitGroup
for _, task := range tasks {
wg.Add(1)
progress.ExecuteTask(task.Description(), func() {
task.Start(ctx)
wg.Done()
})
progress.Increment()
@ -544,9 +539,10 @@ func (s *Manager) StashBoxBatchStudioTag(ctx context.Context, input StashBoxBatc
} else if len(input.Names) > 0 {
// The user is batch adding studios
for i := range input.Names {
if len(input.Names[i]) > 0 {
name := input.Names[i]
if len(name) > 0 {
tasks = append(tasks, StashBoxBatchTagTask{
name: &input.Names[i],
name: &name,
refresh: false,
createParent: input.CreateParent,
box: box,
@ -602,12 +598,9 @@ func (s *Manager) StashBoxBatchStudioTag(ctx context.Context, input StashBoxBatc
logger.Infof("Starting stash-box batch operation for %d studios", len(tasks))
var wg sync.WaitGroup
for _, task := range tasks {
wg.Add(1)
progress.ExecuteTask(task.Description(), func() {
task.Start(ctx)
wg.Done()
})
progress.Increment()

465
internal/manager/task_stash_box_tag.go
View file

@ -4,15 +4,12 @@ import (
"context"
"fmt"
"strconv"
"time"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scraper/stashbox"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
"github.com/stashapp/stash/pkg/studio"
"github.com/stashapp/stash/pkg/txn"
"github.com/stashapp/stash/pkg/utils"
)
type StashBoxTagTaskType int
@ -66,38 +63,9 @@ func (t *StashBoxBatchTagTask) Description() string {
}
func (t *StashBoxBatchTagTask) stashBoxPerformerTag(ctx context.Context) {
var performer *models.ScrapedPerformer
var err error
client := stashbox.NewClient(*t.box, instance.Repository, stashbox.Repository{
Scene: instance.Repository.Scene,
Performer: instance.Repository.Performer,
Tag: instance.Repository.Tag,
Studio: instance.Repository.Studio,
})
if t.refresh {
var performerID string
for _, id := range t.performer.StashIDs.List() {
if id.Endpoint == t.box.Endpoint {
performerID = id.StashID
}
}
if performerID != "" {
performer, err = client.FindStashBoxPerformerByID(ctx, performerID)
}
} else {
var name string
if t.name != nil {
name = *t.name
} else {
name = t.performer.Name
}
performer, err = client.FindStashBoxPerformerByName(ctx, name)
}
performer, err := t.findStashBoxPerformer(ctx)
if err != nil {
logger.Errorf("Error fetching performer data from stash-box: %s", err.Error())
logger.Errorf("Error fetching performer data from stash-box: %v", err)
return
}
@ -106,104 +74,9 @@ func (t *StashBoxBatchTagTask) stashBoxPerformerTag(ctx context.Context) {
excluded[field] = true
}
// performer will have a value if pulling from Stash-box by Stash ID or name was successful
if performer != nil {
if t.performer != nil {
partial := t.getPartial(performer, excluded)
txnErr := txn.WithTxn(ctx, instance.Repository, func(ctx context.Context) error {
r := instance.Repository
_, err := r.Performer.UpdatePartial(ctx, t.performer.ID, partial)
if len(performer.Images) > 0 && !excluded["image"] {
image, err := utils.ReadImageFromURL(ctx, performer.Images[0])
if err == nil {
err = r.Performer.UpdateImage(ctx, t.performer.ID, image)
if err != nil {
return err
}
} else {
logger.Warnf("Failed to read performer image: %v", err)
}
}
if err == nil {
var name string
if performer.Name != nil {
name = *performer.Name
}
logger.Infof("Updated performer %s", name)
}
return err
})
if txnErr != nil {
logger.Warnf("failure to execute partial update of performer: %v", txnErr)
}
} else if t.name != nil && performer.Name != nil {
currentTime := time.Now()
var aliases []string
if performer.Aliases != nil {
aliases = stringslice.FromString(*performer.Aliases, ",")
} else {
aliases = []string{}
}
newPerformer := models.Performer{
Aliases: models.NewRelatedStrings(aliases),
Disambiguation: getString(performer.Disambiguation),
Details: getString(performer.Details),
Birthdate: getDate(performer.Birthdate),
DeathDate: getDate(performer.DeathDate),
CareerLength: getString(performer.CareerLength),
Country: getString(performer.Country),
CreatedAt: currentTime,
Ethnicity: getString(performer.Ethnicity),
EyeColor: getString(performer.EyeColor),
HairColor: getString(performer.HairColor),
FakeTits: getString(performer.FakeTits),
Height: getIntPtr(performer.Height),
Weight: getIntPtr(performer.Weight),
Instagram: getString(performer.Instagram),
Measurements: getString(performer.Measurements),
Name: *performer.Name,
Piercings: getString(performer.Piercings),
Tattoos: getString(performer.Tattoos),
Twitter: getString(performer.Twitter),
URL: getString(performer.URL),
StashIDs: models.NewRelatedStashIDs([]models.StashID{
{
Endpoint: t.box.Endpoint,
StashID: *performer.RemoteSiteID,
},
}),
UpdatedAt: currentTime,
}
if performer.Gender != nil {
v := models.GenderEnum(getString(performer.Gender))
newPerformer.Gender = &v
}
err := txn.WithTxn(ctx, instance.Repository, func(ctx context.Context) error {
r := instance.Repository
err := r.Performer.Create(ctx, &newPerformer)
if err != nil {
return err
}
if len(performer.Images) > 0 {
image, imageErr := utils.ReadImageFromURL(ctx, performer.Images[0])
if imageErr != nil {
return imageErr
}
err = r.Performer.UpdateImage(ctx, newPerformer.ID, image)
}
return err
})
if err != nil {
logger.Errorf("Failed to save performer %s: %s", *t.name, err.Error())
} else {
logger.Infof("Saved performer %s", *t.name)
}
}
t.processMatchedPerformer(ctx, performer, excluded)
} else {
var name string
if t.name != nil {
@ -215,10 +88,131 @@ func (t *StashBoxBatchTagTask) stashBoxPerformerTag(ctx context.Context) {
}
}
func (t *StashBoxBatchTagTask) findStashBoxPerformer(ctx context.Context) (*models.ScrapedPerformer, error) {
var performer *models.ScrapedPerformer
var err error
client := stashbox.NewClient(*t.box, instance.Repository, stashbox.Repository{
Scene: instance.Repository.Scene,
Performer: instance.Repository.Performer,
Tag: instance.Repository.Tag,
Studio: instance.Repository.Studio,
})
if t.refresh {
var remoteID string
if err := txn.WithReadTxn(ctx, instance.Repository, func(ctx context.Context) error {
if !t.performer.StashIDs.Loaded() {
err = t.performer.LoadStashIDs(ctx, instance.Repository.Performer)
if err != nil {
return err
}
}
for _, id := range t.performer.StashIDs.List() {
if id.Endpoint == t.box.Endpoint {
remoteID = id.StashID
}
}
return nil
}); err != nil {
return nil, err
}
if remoteID != "" {
performer, err = client.FindStashBoxPerformerByID(ctx, remoteID)
}
} else {
var name string
if t.name != nil {
name = *t.name
} else {
name = t.performer.Name
}
performer, err = client.FindStashBoxPerformerByName(ctx, name)
}
return performer, err
}
func (t *StashBoxBatchTagTask) processMatchedPerformer(ctx context.Context, p *models.ScrapedPerformer, excluded map[string]bool) {
// Refreshing an existing performer
if t.performer != nil {
storedID, _ := strconv.Atoi(*p.StoredID)
existingStashIDs := getStashIDsForPerformer(ctx, storedID)
partial := p.ToPartial(t.box.Endpoint, excluded, existingStashIDs)
image, err := p.GetImage(ctx, excluded)
if err != nil {
logger.Errorf("Error processing scraped performer image for %s: %v", *p.Name, err)
return
}
// Start the transaction and update the performer
err = txn.WithTxn(ctx, instance.Repository, func(ctx context.Context) error {
qb := instance.Repository.Performer
if _, err := qb.UpdatePartial(ctx, t.performer.ID, partial); err != nil {
return err
}
if len(image) > 0 {
if err := qb.UpdateImage(ctx, t.performer.ID, image); err != nil {
return err
}
}
return nil
})
if err != nil {
logger.Errorf("Failed to update performer %s: %v", *p.Name, err)
} else {
logger.Infof("Updated performer %s", *p.Name)
}
} else if t.name != nil && p.Name != nil {
// Creating a new performer
newPerformer := p.ToPerformer(t.box.Endpoint, excluded)
image, err := p.GetImage(ctx, excluded)
if err != nil {
logger.Errorf("Error processing scraped performer image for %s: %v", *p.Name, err)
return
}
err = txn.WithTxn(ctx, instance.Repository, func(ctx context.Context) error {
qb := instance.Repository.Performer
if err := qb.Create(ctx, newPerformer); err != nil {
return err
}
if len(image) > 0 {
if err := qb.UpdateImage(ctx, newPerformer.ID, image); err != nil {
return err
}
}
return nil
})
if err != nil {
logger.Errorf("Failed to create performer %s: %v", *p.Name, err)
} else {
logger.Infof("Created performer %s", *p.Name)
}
}
}
func getStashIDsForPerformer(ctx context.Context, performerID int) []models.StashID {
tempPerformer := &models.Performer{ID: performerID}
err := tempPerformer.LoadStashIDs(ctx, instance.Repository.Performer)
if err != nil {
return nil
}
return tempPerformer.StashIDs.List()
}
func (t *StashBoxBatchTagTask) stashBoxStudioTag(ctx context.Context) {
studio, err := t.findStashBoxStudio(ctx)
if err != nil {
logger.Errorf("Error fetching studio data from stash-box: %s", err.Error())
logger.Errorf("Error fetching studio data from stash-box: %v", err)
return
}
@ -254,24 +248,20 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
if t.refresh {
var remoteID string
txnErr := txn.WithReadTxn(ctx, instance.Repository, func(ctx context.Context) error {
if err := txn.WithReadTxn(ctx, instance.Repository, func(ctx context.Context) error {
if !t.studio.StashIDs.Loaded() {
err = t.studio.LoadStashIDs(ctx, instance.Repository.Studio)
if err != nil {
return err
}
}
stashids := t.studio.StashIDs.List()
for _, id := range stashids {
for _, id := range t.studio.StashIDs.List() {
if id.Endpoint == t.box.Endpoint {
remoteID = id.StashID
}
}
return nil
})
if txnErr != nil {
logger.Warnf("error while executing read transaction: %v", err)
}); err != nil {
return nil, err
}
if remoteID != "" {
@ -293,6 +283,8 @@ func (t *StashBoxBatchTagTask) findStashBoxStudio(ctx context.Context) (*models.
func (t *StashBoxBatchTagTask) processMatchedStudio(ctx context.Context, s *models.ScrapedStudio, excluded map[string]bool) {
// Refreshing an existing studio
if t.studio != nil {
storedID, _ := strconv.Atoi(*s.StoredID)
if s.Parent != nil && t.createParent {
err := t.processParentStudio(ctx, s.Parent, excluded)
if err != nil {
@ -300,11 +292,12 @@ func (t *StashBoxBatchTagTask) processMatchedStudio(ctx context.Context, s *mode
}
}
existingStashIDs := getStashIDsForStudio(ctx, *s.StoredID)
studioPartial := s.ToPartial(s.StoredID, t.box.Endpoint, excluded, existingStashIDs)
studioImage, err := s.GetImage(ctx, excluded)
existingStashIDs := getStashIDsForStudio(ctx, storedID)
partial := s.ToPartial(s.StoredID, t.box.Endpoint, excluded, existingStashIDs)
image, err := s.GetImage(ctx, excluded)
if err != nil {
logger.Errorf("Failed to make studio partial from scraped studio %s: %s", s.Name, err.Error())
logger.Errorf("Error processing scraped studio image for %s: %v", s.Name, err)
return
}
@ -312,16 +305,16 @@ func (t *StashBoxBatchTagTask) processMatchedStudio(ctx context.Context, s *mode
err = txn.WithTxn(ctx, instance.Repository, func(ctx context.Context) error {
qb := instance.Repository.Studio
if err := studio.ValidateModify(ctx, *studioPartial, qb); err != nil {
if err := studio.ValidateModify(ctx, *partial, qb); err != nil {
return err
}
if _, err := qb.UpdatePartial(ctx, *studioPartial); err != nil {
if _, err := qb.UpdatePartial(ctx, *partial); err != nil {
return err
}
if len(studioImage) > 0 {
if err := qb.UpdateImage(ctx, studioPartial.ID, studioImage); err != nil {
if len(image) > 0 {
if err := qb.UpdateImage(ctx, partial.ID, image); err != nil {
return err
}
}
@ -329,7 +322,7 @@ func (t *StashBoxBatchTagTask) processMatchedStudio(ctx context.Context, s *mode
return nil
})
if err != nil {
logger.Errorf("Failed to update studio %s: %s", s.Name, err.Error())
logger.Errorf("Failed to update studio %s: %v", s.Name, err)
} else {
logger.Infof("Updated studio %s", s.Name)
}
@ -345,7 +338,7 @@ func (t *StashBoxBatchTagTask) processMatchedStudio(ctx context.Context, s *mode
newStudio := s.ToStudio(t.box.Endpoint, excluded)
studioImage, err := s.GetImage(ctx, excluded)
if err != nil {
logger.Errorf("Failed to make studio from scraped studio %s: %s", s.Name, err.Error())
logger.Errorf("Error processing scraped studio image for %s: %v", s.Name, err)
return
}
@ -365,7 +358,7 @@ func (t *StashBoxBatchTagTask) processMatchedStudio(ctx context.Context, s *mode
return nil
})
if err != nil {
logger.Errorf("Failed to create studio %s: %s", s.Name, err.Error())
logger.Errorf("Failed to create studio %s: %v", s.Name, err)
} else {
logger.Infof("Created studio %s", s.Name)
}
@ -376,9 +369,10 @@ func (t *StashBoxBatchTagTask) processParentStudio(ctx context.Context, parent *
if parent.StoredID == nil {
// The parent needs to be created
newParentStudio := parent.ToStudio(t.box.Endpoint, excluded)
studioImage, err := parent.GetImage(ctx, excluded)
image, err := parent.GetImage(ctx, excluded)
if err != nil {
logger.Errorf("Failed to make parent studio from scraped studio %s: %s", parent.Name, err.Error())
logger.Errorf("Error processing scraped studio image for %s: %v", parent.Name, err)
return err
}
@ -389,8 +383,8 @@ func (t *StashBoxBatchTagTask) processParentStudio(ctx context.Context, parent *
return err
}
if len(studioImage) > 0 {
if err := qb.UpdateImage(ctx, newParentStudio.ID, studioImage); err != nil {
if len(image) > 0 {
if err := qb.UpdateImage(ctx, newParentStudio.ID, image); err != nil {
return err
}
}
@ -400,17 +394,21 @@ func (t *StashBoxBatchTagTask) processParentStudio(ctx context.Context, parent *
return nil
})
if err != nil {
logger.Errorf("Failed to create studio %s: %s", parent.Name, err.Error())
return err
logger.Errorf("Failed to create studio %s: %v", parent.Name, err)
} else {
logger.Infof("Created studio %s", parent.Name)
}
logger.Infof("Created studio %s", parent.Name)
return err
} else {
storedID, _ := strconv.Atoi(*parent.StoredID)
// The parent studio matched an existing one and the user has chosen in the UI to link and/or update it
existingStashIDs := getStashIDsForStudio(ctx, *parent.StoredID)
studioPartial := parent.ToPartial(parent.StoredID, t.box.Endpoint, excluded, existingStashIDs)
studioImage, err := parent.GetImage(ctx, excluded)
existingStashIDs := getStashIDsForStudio(ctx, storedID)
partial := parent.ToPartial(parent.StoredID, t.box.Endpoint, excluded, existingStashIDs)
image, err := parent.GetImage(ctx, excluded)
if err != nil {
logger.Errorf("Failed to make parent studio partial from scraped studio %s: %s", parent.Name, err.Error())
logger.Errorf("Error processing scraped studio image for %s: %v", parent.Name, err)
return err
}
@ -418,16 +416,16 @@ func (t *StashBoxBatchTagTask) processParentStudio(ctx context.Context, parent *
err = txn.WithTxn(ctx, instance.Repository, func(ctx context.Context) error {
qb := instance.Repository.Studio
if err := studio.ValidateModify(ctx, *studioPartial, instance.Repository.Studio); err != nil {
if err := studio.ValidateModify(ctx, *partial, instance.Repository.Studio); err != nil {
return err
}
if _, err := qb.UpdatePartial(ctx, *studioPartial); err != nil {
if _, err := qb.UpdatePartial(ctx, *partial); err != nil {
return err
}
if len(studioImage) > 0 {
if err := qb.UpdateImage(ctx, studioPartial.ID, studioImage); err != nil {
if len(image) > 0 {
if err := qb.UpdateImage(ctx, partial.ID, image); err != nil {
return err
}
}
@ -435,17 +433,16 @@ func (t *StashBoxBatchTagTask) processParentStudio(ctx context.Context, parent *
return nil
})
if err != nil {
logger.Errorf("Failed to update studio %s: %s", parent.Name, err.Error())
return err
logger.Errorf("Failed to update studio %s: %v", parent.Name, err)
} else {
logger.Infof("Updated studio %s", parent.Name)
}
logger.Infof("Updated studio %s", parent.Name)
return err
}
return nil
}
func getStashIDsForStudio(ctx context.Context, studioID string) []models.StashID {
id, _ := strconv.Atoi(studioID)
tempStudio := &models.Studio{ID: id}
func getStashIDsForStudio(ctx context.Context, studioID int) []models.StashID {
tempStudio := &models.Studio{ID: studioID}
err := tempStudio.LoadStashIDs(ctx, instance.Repository.Studio)
if err != nil {
@ -453,127 +450,3 @@ func getStashIDsForStudio(ctx context.Context, studioID string) []models.StashID
}
return tempStudio.StashIDs.List()
}
func (t *StashBoxBatchTagTask) getPartial(performer *models.ScrapedPerformer, excluded map[string]bool) models.PerformerPartial {
partial := models.NewPerformerPartial()
if performer.Aliases != nil && !excluded["aliases"] {
partial.Aliases = &models.UpdateStrings{
Values: stringslice.FromString(*performer.Aliases, ","),
Mode: models.RelationshipUpdateModeSet,
}
}
if performer.Birthdate != nil && *performer.Birthdate != "" && !excluded["birthdate"] {
value := getDate(performer.Birthdate)
partial.Birthdate = models.NewOptionalDate(*value)
}
if performer.DeathDate != nil && *performer.DeathDate != "" && !excluded["deathdate"] {
value := getDate(performer.DeathDate)
partial.DeathDate = models.NewOptionalDate(*value)
}
if performer.CareerLength != nil && !excluded["career_length"] {
partial.CareerLength = models.NewOptionalString(*performer.CareerLength)
}
if performer.Country != nil && !excluded["country"] {
partial.Country = models.NewOptionalString(*performer.Country)
}
if performer.Ethnicity != nil && !excluded["ethnicity"] {
partial.Ethnicity = models.NewOptionalString(*performer.Ethnicity)
}
if performer.EyeColor != nil && !excluded["eye_color"] {
partial.EyeColor = models.NewOptionalString(*performer.EyeColor)
}
if performer.HairColor != nil && !excluded["hair_color"] {
partial.HairColor = models.NewOptionalString(*performer.HairColor)
}
if performer.FakeTits != nil && !excluded["fake_tits"] {
partial.FakeTits = models.NewOptionalString(*performer.FakeTits)
}
if performer.Gender != nil && !excluded["gender"] {
partial.Gender = models.NewOptionalString(*performer.Gender)
}
if performer.Height != nil && !excluded["height"] {
h, err := strconv.Atoi(*performer.Height)
if err == nil {
partial.Height = models.NewOptionalInt(h)
}
}
if performer.Weight != nil && !excluded["weight"] {
w, err := strconv.Atoi(*performer.Weight)
if err == nil {
partial.Weight = models.NewOptionalInt(w)
}
}
if performer.Instagram != nil && !excluded["instagram"] {
partial.Instagram = models.NewOptionalString(*performer.Instagram)
}
if performer.Measurements != nil && !excluded["measurements"] {
partial.Measurements = models.NewOptionalString(*performer.Measurements)
}
if performer.Name != nil && !excluded["name"] {
partial.Name = models.NewOptionalString(*performer.Name)
}
if performer.Disambiguation != nil && !excluded["disambiguation"] {
partial.Disambiguation = models.NewOptionalString(*performer.Disambiguation)
}
if performer.Piercings != nil && !excluded["piercings"] {
partial.Piercings = models.NewOptionalString(*performer.Piercings)
}
if performer.Tattoos != nil && !excluded["tattoos"] {
partial.Tattoos = models.NewOptionalString(*performer.Tattoos)
}
if performer.Twitter != nil && !excluded["twitter"] {
partial.Twitter = models.NewOptionalString(*performer.Twitter)
}
if performer.URL != nil && !excluded["url"] {
partial.URL = models.NewOptionalString(*performer.URL)
}
if !t.refresh {
// #3547 - need to overwrite the stash id for the endpoint, but preserve
// existing stash ids for other endpoints
partial.StashIDs = &models.UpdateStashIDs{
StashIDs: t.performer.StashIDs.List(),
Mode: models.RelationshipUpdateModeSet,
}
partial.StashIDs.Set(models.StashID{
Endpoint: t.box.Endpoint,
StashID: *performer.RemoteSiteID,
})
}
return partial
}
func getDate(val *string) *models.Date {
if val == nil {
return nil
}
ret, err := models.ParseDate(*val)
if err != nil {
return nil
}
return &ret
}
func getString(val *string) string {
if val == nil {
return ""
} else {
return *val
}
}
func getIntPtr(val *string) *int {
if val == nil {
return nil
} else {
v, err := strconv.Atoi(*val)
if err != nil {
return nil
}
return &v
}
}

247
pkg/models/model_scraped_item.go
View file

@ -5,6 +5,7 @@ import (
"strconv"
"time"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
"github.com/stashapp/stash/pkg/utils"
)
@ -26,22 +27,25 @@ func (s *ScrapedStudio) ToStudio(endpoint string, excluded map[string]bool) *Stu
// Populate a new studio from the input
newStudio := Studio{
Name: s.Name,
StashIDs: NewRelatedStashIDs([]StashID{
Name: s.Name,
CreatedAt: now,
UpdatedAt: now,
}
if s.RemoteSiteID != nil && endpoint != "" {
newStudio.StashIDs = NewRelatedStashIDs([]StashID{
{
Endpoint: endpoint,
StashID: *s.RemoteSiteID,
},
}),
CreatedAt: now,
UpdatedAt: now,
})
}
if s.URL != nil && !excluded["url"] {
newStudio.URL = *s.URL
}
if s.Parent != nil && s.Parent.StoredID != nil && !excluded["parent"] {
if s.Parent != nil && s.Parent.StoredID != nil && !excluded["parent"] && !excluded["parent_studio"] {
parentId, _ := strconv.Atoi(*s.Parent.StoredID)
newStudio.ParentID = &parentId
}
@ -90,16 +94,17 @@ func (s *ScrapedStudio) ToPartial(id *string, endpoint string, excluded map[stri
partial.ParentID = NewOptionalIntPtr(nil)
}
partial.StashIDs = &UpdateStashIDs{
StashIDs: existingStashIDs,
Mode: RelationshipUpdateModeSet,
if s.RemoteSiteID != nil && endpoint != "" {
partial.StashIDs = &UpdateStashIDs{
StashIDs: existingStashIDs,
Mode: RelationshipUpdateModeSet,
}
partial.StashIDs.Set(StashID{
Endpoint: endpoint,
StashID: *s.RemoteSiteID,
})
}
partial.StashIDs.Set(StashID{
Endpoint: endpoint,
StashID: *s.RemoteSiteID,
})
return &partial
}
@ -139,6 +144,220 @@ type ScrapedPerformer struct {
func (ScrapedPerformer) IsScrapedContent() {}
func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool) *Performer {
ret := NewPerformer(*p.Name)
if p.Aliases != nil && !excluded["aliases"] {
ret.Aliases = NewRelatedStrings(stringslice.FromString(*p.Aliases, ","))
}
if p.Birthdate != nil && !excluded["birthdate"] {
date, err := ParseDate(*p.Birthdate)
if err == nil {
ret.Birthdate = &date
}
}
if p.DeathDate != nil && !excluded["death_date"] {
date, err := ParseDate(*p.DeathDate)
if err == nil {
ret.DeathDate = &date
}
}
if p.CareerLength != nil && !excluded["career_length"] {
ret.CareerLength = *p.CareerLength
}
if p.Country != nil && !excluded["country"] {
ret.Country = *p.Country
}
if p.Ethnicity != nil && !excluded["ethnicity"] {
ret.Ethnicity = *p.Ethnicity
}
if p.EyeColor != nil && !excluded["eye_color"] {
ret.EyeColor = *p.EyeColor
}
if p.HairColor != nil && !excluded["hair_color"] {
ret.HairColor = *p.HairColor
}
if p.FakeTits != nil && !excluded["fake_tits"] {
ret.FakeTits = *p.FakeTits
}
if p.Gender != nil && !excluded["gender"] {
v := GenderEnum(*p.Gender)
if v.IsValid() {
ret.Gender = &v
}
}
if p.Height != nil && !excluded["height"] {
h, err := strconv.Atoi(*p.Height)
if err == nil {
ret.Height = &h
}
}
if p.Weight != nil && !excluded["weight"] {
w, err := strconv.Atoi(*p.Weight)
if err == nil {
ret.Weight = &w
}
}
if p.Instagram != nil && !excluded["instagram"] {
ret.Instagram = *p.Instagram
}
if p.Measurements != nil && !excluded["measurements"] {
ret.Measurements = *p.Measurements
}
if p.Disambiguation != nil && !excluded["disambiguation"] {
ret.Disambiguation = *p.Disambiguation
}
if p.Details != nil && !excluded["details"] {
ret.Details = *p.Details
}
if p.Piercings != nil && !excluded["piercings"] {
ret.Piercings = *p.Piercings
}
if p.Tattoos != nil && !excluded["tattoos"] {
ret.Tattoos = *p.Tattoos
}
if p.PenisLength != nil && !excluded["penis_length"] {
l, err := strconv.ParseFloat(*p.PenisLength, 64)
if err == nil {
ret.PenisLength = &l
}
}
if p.Circumcised != nil && !excluded["circumcised"] {
v := CircumisedEnum(*p.Circumcised)
if v.IsValid() {
ret.Circumcised = &v
}
}
if p.Twitter != nil && !excluded["twitter"] {
ret.Twitter = *p.Twitter
}
if p.URL != nil && !excluded["url"] {
ret.URL = *p.URL
}
if p.RemoteSiteID != nil && endpoint != "" {
ret.StashIDs = NewRelatedStashIDs([]StashID{
{
Endpoint: endpoint,
StashID: *p.RemoteSiteID,
},
})
}
return ret
}
func (p *ScrapedPerformer) GetImage(ctx context.Context, excluded map[string]bool) ([]byte, error) {
// Process the base 64 encoded image string
if len(p.Images) > 0 && !excluded["image"] {
var err error
img, err := utils.ProcessImageInput(ctx, p.Images[0])
if err != nil {
return nil, err
}
return img, nil
}
return nil, nil
}
func (p *ScrapedPerformer) ToPartial(endpoint string, excluded map[string]bool, existingStashIDs []StashID) PerformerPartial {
partial := NewPerformerPartial()
if p.Aliases != nil && !excluded["aliases"] {
partial.Aliases = &UpdateStrings{
Values: stringslice.FromString(*p.Aliases, ","),
Mode: RelationshipUpdateModeSet,
}
}
if p.Birthdate != nil && !excluded["birthdate"] {
date, err := ParseDate(*p.Birthdate)
if err == nil {
partial.Birthdate = NewOptionalDate(date)
}
}
if p.DeathDate != nil && !excluded["death_date"] {
date, err := ParseDate(*p.DeathDate)
if err == nil {
partial.DeathDate = NewOptionalDate(date)
}
}
if p.CareerLength != nil && !excluded["career_length"] {
partial.CareerLength = NewOptionalString(*p.CareerLength)
}
if p.Country != nil && !excluded["country"] {
partial.Country = NewOptionalString(*p.Country)
}
if p.Ethnicity != nil && !excluded["ethnicity"] {
partial.Ethnicity = NewOptionalString(*p.Ethnicity)
}
if p.EyeColor != nil && !excluded["eye_color"] {
partial.EyeColor = NewOptionalString(*p.EyeColor)
}
if p.HairColor != nil && !excluded["hair_color"] {
partial.HairColor = NewOptionalString(*p.HairColor)
}
if p.FakeTits != nil && !excluded["fake_tits"] {
partial.FakeTits = NewOptionalString(*p.FakeTits)
}
if p.Gender != nil && !excluded["gender"] {
partial.Gender = NewOptionalString(*p.Gender)
}
if p.Height != nil && !excluded["height"] {
h, err := strconv.Atoi(*p.Height)
if err == nil {
partial.Height = NewOptionalInt(h)
}
}
if p.Weight != nil && !excluded["weight"] {
w, err := strconv.Atoi(*p.Weight)
if err == nil {
partial.Weight = NewOptionalInt(w)
}
}
if p.Instagram != nil && !excluded["instagram"] {
partial.Instagram = NewOptionalString(*p.Instagram)
}
if p.Measurements != nil && !excluded["measurements"] {
partial.Measurements = NewOptionalString(*p.Measurements)
}
if p.Name != nil && !excluded["name"] {
partial.Name = NewOptionalString(*p.Name)
}
if p.Disambiguation != nil && !excluded["disambiguation"] {
partial.Disambiguation = NewOptionalString(*p.Disambiguation)
}
if p.Details != nil && !excluded["details"] {
partial.Details = NewOptionalString(*p.Details)
}
if p.Piercings != nil && !excluded["piercings"] {
partial.Piercings = NewOptionalString(*p.Piercings)
}
if p.Tattoos != nil && !excluded["tattoos"] {
partial.Tattoos = NewOptionalString(*p.Tattoos)
}
if p.Twitter != nil && !excluded["twitter"] {
partial.Twitter = NewOptionalString(*p.Twitter)
}
if p.URL != nil && !excluded["url"] {
partial.URL = NewOptionalString(*p.URL)
}
if p.RemoteSiteID != nil && endpoint != "" {
partial.StashIDs = &UpdateStashIDs{
StashIDs: existingStashIDs,
Mode: RelationshipUpdateModeSet,
}
partial.StashIDs.Set(StashID{
Endpoint: endpoint,
StashID: *p.RemoteSiteID,
})
}
return partial
}
type ScrapedTag struct {
// Set if tag matched
StoredID *string `json:"stored_id"`

198
pkg/models/model_scraped_item_test.go
View file

@ -1,6 +1,7 @@
package models
import (
"strconv"
"testing"
"time"
@ -10,12 +11,15 @@ import (
func Test_scrapedToStudioInput(t *testing.T) {
const name = "name"
url := "url"
emptyEndpoint := ""
endpoint := "endpoint"
remoteSiteID := "remoteSiteID"
tests := []struct {
name string
studio *ScrapedStudio
want *Studio
name string
studio *ScrapedStudio
endpoint string
want *Studio
}{
{
"set all",
@ -24,27 +28,51 @@ func Test_scrapedToStudioInput(t *testing.T) {
URL: &url,
RemoteSiteID: &remoteSiteID,
},
endpoint,
&Studio{
Name: name,
URL: url,
StashIDs: NewRelatedStashIDs([]StashID{
{
StashID: remoteSiteID,
Endpoint: endpoint,
StashID: remoteSiteID,
},
}),
},
},
{
"set none",
&ScrapedStudio{
Name: name,
},
emptyEndpoint,
&Studio{
Name: name,
},
},
{
"missing remoteSiteID",
&ScrapedStudio{
Name: name,
},
endpoint,
&Studio{
Name: name,
},
},
{
"set stashid",
&ScrapedStudio{
Name: name,
RemoteSiteID: &remoteSiteID,
},
endpoint,
&Studio{
Name: name,
StashIDs: NewRelatedStashIDs([]StashID{
{
StashID: remoteSiteID,
Endpoint: endpoint,
StashID: remoteSiteID,
},
}),
},
@ -52,7 +80,165 @@ func Test_scrapedToStudioInput(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.studio.ToStudio("", nil)
got := tt.studio.ToStudio(tt.endpoint, nil)
assert.NotEqual(t, time.Time{}, got.CreatedAt)
assert.NotEqual(t, time.Time{}, got.UpdatedAt)
got.CreatedAt = time.Time{}
got.UpdatedAt = time.Time{}
assert.Equal(t, tt.want, got)
})
}
}
func Test_scrapedToPerformerInput(t *testing.T) {
name := "name"
emptyEndpoint := ""
endpoint := "endpoint"
remoteSiteID := "remoteSiteID"
var stringValues []string
for i := 0; i < 20; i++ {
stringValues = append(stringValues, strconv.Itoa(i))
}
upTo := 0
nextVal := func() *string {
ret := stringValues[upTo]
upTo = (upTo + 1) % len(stringValues)
return &ret
}
nextIntVal := func() *int {
ret := upTo
upTo = (upTo + 1) % len(stringValues)
return &ret
}
dateFromInt := func(i int) *Date {
t := time.Date(2001, 1, i, 0, 0, 0, 0, time.UTC)
d := Date{Time: t}
return &d
}
dateStrFromInt := func(i int) *string {
s := dateFromInt(i).String()
return &s
}
genderFromInt := func(i int) *GenderEnum {
g := AllGenderEnum[i%len(AllGenderEnum)]
return &g
}
genderStrFromInt := func(i int) *string {
s := genderFromInt(i).String()
return &s
}
tests := []struct {
name string
performer *ScrapedPerformer
endpoint string
want *Performer
}{
{
"set all",
&ScrapedPerformer{
Name: &name,
Disambiguation: nextVal(),
Birthdate: dateStrFromInt(*nextIntVal()),
DeathDate: dateStrFromInt(*nextIntVal()),
Gender: genderStrFromInt(*nextIntVal()),
Ethnicity: nextVal(),
Country: nextVal(),
EyeColor: nextVal(),
HairColor: nextVal(),
Height: nextVal(),
Weight: nextVal(),
Measurements: nextVal(),
FakeTits: nextVal(),
CareerLength: nextVal(),
Tattoos: nextVal(),
Piercings: nextVal(),
Aliases: nextVal(),
Twitter: nextVal(),
Instagram: nextVal(),
URL: nextVal(),
Details: nextVal(),
RemoteSiteID: &remoteSiteID,
},
endpoint,
&Performer{
Name: name,
Disambiguation: *nextVal(),
Birthdate: dateFromInt(*nextIntVal()),
DeathDate: dateFromInt(*nextIntVal()),
Gender: genderFromInt(*nextIntVal()),
Ethnicity: *nextVal(),
Country: *nextVal(),
EyeColor: *nextVal(),
HairColor: *nextVal(),
Height: nextIntVal(),
Weight: nextIntVal(),
Measurements: *nextVal(),
FakeTits: *nextVal(),
CareerLength: *nextVal(),
Tattoos: *nextVal(),
Piercings: *nextVal(),
Aliases: NewRelatedStrings([]string{*nextVal()}),
Twitter: *nextVal(),
Instagram: *nextVal(),
URL: *nextVal(),
Details: *nextVal(),
StashIDs: NewRelatedStashIDs([]StashID{
{
Endpoint: endpoint,
StashID: remoteSiteID,
},
}),
},
},
{
"set none",
&ScrapedPerformer{
Name: &name,
},
emptyEndpoint,
&Performer{
Name: name,
},
},
{
"missing remoteSiteID",
&ScrapedPerformer{
Name: &name,
},
endpoint,
&Performer{
Name: name,
},
},
{
"set stashid",
&ScrapedPerformer{
Name: &name,
RemoteSiteID: &remoteSiteID,
},
endpoint,
&Performer{
Name: name,
StashIDs: NewRelatedStashIDs([]StashID{
{
Endpoint: endpoint,
StashID: remoteSiteID,
},
}),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.performer.ToPerformer(tt.endpoint, nil)
assert.NotEqual(t, time.Time{}, got.CreatedAt)
assert.NotEqual(t, time.Time{}, got.UpdatedAt)

33
pkg/scraper/stashbox/stash_box.go
View file

@ -369,7 +369,7 @@ func (c Client) queryStashBoxPerformer(ctx context.Context, queryStr string) ([]
var ret []*models.ScrapedPerformer
for _, fragment := range performerFragments {
performer := performerFragmentToScrapedScenePerformer(*fragment)
performer := performerFragmentToScrapedPerformer(*fragment)
ret = append(ret, performer)
}
@ -598,12 +598,12 @@ func fetchImage(ctx context.Context, client *http.Client, url string) (*string,
return &img, nil
}
func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *models.ScrapedPerformer {
id := p.ID
func performerFragmentToScrapedPerformer(p graphql.PerformerFragment) *models.ScrapedPerformer {
images := []string{}
for _, image := range p.Images {
images = append(images, image.URL)
}
sp := &models.ScrapedPerformer{
Name: &p.Name,
Disambiguation: p.Disambiguation,
@ -613,7 +613,7 @@ func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *mode
Tattoos: formatBodyModifications(p.Tattoos),
Piercings: formatBodyModifications(p.Piercings),
Twitter: findURL(p.Urls, "TWITTER"),
RemoteSiteID: &id,
RemoteSiteID: &p.ID,
Images: images,
// TODO - tags not currently supported
// graphql schema change to accommodate this. Leave off for now.
@ -772,7 +772,7 @@ func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.Scen
}
for _, p := range s.Performers {
sp := performerFragmentToScrapedScenePerformer(p.Performer)
sp := performerFragmentToScrapedPerformer(p.Performer)
err := match.ScrapedPerformer(ctx, pqb, sp, &c.box.Endpoint)
if err != nil {
@ -809,7 +809,15 @@ func (c Client) FindStashBoxPerformerByID(ctx context.Context, id string) (*mode
return nil, err
}
ret := performerFragmentToScrapedScenePerformer(*performer.FindPerformer)
ret := performerFragmentToScrapedPerformer(*performer.FindPerformer)
if err := txn.WithReadTxn(ctx, c.txnManager, func(ctx context.Context) error {
err := match.ScrapedPerformer(ctx, c.repository.Performer, ret, &c.box.Endpoint)
return err
}); err != nil {
return nil, err
}
return ret, nil
}
@ -822,10 +830,21 @@ func (c Client) FindStashBoxPerformerByName(ctx context.Context, name string) (*
var ret *models.ScrapedPerformer
for _, performer := range performers.SearchPerformer {
if strings.EqualFold(performer.Name, name) {
ret = performerFragmentToScrapedScenePerformer(*performer)
ret = performerFragmentToScrapedPerformer(*performer)
}
}
if ret == nil {
return nil, nil
}
if err := txn.WithReadTxn(ctx, c.txnManager, func(ctx context.Context) error {
err := match.ScrapedPerformer(ctx, c.repository.Performer, ret, &c.box.Endpoint)
return err
}); err != nil {
return nil, err
}
return ret, nil
}

22
ui/v2.5/src/components/Tagger/PerformerFieldSelector.tsx
View file

@ -5,17 +5,15 @@ import { useIntl } from "react-intl";
import { ModalComponent } from "../Shared/Modal";
import { Icon } from "../Shared/Icon";
import TextUtils from "src/utils/text";
import { PERFORMER_FIELDS } from "./constants";
interface IProps {
fields: string[];
show: boolean;
excludedFields: string[];
onSelect: (fields: string[]) => void;
}
const PerformerFieldSelect: React.FC<IProps> = ({
fields,
show,
excludedFields,
onSelect,
@ -25,22 +23,22 @@ const PerformerFieldSelect: React.FC<IProps> = ({
excludedFields.reduce((dict, field) => ({ ...dict, [field]: true }), {})
);
const toggleField = (name: string) =>
const toggleField = (field: string) =>
setExcluded({
...excluded,
[name]: !excluded[name],
[field]: !excluded[field],
});
const renderField = (name: string) => (
<Col xs={6} className="mb-1" key={name}>
const renderField = (field: string) => (
<Col xs={6} className="mb-1" key={field}>
<Button
onClick={() => toggleField(name)}
onClick={() => toggleField(field)}
variant="secondary"
className={excluded[name] ? "text-muted" : "text-success"}
className={excluded[field] ? "text-muted" : "text-success"}
>
<Icon icon={excluded[name] ? faTimes : faCheck} />
<Icon icon={excluded[field] ? faTimes : faCheck} />
</Button>
<span className="ml-3">{TextUtils.capitalize(name)}</span>
<span className="ml-3">{intl.formatMessage({ id: field })}</span>
</Col>
);
@ -59,7 +57,7 @@ const PerformerFieldSelect: React.FC<IProps> = ({
<div className="mb-2">
These fields will be tagged by default. Click the button to toggle.
</div>
<Row>{fields.map((f) => renderField(f))}</Row>
<Row>{PERFORMER_FIELDS.map((f) => renderField(f))}</Row>
</ModalComponent>
);
};

17
ui/v2.5/src/components/Tagger/constants.ts
View file

@ -58,26 +58,29 @@ export interface ITaggerConfig {
export const PERFORMER_FIELDS = [
"name",
"aliases",
"image",
"disambiguation",
"aliases",
"gender",
"birthdate",
"ethnicity",
"death_date",
"country",
"eye_color",
"ethnicity",
"hair_color",
"eye_color",
"height",
"weight",
"penis_length",
"circumcised",
"measurements",
"fake_tits",
"career_length",
"tattoos",
"piercings",
"career_length",
"url",
"twitter",
"instagram",
"details",
"death_date",
"weight",
];
export const STUDIO_FIELDS = ["name", "image", "url", "parent"];
export const STUDIO_FIELDS = ["name", "image", "url", "parent_studio"];

6
ui/v2.5/src/components/Tagger/performers/Config.tsx
View file

@ -3,8 +3,7 @@ import { Badge, Button, Card, Collapse, Form } from "react-bootstrap";
import { FormattedMessage } from "react-intl";
import { ConfigurationContext } from "src/hooks/Config";
import TextUtils from "src/utils/text";
import { ITaggerConfig, PERFORMER_FIELDS } from "../constants";
import { ITaggerConfig } from "../constants";
import PerformerFieldSelector from "../PerformerFieldSelector";
interface IConfigProps {
@ -52,7 +51,7 @@ const Config: React.FC<IConfigProps> = ({ show, config, setConfig }) => {
{excludedFields.length > 0 ? (
excludedFields.map((f) => (
<Badge variant="secondary" className="tag-item" key={f}>
{TextUtils.capitalize(f)}
<FormattedMessage id={f} />
</Badge>
))
) : (
@ -100,7 +99,6 @@ const Config: React.FC<IConfigProps> = ({ show, config, setConfig }) => {
</Card>
</Collapse>
<PerformerFieldSelector
fields={PERFORMER_FIELDS}
show={showExclusionModal}
onSelect={handleFieldSelect}
excludedFields={excludedFields}

58
ui/v2.5/src/components/Tagger/performers/PerformerTagger.tsx
View file

@ -12,6 +12,9 @@ import {
stashBoxPerformerQuery,
useJobsSubscribe,
mutateStashBoxBatchPerformerTag,
getClient,
evictQueries,
performerMutationImpactedQueries,
} from "src/core/StashService";
import { Manual } from "src/components/Help/Manual";
import { ConfigurationContext } from "src/hooks/Config";
@ -112,7 +115,7 @@ const PerformerBatchUpdateModal: React.FC<IPerformerBatchUpdateModal> = ({
type="radio"
name="performer-query"
label={<FormattedMessage id="performer_tagger.current_page" />}
defaultChecked={!queryAll}
checked={!queryAll}
onChange={() => setQueryAll(false)}
/>
<Form.Check
@ -122,8 +125,8 @@ const PerformerBatchUpdateModal: React.FC<IPerformerBatchUpdateModal> = ({
label={intl.formatMessage({
id: "performer_tagger.query_all_performers_in_the_database",
})}
defaultChecked={false}
onChange={() => setQueryAll(queryAll)}
checked={queryAll}
onChange={() => setQueryAll(true)}
/>
</Form.Group>
<Form.Group>
@ -139,7 +142,7 @@ const PerformerBatchUpdateModal: React.FC<IPerformerBatchUpdateModal> = ({
label={intl.formatMessage({
id: "performer_tagger.untagged_performers",
})}
defaultChecked={!refresh}
checked={!refresh}
onChange={() => setRefresh(false)}
/>
<Form.Text>
@ -152,8 +155,8 @@ const PerformerBatchUpdateModal: React.FC<IPerformerBatchUpdateModal> = ({
label={intl.formatMessage({
id: "performer_tagger.refresh_tagged_performers",
})}
defaultChecked={false}
onChange={() => setRefresh(refresh)}
checked={refresh}
onChange={() => setRefresh(true)}
/>
<Form.Text>
<FormattedMessage id="performer_tagger.refreshing_will_update_the_data" />
@ -346,6 +349,24 @@ const PerformerTaggerList: React.FC<IPerformerTaggerListProps> = ({
const updatePerformer = useUpdatePerformer();
function handleSaveError(performerID: string, name: string, message: string) {
setError({
...error,
[performerID]: {
message: intl.formatMessage(
{ id: "performer_tagger.failed_to_save_performer" },
{ studio: modalPerformer?.name }
),
details:
message === "UNIQUE constraint failed: performers.name"
? intl.formatMessage({
id: "performer_tagger.name_already_exists",
})
: message,
},
});
}
const handlePerformerUpdate = async (input: GQL.PerformerCreateInput) => {
setModalPerformer(undefined);
const performerID = modalPerformer?.stored_id;
@ -357,22 +378,11 @@ const PerformerTaggerList: React.FC<IPerformerTaggerListProps> = ({
const res = await updatePerformer(updateData);
if (!res.data?.performerUpdate)
setError({
...error,
[performerID]: {
message: intl.formatMessage(
{ id: "performer_tagger.failed_to_save_performer" },
{ performer: modalPerformer?.name }
),
details:
res?.errors?.[0].message ===
"UNIQUE constraint failed: performers.checksum"
? intl.formatMessage({
id: "performer_tagger.name_already_exists",
})
: res?.errors?.[0].message,
},
});
handleSaveError(
performerID,
modalPerformer?.name ?? "",
res?.errors?.[0]?.message ?? ""
);
}
};
@ -631,6 +641,10 @@ export const PerformerTagger: React.FC<ITaggerProps> = ({ performers }) => {
} else {
setBatchJob(undefined);
setBatchJobID(undefined);
// Once the performer batch is complete, refresh all local performer data
const ac = getClient();
evictQueries(ac.cache, performerMutationImpactedQueries);
}
}, [jobsSubscribe, batchJobID]);

7
ui/v2.5/src/components/Tagger/performers/StashSearchResult.tsx
View file

@ -24,9 +24,8 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
excludedPerformerFields,
endpoint,
}) => {
const [modalPerformer, setModalPerformer] = useState<
GQL.ScrapedPerformerDataFragment | undefined
>();
const [modalPerformer, setModalPerformer] =
useState<GQL.ScrapedPerformerDataFragment>();
const [saveState, setSaveState] = useState<string>("");
const [error, setError] = useState<{ message?: string; details?: string }>(
{}
@ -51,7 +50,7 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
message: `Failed to save performer "${performer.name}"`,
details:
res?.errors?.[0].message ===
"UNIQUE constraint failed: performers.checksum"
"UNIQUE constraint failed: performers.name"
? "Name already exists"
: res?.errors?.[0].message,
});

6
ui/v2.5/src/components/Tagger/studios/Config.tsx
View file

@ -3,8 +3,7 @@ import { Badge, Button, Card, Collapse, Form } from "react-bootstrap";
import { FormattedMessage } from "react-intl";
import { ConfigurationContext } from "src/hooks/Config";
import TextUtils from "src/utils/text";
import { ITaggerConfig, STUDIO_FIELDS } from "../constants";
import { ITaggerConfig } from "../constants";
import StudioFieldSelector from "./StudioFieldSelector";
interface IConfigProps {
@ -72,7 +71,7 @@ const Config: React.FC<IConfigProps> = ({ show, config, setConfig }) => {
{excludedFields.length > 0 ? (
excludedFields.map((f) => (
<Badge variant="secondary" className="tag-item" key={f}>
{TextUtils.capitalize(f)}
<FormattedMessage id={f} />
</Badge>
))
) : (
@ -120,7 +119,6 @@ const Config: React.FC<IConfigProps> = ({ show, config, setConfig }) => {
</Card>
</Collapse>
<StudioFieldSelector
fields={STUDIO_FIELDS}
show={showExclusionModal}
onSelect={handleFieldSelect}
excludedFields={excludedFields}

7
ui/v2.5/src/components/Tagger/studios/StashSearchResult.tsx
View file

@ -28,9 +28,8 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
}) => {
const intl = useIntl();
const [modalStudio, setModalStudio] = useState<
GQL.ScrapedStudioDataFragment | undefined
>();
const [modalStudio, setModalStudio] =
useState<GQL.ScrapedStudioDataFragment>();
const [saveState, setSaveState] = useState<string>("");
const [error, setError] = useState<{ message?: string; details?: string }>(
{}
@ -46,7 +45,7 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
{ studio: name }
),
details:
message === "UNIQUE constraint failed: studios.checksum"
message === "UNIQUE constraint failed: studios.name"
? "Name already exists"
: message,
});

22
ui/v2.5/src/components/Tagger/studios/StudioFieldSelector.tsx
View file

@ -5,17 +5,15 @@ import { useIntl } from "react-intl";
import { ModalComponent } from "../../Shared/Modal";
import { Icon } from "../../Shared/Icon";
import TextUtils from "src/utils/text";
import { STUDIO_FIELDS } from "../constants";
interface IProps {
fields: string[];
show: boolean;
excludedFields: string[];
onSelect: (fields: string[]) => void;
}
const StudioFieldSelect: React.FC<IProps> = ({
fields,
show,
excludedFields,
onSelect,
@ -25,22 +23,22 @@ const StudioFieldSelect: React.FC<IProps> = ({
excludedFields.reduce((dict, field) => ({ ...dict, [field]: true }), {})
);
const toggleField = (name: string) =>
const toggleField = (field: string) =>
setExcluded({
...excluded,
[name]: !excluded[name],
[field]: !excluded[field],
});
const renderField = (name: string) => (
<Col xs={6} className="mb-1" key={name}>
const renderField = (field: string) => (
<Col xs={6} className="mb-1" key={field}>
<Button
onClick={() => toggleField(name)}
onClick={() => toggleField(field)}
variant="secondary"
className={excluded[name] ? "text-muted" : "text-success"}
className={excluded[field] ? "text-muted" : "text-success"}
>
<Icon icon={excluded[name] ? faTimes : faCheck} />
<Icon icon={excluded[field] ? faTimes : faCheck} />
</Button>
<span className="ml-3">{TextUtils.capitalize(name)}</span>
<span className="ml-3">{intl.formatMessage({ id: field })}</span>
</Col>
);
@ -59,7 +57,7 @@ const StudioFieldSelect: React.FC<IProps> = ({
<div className="mb-2">
These fields will be tagged by default. Click the button to toggle.
</div>
<Row>{fields.map((f) => renderField(f))}</Row>
<Row>{STUDIO_FIELDS.map((f) => renderField(f))}</Row>
</ModalComponent>
);
};

10
ui/v2.5/src/components/Tagger/studios/StudioTagger.tsx
View file

@ -120,7 +120,7 @@ const StudioBatchUpdateModal: React.FC<IStudioBatchUpdateModal> = ({
type="radio"
name="studio-query"
label={<FormattedMessage id="studio_tagger.current_page" />}
defaultChecked={!queryAll}
checked={!queryAll}
onChange={() => setQueryAll(false)}
/>
<Form.Check
@ -130,7 +130,7 @@ const StudioBatchUpdateModal: React.FC<IStudioBatchUpdateModal> = ({
label={intl.formatMessage({
id: "studio_tagger.query_all_studios_in_the_database",
})}
defaultChecked={queryAll}
checked={queryAll}
onChange={() => setQueryAll(true)}
/>
</Form.Group>
@ -147,7 +147,7 @@ const StudioBatchUpdateModal: React.FC<IStudioBatchUpdateModal> = ({
label={intl.formatMessage({
id: "studio_tagger.untagged_studios",
})}
defaultChecked={!refresh}
checked={!refresh}
onChange={() => setRefresh(false)}
/>
<Form.Text>
@ -160,7 +160,7 @@ const StudioBatchUpdateModal: React.FC<IStudioBatchUpdateModal> = ({
label={intl.formatMessage({
id: "studio_tagger.refresh_tagged_studios",
})}
defaultChecked={refresh}
checked={refresh}
onChange={() => setRefresh(true)}
/>
<Form.Text>
@ -400,7 +400,7 @@ const StudioTaggerList: React.FC<IStudioTaggerListProps> = ({
{ studio: modalStudio?.name }
),
details:
message === "UNIQUE constraint failed: studios.checksum"
message === "UNIQUE constraint failed: studios.name"
? intl.formatMessage({
id: "studio_tagger.name_already_exists",
})

2
ui/v2.5/src/core/StashService.ts
View file

@ -1358,7 +1358,7 @@ const performerMutationImpactedTypeFields = {
Tag: ["performer_count"],
};
const performerMutationImpactedQueries = [
export const performerMutationImpactedQueries = [
GQL.FindScenesDocument, // filter by performer tags
GQL.FindImagesDocument, // filter by performer tags
GQL.FindGalleriesDocument, // filter by performer tags

4
ui/v2.5/src/utils/duration.ts
View file

@ -4,10 +4,10 @@ const secondsToString = (seconds: number) => {
let ret = TextUtils.secondsToTimestamp(seconds);
if (ret.startsWith("00:")) {
ret = ret.substr(3);
ret = ret.substring(3);
if (ret.startsWith("0")) {
ret = ret.substr(1);
ret = ret.substring(1);
}
}

12
ui/v2.5/src/utils/text.ts
View file

@ -157,15 +157,15 @@ const fileSizeFractionalDigits = (unit: Unit) => {
};
const secondsToTimestamp = (seconds: number) => {
let ret = new Date(seconds * 1000).toISOString().substr(11, 8);
let ret = new Date(seconds * 1000).toISOString().substring(11, 19);
if (ret.startsWith("00")) {
// strip hours if under one hour
ret = ret.substr(3);
ret = ret.substring(3);
}
if (ret.startsWith("0")) {
// for duration under a minute, leave one leading zero
ret = ret.substr(1);
ret = ret.substring(1);
}
return ret;
};
@ -387,11 +387,6 @@ const formatDateTime = (intl: IntlShape, dateTime?: string, utc = false) =>
timeZone: utc ? "utc" : undefined,
})}`;
const capitalize = (val: string) =>
val
.replace(/^[-_]*(.)/, (_, c) => c.toUpperCase())
.replace(/[-_]+(.)/g, (_, c) => ` ${c.toUpperCase()}`);
type CountUnit = "" | "K" | "M" | "B";
const CountUnits: CountUnit[] = ["", "K", "M", "B"];
@ -435,7 +430,6 @@ const TextUtils = {
instagramURL,
formatDate,
formatDateTime,
capitalize,
secondsAsTimeString,
abbreviateCounter,
};