From f26766033e03fba06f2b4bd9d74ea2f0f469b57e Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 18 Jun 2024 13:41:05 +1000 Subject: [PATCH] Performer urls (#4958) * Populate URLs from legacy fields * Return nil properly in xpath/json scrapers * Improve migration logging --- graphql/schema/types/performer.graphql | 28 +-- .../schema/types/scraped-performer.graphql | 14 +- internal/api/resolver_model_performer.go | 73 ++++++++ internal/api/resolver_mutation_performer.go | 159 +++++++++++++++++- internal/manager/task/migrate.go | 31 ++-- pkg/models/jsonschema/performer.go | 23 +-- pkg/models/mocks/PerformerReaderWriter.go | 23 +++ pkg/models/model_performer.go | 14 +- pkg/models/model_scraped_item.go | 72 +++++--- pkg/models/model_scraped_item_test.go | 6 +- pkg/models/performer.go | 14 +- pkg/models/repository_performer.go | 1 + pkg/performer/export.go | 9 +- pkg/performer/export_test.go | 10 +- pkg/performer/import.go | 22 ++- pkg/performer/url.go | 18 ++ pkg/scraper/json.go | 26 ++- pkg/scraper/performer.go | 51 +++--- pkg/scraper/postprocessing.go | 26 +++ pkg/scraper/scraper.go | 6 + pkg/scraper/stashbox/stash_box.go | 34 ++-- pkg/scraper/xpath.go | 26 ++- pkg/sqlite/anonymise.go | 16 +- pkg/sqlite/database.go | 2 +- .../migrations/62_performer_urls.up.sql | 155 +++++++++++++++++ pkg/sqlite/performer.go | 38 +++-- pkg/sqlite/performer_filter.go | 19 ++- pkg/sqlite/performer_test.go | 72 +++++--- pkg/sqlite/setup_test.go | 33 ++-- pkg/sqlite/tables.go | 9 + pkg/utils/url.go | 15 ++ pkg/utils/url_test.go | 47 ++++++ ui/v2.5/graphql/data/performer-slim.graphql | 4 +- ui/v2.5/graphql/data/performer.graphql | 4 +- ui/v2.5/graphql/data/scrapers.graphql | 8 +- .../Performers/EditPerformersDialog.tsx | 12 -- .../Performers/PerformerDetails/Performer.tsx | 84 ++++----- .../PerformerDetails/PerformerEditPanel.tsx | 40 +---- .../PerformerScrapeDialog.tsx | 43 ++--- .../PerformerDetails/PerformerScrapeModal.tsx | 4 +- .../components/Shared/ExternalLinksButton.tsx | 7 +- .../src/components/Tagger/PerformerModal.tsx | 50 +++++- ui/v2.5/src/components/Tagger/constants.ts | 6 +- ui/v2.5/src/components/Tagger/styles.scss | 6 + ui/v2.5/src/core/performers.ts | 4 +- .../models/list-filter/criteria/is-missing.ts | 2 - ui/v2.5/src/utils/text.ts | 5 - 47 files changed, 992 insertions(+), 379 deletions(-) create mode 100644 pkg/performer/url.go create mode 100644 pkg/sqlite/migrations/62_performer_urls.up.sql create mode 100644 pkg/utils/url.go create mode 100644 pkg/utils/url_test.go diff --git a/graphql/schema/types/performer.graphql b/graphql/schema/types/performer.graphql index c5d328425..d6d6b2696 100644 --- a/graphql/schema/types/performer.graphql +++ b/graphql/schema/types/performer.graphql @@ -16,10 +16,11 @@ type Performer { id: ID! name: String! disambiguation: String - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!] gender: GenderEnum - twitter: String - instagram: String + twitter: String @deprecated(reason: "Use urls") + instagram: String @deprecated(reason: "Use urls") birthdate: String ethnicity: String country: String @@ -60,7 +61,8 @@ type Performer { input PerformerCreateInput { name: String! disambiguation: String - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!] gender: GenderEnum birthdate: String ethnicity: String @@ -75,8 +77,8 @@ input PerformerCreateInput { tattoos: String piercings: String alias_list: [String!] - twitter: String - instagram: String + twitter: String @deprecated(reason: "Use urls") + instagram: String @deprecated(reason: "Use urls") favorite: Boolean tag_ids: [ID!] "This should be a URL or a base64 encoded data URL" @@ -95,7 +97,8 @@ input PerformerUpdateInput { id: ID! name: String disambiguation: String - url: String + url: String @deprecated(reason: "Use urls") + urls: [String!] gender: GenderEnum birthdate: String ethnicity: String @@ -110,8 +113,8 @@ input PerformerUpdateInput { tattoos: String piercings: String alias_list: [String!] - twitter: String - instagram: String + twitter: String @deprecated(reason: "Use urls") + instagram: String @deprecated(reason: "Use urls") favorite: Boolean tag_ids: [ID!] "This should be a URL or a base64 encoded data URL" @@ -135,7 +138,8 @@ input BulkPerformerUpdateInput { clientMutationId: String ids: [ID!] disambiguation: String - url: String + url: String @deprecated(reason: "Use urls") + urls: BulkUpdateStrings gender: GenderEnum birthdate: String ethnicity: String @@ -150,8 +154,8 @@ input BulkPerformerUpdateInput { tattoos: String piercings: String alias_list: BulkUpdateStrings - twitter: String - instagram: String + twitter: String @deprecated(reason: "Use urls") + instagram: String @deprecated(reason: "Use urls") favorite: Boolean tag_ids: BulkUpdateIds # rating expressed as 1-100 diff --git a/graphql/schema/types/scraped-performer.graphql b/graphql/schema/types/scraped-performer.graphql index 92ba94d32..487c89516 100644 --- a/graphql/schema/types/scraped-performer.graphql +++ b/graphql/schema/types/scraped-performer.graphql @@ -5,9 +5,10 @@ type ScrapedPerformer { name: String disambiguation: String gender: String - url: String - twitter: String - instagram: String + url: String @deprecated(reason: "use urls") + urls: [String!] + twitter: String @deprecated(reason: "use urls") + instagram: String @deprecated(reason: "use urls") birthdate: String ethnicity: String country: String @@ -40,9 +41,10 @@ input ScrapedPerformerInput { name: String disambiguation: String gender: String - url: String - twitter: String - instagram: String + url: String @deprecated(reason: "use urls") + urls: [String!] + twitter: String @deprecated(reason: "use urls") + instagram: String @deprecated(reason: "use urls") birthdate: String ethnicity: String country: String diff --git a/internal/api/resolver_model_performer.go b/internal/api/resolver_model_performer.go index 6164ff297..58fac77ff 100644 --- a/internal/api/resolver_model_performer.go +++ b/internal/api/resolver_model_performer.go @@ -24,6 +24,79 @@ func (r *performerResolver) AliasList(ctx context.Context, obj *models.Performer return obj.Aliases.List(), nil } +func (r *performerResolver) URL(ctx context.Context, obj *models.Performer) (*string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Performer) + }); err != nil { + return nil, err + } + } + + urls := obj.URLs.List() + if len(urls) == 0 { + return nil, nil + } + + return &urls[0], nil +} + +func (r *performerResolver) Twitter(ctx context.Context, obj *models.Performer) (*string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Performer) + }); err != nil { + return nil, err + } + } + + urls := obj.URLs.List() + + // find the first twitter url + for _, url := range urls { + if performer.IsTwitterURL(url) { + u := url + return &u, nil + } + } + + return nil, nil +} + +func (r *performerResolver) Instagram(ctx context.Context, obj *models.Performer) (*string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Performer) + }); err != nil { + return nil, err + } + } + + urls := obj.URLs.List() + + // find the first instagram url + for _, url := range urls { + if performer.IsInstagramURL(url) { + u := url + return &u, nil + } + } + + return nil, nil +} + +func (r *performerResolver) Urls(ctx context.Context, obj *models.Performer) ([]string, error) { + if !obj.URLs.Loaded() { + if err := r.withReadTxn(ctx, func(ctx context.Context) error { + return obj.LoadURLs(ctx, r.repository.Performer) + }); err != nil { + return nil, err + } + } + + return obj.URLs.List(), nil +} + func (r *performerResolver) Height(ctx context.Context, obj *models.Performer) (*string, error) { if obj.Height != nil { ret := strconv.Itoa(*obj.Height) diff --git a/internal/api/resolver_mutation_performer.go b/internal/api/resolver_mutation_performer.go index 202778e74..7263cc709 100644 --- a/internal/api/resolver_mutation_performer.go +++ b/internal/api/resolver_mutation_performer.go @@ -12,6 +12,11 @@ import ( "github.com/stashapp/stash/pkg/utils" ) +const ( + twitterURL = "https://twitter.com" + instagramURL = "https://instagram.com" +) + // used to refetch performer after hooks run func (r *mutationResolver) getPerformer(ctx context.Context, id int) (ret *models.Performer, err error) { if err := r.withTxn(ctx, func(ctx context.Context) error { @@ -35,7 +40,6 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per newPerformer.Name = input.Name newPerformer.Disambiguation = translator.string(input.Disambiguation) newPerformer.Aliases = models.NewRelatedStrings(input.AliasList) - newPerformer.URL = translator.string(input.URL) newPerformer.Gender = input.Gender newPerformer.Ethnicity = translator.string(input.Ethnicity) newPerformer.Country = translator.string(input.Country) @@ -47,8 +51,6 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per newPerformer.CareerLength = translator.string(input.CareerLength) newPerformer.Tattoos = translator.string(input.Tattoos) newPerformer.Piercings = translator.string(input.Piercings) - newPerformer.Twitter = translator.string(input.Twitter) - newPerformer.Instagram = translator.string(input.Instagram) newPerformer.Favorite = translator.bool(input.Favorite) newPerformer.Rating = input.Rating100 newPerformer.Details = translator.string(input.Details) @@ -58,6 +60,21 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per newPerformer.IgnoreAutoTag = translator.bool(input.IgnoreAutoTag) newPerformer.StashIDs = models.NewRelatedStashIDs(input.StashIds) + newPerformer.URLs = models.NewRelatedStrings([]string{}) + if input.URL != nil { + newPerformer.URLs.Add(*input.URL) + } + if input.Twitter != nil { + newPerformer.URLs.Add(utils.URLFromHandle(*input.Twitter, twitterURL)) + } + if input.Instagram != nil { + newPerformer.URLs.Add(utils.URLFromHandle(*input.Instagram, instagramURL)) + } + + if input.Urls != nil { + newPerformer.URLs.Add(input.Urls...) + } + var err error newPerformer.Birthdate, err = translator.datePtr(input.Birthdate) @@ -112,6 +129,96 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per return r.getPerformer(ctx, newPerformer.ID) } +func (r *mutationResolver) validateNoLegacyURLs(translator changesetTranslator) error { + // ensure url/twitter/instagram are not included in the input + if translator.hasField("url") { + return fmt.Errorf("url field must not be included if urls is included") + } + if translator.hasField("twitter") { + return fmt.Errorf("twitter field must not be included if urls is included") + } + if translator.hasField("instagram") { + return fmt.Errorf("instagram field must not be included if urls is included") + } + + return nil +} + +func (r *mutationResolver) handleLegacyURLs(ctx context.Context, performerID int, legacyURL, legacyTwitter, legacyInstagram models.OptionalString, updatedPerformer *models.PerformerPartial) error { + qb := r.repository.Performer + + // we need to be careful with URL/Twitter/Instagram + // treat URL as replacing the first non-Twitter/Instagram URL in the list + // twitter should replace any existing twitter URL + // instagram should replace any existing instagram URL + p, err := qb.Find(ctx, performerID) + if err != nil { + return err + } + + if err := p.LoadURLs(ctx, qb); err != nil { + return fmt.Errorf("loading performer URLs: %w", err) + } + + existingURLs := p.URLs.List() + + // performer partial URLs should be empty + if legacyURL.Set { + replaced := false + for i, url := range existingURLs { + if !performer.IsTwitterURL(url) && !performer.IsInstagramURL(url) { + existingURLs[i] = legacyURL.Value + replaced = true + break + } + } + + if !replaced { + existingURLs = append(existingURLs, legacyURL.Value) + } + } + + if legacyTwitter.Set { + value := utils.URLFromHandle(legacyTwitter.Value, twitterURL) + found := false + // find and replace the first twitter URL + for i, url := range existingURLs { + if performer.IsTwitterURL(url) { + existingURLs[i] = value + found = true + break + } + } + + if !found { + existingURLs = append(existingURLs, value) + } + } + if legacyInstagram.Set { + found := false + value := utils.URLFromHandle(legacyInstagram.Value, instagramURL) + // find and replace the first instagram URL + for i, url := range existingURLs { + if performer.IsInstagramURL(url) { + existingURLs[i] = value + found = true + break + } + } + + if !found { + existingURLs = append(existingURLs, value) + } + } + + updatedPerformer.URLs = &models.UpdateStrings{ + Values: existingURLs, + Mode: models.RelationshipUpdateModeSet, + } + + return nil +} + func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.PerformerUpdateInput) (*models.Performer, error) { performerID, err := strconv.Atoi(input.ID) if err != nil { @@ -127,7 +234,6 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per updatedPerformer.Name = translator.optionalString(input.Name, "name") updatedPerformer.Disambiguation = translator.optionalString(input.Disambiguation, "disambiguation") - updatedPerformer.URL = translator.optionalString(input.URL, "url") updatedPerformer.Gender = translator.optionalString((*string)(input.Gender), "gender") updatedPerformer.Ethnicity = translator.optionalString(input.Ethnicity, "ethnicity") updatedPerformer.Country = translator.optionalString(input.Country, "country") @@ -139,8 +245,6 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per updatedPerformer.CareerLength = translator.optionalString(input.CareerLength, "career_length") updatedPerformer.Tattoos = translator.optionalString(input.Tattoos, "tattoos") updatedPerformer.Piercings = translator.optionalString(input.Piercings, "piercings") - updatedPerformer.Twitter = translator.optionalString(input.Twitter, "twitter") - updatedPerformer.Instagram = translator.optionalString(input.Instagram, "instagram") updatedPerformer.Favorite = translator.optionalBool(input.Favorite, "favorite") updatedPerformer.Rating = translator.optionalInt(input.Rating100, "rating100") updatedPerformer.Details = translator.optionalString(input.Details, "details") @@ -149,6 +253,19 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per updatedPerformer.IgnoreAutoTag = translator.optionalBool(input.IgnoreAutoTag, "ignore_auto_tag") updatedPerformer.StashIDs = translator.updateStashIDs(input.StashIds, "stash_ids") + if translator.hasField("urls") { + // ensure url/twitter/instagram are not included in the input + if err := r.validateNoLegacyURLs(translator); err != nil { + return nil, err + } + + updatedPerformer.URLs = translator.updateStrings(input.Urls, "urls") + } + + legacyURL := translator.optionalString(input.URL, "url") + legacyTwitter := translator.optionalString(input.Twitter, "twitter") + legacyInstagram := translator.optionalString(input.Instagram, "instagram") + updatedPerformer.Birthdate, err = translator.optionalDate(input.Birthdate, "birthdate") if err != nil { return nil, fmt.Errorf("converting birthdate: %w", err) @@ -186,6 +303,12 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per if err := r.withTxn(ctx, func(ctx context.Context) error { qb := r.repository.Performer + if legacyURL.Set || legacyTwitter.Set || legacyInstagram.Set { + if err := r.handleLegacyURLs(ctx, performerID, legacyURL, legacyTwitter, legacyInstagram, &updatedPerformer); err != nil { + return err + } + } + if err := performer.ValidateUpdate(ctx, performerID, updatedPerformer, qb); err != nil { return err } @@ -225,7 +348,7 @@ func (r *mutationResolver) BulkPerformerUpdate(ctx context.Context, input BulkPe updatedPerformer := models.NewPerformerPartial() updatedPerformer.Disambiguation = translator.optionalString(input.Disambiguation, "disambiguation") - updatedPerformer.URL = translator.optionalString(input.URL, "url") + updatedPerformer.Gender = translator.optionalString((*string)(input.Gender), "gender") updatedPerformer.Ethnicity = translator.optionalString(input.Ethnicity, "ethnicity") updatedPerformer.Country = translator.optionalString(input.Country, "country") @@ -237,8 +360,7 @@ func (r *mutationResolver) BulkPerformerUpdate(ctx context.Context, input BulkPe updatedPerformer.CareerLength = translator.optionalString(input.CareerLength, "career_length") updatedPerformer.Tattoos = translator.optionalString(input.Tattoos, "tattoos") updatedPerformer.Piercings = translator.optionalString(input.Piercings, "piercings") - updatedPerformer.Twitter = translator.optionalString(input.Twitter, "twitter") - updatedPerformer.Instagram = translator.optionalString(input.Instagram, "instagram") + updatedPerformer.Favorite = translator.optionalBool(input.Favorite, "favorite") updatedPerformer.Rating = translator.optionalInt(input.Rating100, "rating100") updatedPerformer.Details = translator.optionalString(input.Details, "details") @@ -246,6 +368,19 @@ func (r *mutationResolver) BulkPerformerUpdate(ctx context.Context, input BulkPe updatedPerformer.Weight = translator.optionalInt(input.Weight, "weight") updatedPerformer.IgnoreAutoTag = translator.optionalBool(input.IgnoreAutoTag, "ignore_auto_tag") + if translator.hasField("urls") { + // ensure url/twitter/instagram are not included in the input + if err := r.validateNoLegacyURLs(translator); err != nil { + return nil, err + } + + updatedPerformer.URLs = translator.updateStringsBulk(input.Urls, "urls") + } + + legacyURL := translator.optionalString(input.URL, "url") + legacyTwitter := translator.optionalString(input.Twitter, "twitter") + legacyInstagram := translator.optionalString(input.Instagram, "instagram") + updatedPerformer.Birthdate, err = translator.optionalDate(input.Birthdate, "birthdate") if err != nil { return nil, fmt.Errorf("converting birthdate: %w", err) @@ -277,6 +412,12 @@ func (r *mutationResolver) BulkPerformerUpdate(ctx context.Context, input BulkPe qb := r.repository.Performer for _, performerID := range performerIDs { + if legacyURL.Set || legacyTwitter.Set || legacyInstagram.Set { + if err := r.handleLegacyURLs(ctx, performerID, legacyURL, legacyTwitter, legacyInstagram, &updatedPerformer); err != nil { + return err + } + } + if err := performer.ValidateUpdate(ctx, performerID, updatedPerformer, qb); err != nil { return err } diff --git a/internal/manager/task/migrate.go b/internal/manager/task/migrate.go index 48ba15a26..37062329e 100644 --- a/internal/manager/task/migrate.go +++ b/internal/manager/task/migrate.go @@ -23,19 +23,27 @@ type MigrateJob struct { Database *sqlite.Database } +type databaseSchemaInfo struct { + CurrentSchemaVersion uint + RequiredSchemaVersion uint + StepsRequired uint +} + func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error { - required, err := s.required() + schemaInfo, err := s.required() if err != nil { return err } - if required == 0 { + if schemaInfo.StepsRequired == 0 { logger.Infof("database is already at the latest schema version") return nil } + logger.Infof("Migrating database from %d to %d", schemaInfo.CurrentSchemaVersion, schemaInfo.RequiredSchemaVersion) + // set the number of tasks = required steps + optimise - progress.SetTotal(int(required + 1)) + progress.SetTotal(int(schemaInfo.StepsRequired + 1)) database := s.Database @@ -79,28 +87,31 @@ func (s *MigrateJob) Execute(ctx context.Context, progress *job.Progress) error } } + logger.Infof("Database migration complete") + return nil } -func (s *MigrateJob) required() (uint, error) { +func (s *MigrateJob) required() (ret databaseSchemaInfo, err error) { database := s.Database m, err := sqlite.NewMigrator(database) if err != nil { - return 0, err + return } defer m.Close() - currentSchemaVersion := m.CurrentSchemaVersion() - targetSchemaVersion := m.RequiredSchemaVersion() + ret.CurrentSchemaVersion = m.CurrentSchemaVersion() + ret.RequiredSchemaVersion = m.RequiredSchemaVersion() - if targetSchemaVersion < currentSchemaVersion { + if ret.RequiredSchemaVersion < ret.CurrentSchemaVersion { // shouldn't happen - return 0, nil + return } - return targetSchemaVersion - currentSchemaVersion, nil + ret.StepsRequired = ret.RequiredSchemaVersion - ret.CurrentSchemaVersion + return } func (s *MigrateJob) runMigrations(ctx context.Context, progress *job.Progress) error { diff --git a/pkg/models/jsonschema/performer.go b/pkg/models/jsonschema/performer.go index 248cf9557..7ffa69983 100644 --- a/pkg/models/jsonschema/performer.go +++ b/pkg/models/jsonschema/performer.go @@ -34,16 +34,14 @@ func (s *StringOrStringList) UnmarshalJSON(data []byte) error { } type Performer struct { - Name string `json:"name,omitempty"` - Disambiguation string `json:"disambiguation,omitempty"` - Gender string `json:"gender,omitempty"` - URL string `json:"url,omitempty"` - Twitter string `json:"twitter,omitempty"` - Instagram string `json:"instagram,omitempty"` - Birthdate string `json:"birthdate,omitempty"` - Ethnicity string `json:"ethnicity,omitempty"` - Country string `json:"country,omitempty"` - EyeColor string `json:"eye_color,omitempty"` + Name string `json:"name,omitempty"` + Disambiguation string `json:"disambiguation,omitempty"` + Gender string `json:"gender,omitempty"` + URLs []string `json:"urls,omitempty"` + Birthdate string `json:"birthdate,omitempty"` + Ethnicity string `json:"ethnicity,omitempty"` + Country string `json:"country,omitempty"` + EyeColor string `json:"eye_color,omitempty"` // this should be int, but keeping string for backwards compatibility Height string `json:"height,omitempty"` Measurements string `json:"measurements,omitempty"` @@ -66,6 +64,11 @@ type Performer struct { Weight int `json:"weight,omitempty"` StashIDs []models.StashID `json:"stash_ids,omitempty"` IgnoreAutoTag bool `json:"ignore_auto_tag,omitempty"` + + // deprecated - for import only + URL string `json:"url,omitempty"` + Twitter string `json:"twitter,omitempty"` + Instagram string `json:"instagram,omitempty"` } func (s Performer) Filename() string { diff --git a/pkg/models/mocks/PerformerReaderWriter.go b/pkg/models/mocks/PerformerReaderWriter.go index 7bbc6ef79..0f3e2be02 100644 --- a/pkg/models/mocks/PerformerReaderWriter.go +++ b/pkg/models/mocks/PerformerReaderWriter.go @@ -383,6 +383,29 @@ func (_m *PerformerReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ( return r0, r1 } +// GetURLs provides a mock function with given fields: ctx, relatedID +func (_m *PerformerReaderWriter) GetURLs(ctx context.Context, relatedID int) ([]string, error) { + ret := _m.Called(ctx, relatedID) + + var r0 []string + if rf, ok := ret.Get(0).(func(context.Context, int) []string); ok { + r0 = rf(ctx, relatedID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int) error); ok { + r1 = rf(ctx, relatedID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // HasImage provides a mock function with given fields: ctx, performerID func (_m *PerformerReaderWriter) HasImage(ctx context.Context, performerID int) (bool, error) { ret := _m.Called(ctx, performerID) diff --git a/pkg/models/model_performer.go b/pkg/models/model_performer.go index 09f92e13c..85257ba38 100644 --- a/pkg/models/model_performer.go +++ b/pkg/models/model_performer.go @@ -10,9 +10,6 @@ type Performer struct { Name string `json:"name"` Disambiguation string `json:"disambiguation"` Gender *GenderEnum `json:"gender"` - URL string `json:"url"` - Twitter string `json:"twitter"` - Instagram string `json:"instagram"` Birthdate *Date `json:"birthdate"` Ethnicity string `json:"ethnicity"` Country string `json:"country"` @@ -37,6 +34,7 @@ type Performer struct { IgnoreAutoTag bool `json:"ignore_auto_tag"` Aliases RelatedStrings `json:"aliases"` + URLs RelatedStrings `json:"urls"` TagIDs RelatedIDs `json:"tag_ids"` StashIDs RelatedStashIDs `json:"stash_ids"` } @@ -55,9 +53,7 @@ type PerformerPartial struct { Name OptionalString Disambiguation OptionalString Gender OptionalString - URL OptionalString - Twitter OptionalString - Instagram OptionalString + URLs *UpdateStrings Birthdate OptionalDate Ethnicity OptionalString Country OptionalString @@ -99,6 +95,12 @@ func (s *Performer) LoadAliases(ctx context.Context, l AliasLoader) error { }) } +func (s *Performer) LoadURLs(ctx context.Context, l URLLoader) error { + return s.URLs.load(func() ([]string, error) { + return l.GetURLs(ctx, s.ID) + }) +} + func (s *Performer) LoadTagIDs(ctx context.Context, l TagIDLoader) error { return s.TagIDs.load(func() ([]int, error) { return l.GetTagIDs(ctx, s.ID) diff --git a/pkg/models/model_scraped_item.go b/pkg/models/model_scraped_item.go index 5cc5c679c..206f1109b 100644 --- a/pkg/models/model_scraped_item.go +++ b/pkg/models/model_scraped_item.go @@ -107,9 +107,10 @@ type ScrapedPerformer struct { Name *string `json:"name"` Disambiguation *string `json:"disambiguation"` Gender *string `json:"gender"` - URL *string `json:"url"` - Twitter *string `json:"twitter"` - Instagram *string `json:"instagram"` + URLs []string `json:"urls"` + URL *string `json:"url"` // deprecated + Twitter *string `json:"twitter"` // deprecated + Instagram *string `json:"instagram"` // deprecated Birthdate *string `json:"birthdate"` Ethnicity *string `json:"ethnicity"` Country *string `json:"country"` @@ -191,9 +192,7 @@ func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool ret.Weight = &w } } - if p.Instagram != nil && !excluded["instagram"] { - ret.Instagram = *p.Instagram - } + if p.Measurements != nil && !excluded["measurements"] { ret.Measurements = *p.Measurements } @@ -221,11 +220,27 @@ func (p *ScrapedPerformer) ToPerformer(endpoint string, excluded map[string]bool ret.Circumcised = &v } } - if p.Twitter != nil && !excluded["twitter"] { - ret.Twitter = *p.Twitter - } - if p.URL != nil && !excluded["url"] { - ret.URL = *p.URL + + // if URLs are provided, only use those + if len(p.URLs) > 0 { + if !excluded["urls"] { + ret.URLs = NewRelatedStrings(p.URLs) + } + } else { + urls := []string{} + if p.URL != nil && !excluded["url"] { + urls = append(urls, *p.URL) + } + if p.Twitter != nil && !excluded["twitter"] { + urls = append(urls, *p.Twitter) + } + if p.Instagram != nil && !excluded["instagram"] { + urls = append(urls, *p.Instagram) + } + + if len(urls) > 0 { + ret.URLs = NewRelatedStrings(urls) + } } if p.RemoteSiteID != nil && endpoint != "" { @@ -309,9 +324,6 @@ func (p *ScrapedPerformer) ToPartial(endpoint string, excluded map[string]bool, ret.Weight = NewOptionalInt(w) } } - if p.Instagram != nil && !excluded["instagram"] { - ret.Instagram = NewOptionalString(*p.Instagram) - } if p.Measurements != nil && !excluded["measurements"] { ret.Measurements = NewOptionalString(*p.Measurements) } @@ -330,11 +342,33 @@ func (p *ScrapedPerformer) ToPartial(endpoint string, excluded map[string]bool, if p.Tattoos != nil && !excluded["tattoos"] { ret.Tattoos = NewOptionalString(*p.Tattoos) } - if p.Twitter != nil && !excluded["twitter"] { - ret.Twitter = NewOptionalString(*p.Twitter) - } - if p.URL != nil && !excluded["url"] { - ret.URL = NewOptionalString(*p.URL) + + // if URLs are provided, only use those + if len(p.URLs) > 0 { + if !excluded["urls"] { + ret.URLs = &UpdateStrings{ + Values: p.URLs, + Mode: RelationshipUpdateModeSet, + } + } + } else { + urls := []string{} + if p.URL != nil && !excluded["url"] { + urls = append(urls, *p.URL) + } + if p.Twitter != nil && !excluded["twitter"] { + urls = append(urls, *p.Twitter) + } + if p.Instagram != nil && !excluded["instagram"] { + urls = append(urls, *p.Instagram) + } + + if len(urls) > 0 { + ret.URLs = &UpdateStrings{ + Values: urls, + Mode: RelationshipUpdateModeSet, + } + } } if p.RemoteSiteID != nil && endpoint != "" { diff --git a/pkg/models/model_scraped_item_test.go b/pkg/models/model_scraped_item_test.go index a6e42f2fd..50657188d 100644 --- a/pkg/models/model_scraped_item_test.go +++ b/pkg/models/model_scraped_item_test.go @@ -161,9 +161,9 @@ func Test_scrapedToPerformerInput(t *testing.T) { Tattoos: nextVal(), Piercings: nextVal(), Aliases: nextVal(), + URL: nextVal(), Twitter: nextVal(), Instagram: nextVal(), - URL: nextVal(), Details: nextVal(), RemoteSiteID: &remoteSiteID, }, @@ -186,9 +186,7 @@ func Test_scrapedToPerformerInput(t *testing.T) { Tattoos: *nextVal(), Piercings: *nextVal(), Aliases: NewRelatedStrings([]string{*nextVal()}), - Twitter: *nextVal(), - Instagram: *nextVal(), - URL: *nextVal(), + URLs: NewRelatedStrings([]string{*nextVal(), *nextVal(), *nextVal()}), Details: *nextVal(), StashIDs: NewRelatedStashIDs([]StashID{ { diff --git a/pkg/models/performer.go b/pkg/models/performer.go index 75b0f85af..b14f60044 100644 --- a/pkg/models/performer.go +++ b/pkg/models/performer.go @@ -203,7 +203,8 @@ type PerformerFilterType struct { type PerformerCreateInput struct { Name string `json:"name"` Disambiguation *string `json:"disambiguation"` - URL *string `json:"url"` + URL *string `json:"url"` // deprecated + Urls []string `json:"urls"` Gender *GenderEnum `json:"gender"` Birthdate *string `json:"birthdate"` Ethnicity *string `json:"ethnicity"` @@ -220,8 +221,8 @@ type PerformerCreateInput struct { Piercings *string `json:"piercings"` Aliases *string `json:"aliases"` AliasList []string `json:"alias_list"` - Twitter *string `json:"twitter"` - Instagram *string `json:"instagram"` + Twitter *string `json:"twitter"` // deprecated + Instagram *string `json:"instagram"` // deprecated Favorite *bool `json:"favorite"` TagIds []string `json:"tag_ids"` // This should be a URL or a base64 encoded data URL @@ -239,7 +240,8 @@ type PerformerUpdateInput struct { ID string `json:"id"` Name *string `json:"name"` Disambiguation *string `json:"disambiguation"` - URL *string `json:"url"` + URL *string `json:"url"` // deprecated + Urls []string `json:"urls"` Gender *GenderEnum `json:"gender"` Birthdate *string `json:"birthdate"` Ethnicity *string `json:"ethnicity"` @@ -256,8 +258,8 @@ type PerformerUpdateInput struct { Piercings *string `json:"piercings"` Aliases *string `json:"aliases"` AliasList []string `json:"alias_list"` - Twitter *string `json:"twitter"` - Instagram *string `json:"instagram"` + Twitter *string `json:"twitter"` // deprecated + Instagram *string `json:"instagram"` // deprecated Favorite *bool `json:"favorite"` TagIds []string `json:"tag_ids"` // This should be a URL or a base64 encoded data URL diff --git a/pkg/models/repository_performer.go b/pkg/models/repository_performer.go index 22ade1d1d..3fd936190 100644 --- a/pkg/models/repository_performer.go +++ b/pkg/models/repository_performer.go @@ -78,6 +78,7 @@ type PerformerReader interface { AliasLoader StashIDLoader TagIDLoader + URLLoader All(ctx context.Context) ([]*Performer, error) GetImage(ctx context.Context, performerID int) ([]byte, error) diff --git a/pkg/performer/export.go b/pkg/performer/export.go index 9aec8b34e..8f720338f 100644 --- a/pkg/performer/export.go +++ b/pkg/performer/export.go @@ -16,6 +16,7 @@ type ImageAliasStashIDGetter interface { GetImage(ctx context.Context, performerID int) ([]byte, error) models.AliasLoader models.StashIDLoader + models.URLLoader } // ToJSON converts a Performer object into its JSON equivalent. @@ -23,7 +24,6 @@ func ToJSON(ctx context.Context, reader ImageAliasStashIDGetter, performer *mode newPerformerJSON := jsonschema.Performer{ Name: performer.Name, Disambiguation: performer.Disambiguation, - URL: performer.URL, Ethnicity: performer.Ethnicity, Country: performer.Country, EyeColor: performer.EyeColor, @@ -32,8 +32,6 @@ func ToJSON(ctx context.Context, reader ImageAliasStashIDGetter, performer *mode CareerLength: performer.CareerLength, Tattoos: performer.Tattoos, Piercings: performer.Piercings, - Twitter: performer.Twitter, - Instagram: performer.Instagram, Favorite: performer.Favorite, Details: performer.Details, HairColor: performer.HairColor, @@ -78,6 +76,11 @@ func ToJSON(ctx context.Context, reader ImageAliasStashIDGetter, performer *mode newPerformerJSON.Aliases = performer.Aliases.List() + if err := performer.LoadURLs(ctx, reader); err != nil { + return nil, fmt.Errorf("loading performer urls: %w", err) + } + newPerformerJSON.URLs = performer.URLs.List() + if err := performer.LoadStashIDs(ctx, reader); err != nil { return nil, fmt.Errorf("loading performer stash ids: %w", err) } diff --git a/pkg/performer/export_test.go b/pkg/performer/export_test.go index 572634aa6..36353b17d 100644 --- a/pkg/performer/export_test.go +++ b/pkg/performer/export_test.go @@ -77,7 +77,7 @@ func createFullPerformer(id int, name string) *models.Performer { ID: id, Name: name, Disambiguation: disambiguation, - URL: url, + URLs: models.NewRelatedStrings([]string{url, twitter, instagram}), Aliases: models.NewRelatedStrings(aliases), Birthdate: &birthDate, CareerLength: careerLength, @@ -90,11 +90,9 @@ func createFullPerformer(id int, name string) *models.Performer { Favorite: true, Gender: &genderEnum, Height: &height, - Instagram: instagram, Measurements: measurements, Piercings: piercings, Tattoos: tattoos, - Twitter: twitter, CreatedAt: createTime, UpdatedAt: updateTime, Rating: &rating, @@ -114,6 +112,7 @@ func createEmptyPerformer(id int) models.Performer { CreatedAt: createTime, UpdatedAt: updateTime, Aliases: models.NewRelatedStrings([]string{}), + URLs: models.NewRelatedStrings([]string{}), TagIDs: models.NewRelatedIDs([]int{}), StashIDs: models.NewRelatedStashIDs([]models.StashID{}), } @@ -123,7 +122,7 @@ func createFullJSONPerformer(name string, image string) *jsonschema.Performer { return &jsonschema.Performer{ Name: name, Disambiguation: disambiguation, - URL: url, + URLs: []string{url, twitter, instagram}, Aliases: aliases, Birthdate: birthDate.String(), CareerLength: careerLength, @@ -136,11 +135,9 @@ func createFullJSONPerformer(name string, image string) *jsonschema.Performer { Favorite: true, Gender: gender, Height: strconv.Itoa(height), - Instagram: instagram, Measurements: measurements, Piercings: piercings, Tattoos: tattoos, - Twitter: twitter, CreatedAt: json.JSONTime{ Time: createTime, }, @@ -161,6 +158,7 @@ func createFullJSONPerformer(name string, image string) *jsonschema.Performer { func createEmptyJSONPerformer() *jsonschema.Performer { return &jsonschema.Performer{ Aliases: []string{}, + URLs: []string{}, StashIDs: []models.StashID{}, CreatedAt: json.JSONTime{ Time: createTime, diff --git a/pkg/performer/import.go b/pkg/performer/import.go index afa6cd4bc..d50384fa3 100644 --- a/pkg/performer/import.go +++ b/pkg/performer/import.go @@ -188,7 +188,6 @@ func performerJSONToPerformer(performerJSON jsonschema.Performer) models.Perform newPerformer := models.Performer{ Name: performerJSON.Name, Disambiguation: performerJSON.Disambiguation, - URL: performerJSON.URL, Ethnicity: performerJSON.Ethnicity, Country: performerJSON.Country, EyeColor: performerJSON.EyeColor, @@ -198,8 +197,6 @@ func performerJSONToPerformer(performerJSON jsonschema.Performer) models.Perform Tattoos: performerJSON.Tattoos, Piercings: performerJSON.Piercings, Aliases: models.NewRelatedStrings(performerJSON.Aliases), - Twitter: performerJSON.Twitter, - Instagram: performerJSON.Instagram, Details: performerJSON.Details, HairColor: performerJSON.HairColor, Favorite: performerJSON.Favorite, @@ -211,6 +208,25 @@ func performerJSONToPerformer(performerJSON jsonschema.Performer) models.Perform StashIDs: models.NewRelatedStashIDs(performerJSON.StashIDs), } + if len(performerJSON.URLs) > 0 { + newPerformer.URLs = models.NewRelatedStrings(performerJSON.URLs) + } else { + urls := []string{} + if performerJSON.URL != "" { + urls = append(urls, performerJSON.URL) + } + if performerJSON.Twitter != "" { + urls = append(urls, performerJSON.Twitter) + } + if performerJSON.Instagram != "" { + urls = append(urls, performerJSON.Instagram) + } + + if len(urls) > 0 { + newPerformer.URLs = models.NewRelatedStrings([]string{performerJSON.URL}) + } + } + if performerJSON.Gender != "" { v := models.GenderEnum(performerJSON.Gender) newPerformer.Gender = &v diff --git a/pkg/performer/url.go b/pkg/performer/url.go new file mode 100644 index 000000000..4b52adad5 --- /dev/null +++ b/pkg/performer/url.go @@ -0,0 +1,18 @@ +package performer + +import ( + "regexp" +) + +var ( + twitterURLRE = regexp.MustCompile(`^https?:\/\/(?:www\.)?twitter\.com\/`) + instagramURLRE = regexp.MustCompile(`^https?:\/\/(?:www\.)?instagram\.com\/`) +) + +func IsTwitterURL(url string) bool { + return twitterURLRE.MatchString(url) +} + +func IsInstagramURL(url string) bool { + return instagramURLRE.MatchString(url) +} diff --git a/pkg/scraper/json.go b/pkg/scraper/json.go index 98e853785..ae96ecb06 100644 --- a/pkg/scraper/json.go +++ b/pkg/scraper/json.go @@ -81,15 +81,33 @@ func (s *jsonScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCont } q := s.getJsonQuery(doc) + // if these just return the return values from scraper.scrape* functions then + // it ends up returning ScrapedContent(nil) rather than nil switch ty { case ScrapeContentTypePerformer: - return scraper.scrapePerformer(ctx, q) + ret, err := scraper.scrapePerformer(ctx, q) + if err != nil || ret == nil { + return nil, err + } + return ret, nil case ScrapeContentTypeScene: - return scraper.scrapeScene(ctx, q) + ret, err := scraper.scrapeScene(ctx, q) + if err != nil || ret == nil { + return nil, err + } + return ret, nil case ScrapeContentTypeGallery: - return scraper.scrapeGallery(ctx, q) + ret, err := scraper.scrapeGallery(ctx, q) + if err != nil || ret == nil { + return nil, err + } + return ret, nil case ScrapeContentTypeMovie: - return scraper.scrapeMovie(ctx, q) + ret, err := scraper.scrapeMovie(ctx, q) + if err != nil || ret == nil { + return nil, err + } + return ret, nil } return nil, ErrNotSupported diff --git a/pkg/scraper/performer.go b/pkg/scraper/performer.go index 269368823..98e931762 100644 --- a/pkg/scraper/performer.go +++ b/pkg/scraper/performer.go @@ -2,29 +2,30 @@ package scraper type ScrapedPerformerInput struct { // Set if performer matched - StoredID *string `json:"stored_id"` - Name *string `json:"name"` - Disambiguation *string `json:"disambiguation"` - Gender *string `json:"gender"` - URL *string `json:"url"` - Twitter *string `json:"twitter"` - Instagram *string `json:"instagram"` - Birthdate *string `json:"birthdate"` - Ethnicity *string `json:"ethnicity"` - Country *string `json:"country"` - EyeColor *string `json:"eye_color"` - Height *string `json:"height"` - Measurements *string `json:"measurements"` - FakeTits *string `json:"fake_tits"` - PenisLength *string `json:"penis_length"` - Circumcised *string `json:"circumcised"` - CareerLength *string `json:"career_length"` - Tattoos *string `json:"tattoos"` - Piercings *string `json:"piercings"` - Aliases *string `json:"aliases"` - Details *string `json:"details"` - DeathDate *string `json:"death_date"` - HairColor *string `json:"hair_color"` - Weight *string `json:"weight"` - RemoteSiteID *string `json:"remote_site_id"` + StoredID *string `json:"stored_id"` + Name *string `json:"name"` + Disambiguation *string `json:"disambiguation"` + Gender *string `json:"gender"` + URLs []string `json:"urls"` + URL *string `json:"url"` // deprecated + Twitter *string `json:"twitter"` // deprecated + Instagram *string `json:"instagram"` // deprecated + Birthdate *string `json:"birthdate"` + Ethnicity *string `json:"ethnicity"` + Country *string `json:"country"` + EyeColor *string `json:"eye_color"` + Height *string `json:"height"` + Measurements *string `json:"measurements"` + FakeTits *string `json:"fake_tits"` + PenisLength *string `json:"penis_length"` + Circumcised *string `json:"circumcised"` + CareerLength *string `json:"career_length"` + Tattoos *string `json:"tattoos"` + Piercings *string `json:"piercings"` + Aliases *string `json:"aliases"` + Details *string `json:"details"` + DeathDate *string `json:"death_date"` + HairColor *string `json:"hair_color"` + Weight *string `json:"weight"` + RemoteSiteID *string `json:"remote_site_id"` } diff --git a/pkg/scraper/postprocessing.go b/pkg/scraper/postprocessing.go index a375b5058..e153c5616 100644 --- a/pkg/scraper/postprocessing.go +++ b/pkg/scraper/postprocessing.go @@ -6,6 +6,7 @@ import ( "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/match" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" ) // postScrape handles post-processing of scraped content. If the content @@ -67,6 +68,31 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme p.Country = resolveCountryName(p.Country) + // populate URL/URLs + // if URLs are provided, only use those + if len(p.URLs) > 0 { + p.URL = &p.URLs[0] + } else { + urls := []string{} + if p.URL != nil { + urls = append(urls, *p.URL) + } + if p.Twitter != nil && *p.Twitter != "" { + // handle twitter profile names + u := utils.URLFromHandle(*p.Twitter, "https://twitter.com") + urls = append(urls, u) + } + if p.Instagram != nil && *p.Instagram != "" { + // handle instagram profile names + u := utils.URLFromHandle(*p.Instagram, "https://instagram.com") + urls = append(urls, u) + } + + if len(urls) > 0 { + p.URLs = urls + } + } + return p, nil } diff --git a/pkg/scraper/scraper.go b/pkg/scraper/scraper.go index 23ad411bd..4eb67dcf4 100644 --- a/pkg/scraper/scraper.go +++ b/pkg/scraper/scraper.go @@ -163,6 +163,12 @@ func (i *Input) populateURL() { if i.Scene != nil && i.Scene.URL == nil && len(i.Scene.URLs) > 0 { i.Scene.URL = &i.Scene.URLs[0] } + if i.Gallery != nil && i.Gallery.URL == nil && len(i.Gallery.URLs) > 0 { + i.Gallery.URL = &i.Gallery.URLs[0] + } + if i.Performer != nil && i.Performer.URL == nil && len(i.Performer.URLs) > 0 { + i.Performer.URL = &i.Performer.URLs[0] + } } // simple type definitions that can help customize diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 407238dae..350bac5c4 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -9,7 +9,6 @@ import ( "io" "mime/multipart" "net/http" - "regexp" "strconv" "strings" @@ -41,6 +40,7 @@ type PerformerReader interface { match.PerformerFinder models.AliasLoader models.StashIDLoader + models.URLLoader FindBySceneID(ctx context.Context, sceneID int) ([]*models.Performer, error) GetImage(ctx context.Context, performerID int) ([]byte, error) } @@ -685,6 +685,10 @@ func performerFragmentToScrapedPerformer(p graphql.PerformerFragment) *models.Sc sp.Aliases = &alias } + for _, u := range p.Urls { + sp.URLs = append(sp.URLs, u.URL) + } + return sp } @@ -1128,6 +1132,10 @@ func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Perf return nil, err } + if err := performer.LoadURLs(ctx, pqb); err != nil { + return nil, err + } + if err := performer.LoadStashIDs(ctx, pqb); err != nil { return nil, err } @@ -1195,28 +1203,8 @@ func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Perf } } - var urls []string - if len(strings.TrimSpace(performer.Twitter)) > 0 { - reg := regexp.MustCompile(`https?:\/\/(?:www\.)?twitter\.com`) - if reg.MatchString(performer.Twitter) { - urls = append(urls, strings.TrimSpace(performer.Twitter)) - } else { - urls = append(urls, "https://twitter.com/"+strings.TrimSpace(performer.Twitter)) - } - } - if len(strings.TrimSpace(performer.Instagram)) > 0 { - reg := regexp.MustCompile(`https?:\/\/(?:www\.)?instagram\.com`) - if reg.MatchString(performer.Instagram) { - urls = append(urls, strings.TrimSpace(performer.Instagram)) - } else { - urls = append(urls, "https://instagram.com/"+strings.TrimSpace(performer.Instagram)) - } - } - if len(strings.TrimSpace(performer.URL)) > 0 { - urls = append(urls, strings.TrimSpace(performer.URL)) - } - if len(urls) > 0 { - draft.Urls = urls + if len(performer.URLs.List()) > 0 { + draft.Urls = performer.URLs.List() } stashIDs, err := pqb.GetStashIDs(ctx, performer.ID) diff --git a/pkg/scraper/xpath.go b/pkg/scraper/xpath.go index 29a4b0a19..d13c8e4c0 100644 --- a/pkg/scraper/xpath.go +++ b/pkg/scraper/xpath.go @@ -62,15 +62,33 @@ func (s *xpathScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCon } q := s.getXPathQuery(doc) + // if these just return the return values from scraper.scrape* functions then + // it ends up returning ScrapedContent(nil) rather than nil switch ty { case ScrapeContentTypePerformer: - return scraper.scrapePerformer(ctx, q) + ret, err := scraper.scrapePerformer(ctx, q) + if err != nil || ret == nil { + return nil, err + } + return ret, nil case ScrapeContentTypeScene: - return scraper.scrapeScene(ctx, q) + ret, err := scraper.scrapeScene(ctx, q) + if err != nil || ret == nil { + return nil, err + } + return ret, nil case ScrapeContentTypeGallery: - return scraper.scrapeGallery(ctx, q) + ret, err := scraper.scrapeGallery(ctx, q) + if err != nil || ret == nil { + return nil, err + } + return ret, nil case ScrapeContentTypeMovie: - return scraper.scrapeMovie(ctx, q) + ret, err := scraper.scrapeMovie(ctx, q) + if err != nil || ret == nil { + return nil, err + } + return ret, nil } return nil, ErrNotSupported diff --git a/pkg/sqlite/anonymise.go b/pkg/sqlite/anonymise.go index 44381c070..465e6cad5 100644 --- a/pkg/sqlite/anonymise.go +++ b/pkg/sqlite/anonymise.go @@ -495,9 +495,6 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error { table.Col(idColumn), table.Col("name"), table.Col("details"), - table.Col("url"), - table.Col("twitter"), - table.Col("instagram"), table.Col("tattoos"), table.Col("piercings"), ).Where(table.Col(idColumn).Gt(lastID)).Limit(1000) @@ -510,9 +507,6 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error { id int name sql.NullString details sql.NullString - url sql.NullString - twitter sql.NullString - instagram sql.NullString tattoos sql.NullString piercings sql.NullString ) @@ -521,9 +515,6 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error { &id, &name, &details, - &url, - &twitter, - &instagram, &tattoos, &piercings, ); err != nil { @@ -533,9 +524,6 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error { set := goqu.Record{} db.obfuscateNullString(set, "name", name) db.obfuscateNullString(set, "details", details) - db.obfuscateNullString(set, "url", url) - db.obfuscateNullString(set, "twitter", twitter) - db.obfuscateNullString(set, "instagram", instagram) db.obfuscateNullString(set, "tattoos", tattoos) db.obfuscateNullString(set, "piercings", piercings) @@ -566,6 +554,10 @@ func (db *Anonymiser) anonymisePerformers(ctx context.Context) error { return err } + if err := db.anonymiseURLs(ctx, goqu.T(performerURLsTable), "performer_id"); err != nil { + return err + } + return nil } diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index 7cfcd2003..cf502392f 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -30,7 +30,7 @@ const ( dbConnTimeout = 30 ) -var appSchemaVersion uint = 61 +var appSchemaVersion uint = 62 //go:embed migrations/*.sql var migrationsBox embed.FS diff --git a/pkg/sqlite/migrations/62_performer_urls.up.sql b/pkg/sqlite/migrations/62_performer_urls.up.sql new file mode 100644 index 000000000..cebfa86d6 --- /dev/null +++ b/pkg/sqlite/migrations/62_performer_urls.up.sql @@ -0,0 +1,155 @@ +PRAGMA foreign_keys=OFF; + +CREATE TABLE `performer_urls` ( + `performer_id` integer NOT NULL, + `position` integer NOT NULL, + `url` varchar(255) NOT NULL, + foreign key(`performer_id`) references `performers`(`id`) on delete CASCADE, + PRIMARY KEY(`performer_id`, `position`, `url`) +); + +CREATE INDEX `performers_urls_url` on `performer_urls` (`url`); + +-- drop url, twitter and instagram +-- make name not null +CREATE TABLE `performers_new` ( + `id` integer not null primary key autoincrement, + `name` varchar(255) not null, + `disambiguation` varchar(255), + `gender` varchar(20), + `birthdate` date, + `ethnicity` varchar(255), + `country` varchar(255), + `eye_color` varchar(255), + `height` int, + `measurements` varchar(255), + `fake_tits` varchar(255), + `career_length` varchar(255), + `tattoos` varchar(255), + `piercings` varchar(255), + `favorite` boolean not null default '0', + `created_at` datetime not null, + `updated_at` datetime not null, + `details` text, + `death_date` date, + `hair_color` varchar(255), + `weight` integer, + `rating` tinyint, + `ignore_auto_tag` boolean not null default '0', + `image_blob` varchar(255) REFERENCES `blobs`(`checksum`), + `penis_length` float, + `circumcised` varchar[10] +); + +INSERT INTO `performers_new` + ( + `id`, + `name`, + `disambiguation`, + `gender`, + `birthdate`, + `ethnicity`, + `country`, + `eye_color`, + `height`, + `measurements`, + `fake_tits`, + `career_length`, + `tattoos`, + `piercings`, + `favorite`, + `created_at`, + `updated_at`, + `details`, + `death_date`, + `hair_color`, + `weight`, + `rating`, + `ignore_auto_tag`, + `image_blob`, + `penis_length`, + `circumcised` + ) + SELECT + `id`, + `name`, + `disambiguation`, + `gender`, + `birthdate`, + `ethnicity`, + `country`, + `eye_color`, + `height`, + `measurements`, + `fake_tits`, + `career_length`, + `tattoos`, + `piercings`, + `favorite`, + `created_at`, + `updated_at`, + `details`, + `death_date`, + `hair_color`, + `weight`, + `rating`, + `ignore_auto_tag`, + `image_blob`, + `penis_length`, + `circumcised` + FROM `performers`; + +INSERT INTO `performer_urls` + ( + `performer_id`, + `position`, + `url` + ) + SELECT + `id`, + '0', + `url` + FROM `performers` + WHERE `performers`.`url` IS NOT NULL AND `performers`.`url` != ''; + +INSERT INTO `performer_urls` + ( + `performer_id`, + `position`, + `url` + ) + SELECT + `id`, + (SELECT count(*) FROM `performer_urls` WHERE `performer_id` = `performers`.`id`)+1, + CASE + WHEN `twitter` LIKE 'http%://%' THEN `twitter` + ELSE 'https://www.twitter.com/' || `twitter` + END + FROM `performers` + WHERE `performers`.`twitter` IS NOT NULL AND `performers`.`twitter` != ''; + +INSERT INTO `performer_urls` + ( + `performer_id`, + `position`, + `url` + ) + SELECT + `id`, + (SELECT count(*) FROM `performer_urls` WHERE `performer_id` = `performers`.`id`)+1, + CASE + WHEN `instagram` LIKE 'http%://%' THEN `instagram` + ELSE 'https://www.instagram.com/' || `instagram` + END + FROM `performers` + WHERE `performers`.`instagram` IS NOT NULL AND `performers`.`instagram` != ''; + +DROP INDEX `performers_name_disambiguation_unique`; +DROP INDEX `performers_name_unique`; +DROP TABLE `performers`; +ALTER TABLE `performers_new` rename to `performers`; + +CREATE UNIQUE INDEX `performers_name_disambiguation_unique` on `performers` (`name`, `disambiguation`) WHERE `disambiguation` IS NOT NULL; +CREATE UNIQUE INDEX `performers_name_unique` on `performers` (`name`) WHERE `disambiguation` IS NULL; + +PRAGMA foreign_keys=ON; diff --git a/pkg/sqlite/performer.go b/pkg/sqlite/performer.go index 4ba05168d..0c2f1d78f 100644 --- a/pkg/sqlite/performer.go +++ b/pkg/sqlite/performer.go @@ -23,6 +23,9 @@ const ( performerAliasColumn = "alias" performersTagsTable = "performers_tags" + performerURLsTable = "performer_urls" + performerURLColumn = "url" + performerImageBlobColumn = "image_blob" ) @@ -31,9 +34,6 @@ type performerRow struct { Name null.String `db:"name"` // TODO: make schema non-nullable Disambigation zero.String `db:"disambiguation"` Gender zero.String `db:"gender"` - URL zero.String `db:"url"` - Twitter zero.String `db:"twitter"` - Instagram zero.String `db:"instagram"` Birthdate NullDate `db:"birthdate"` Ethnicity zero.String `db:"ethnicity"` Country zero.String `db:"country"` @@ -68,9 +68,6 @@ func (r *performerRow) fromPerformer(o models.Performer) { if o.Gender != nil && o.Gender.IsValid() { r.Gender = zero.StringFrom(o.Gender.String()) } - r.URL = zero.StringFrom(o.URL) - r.Twitter = zero.StringFrom(o.Twitter) - r.Instagram = zero.StringFrom(o.Instagram) r.Birthdate = NullDateFromDatePtr(o.Birthdate) r.Ethnicity = zero.StringFrom(o.Ethnicity) r.Country = zero.StringFrom(o.Country) @@ -101,9 +98,6 @@ func (r *performerRow) resolve() *models.Performer { ID: r.ID, Name: r.Name.String, Disambiguation: r.Disambigation.String, - URL: r.URL.String, - Twitter: r.Twitter.String, - Instagram: r.Instagram.String, Birthdate: r.Birthdate.DatePtr(), Ethnicity: r.Ethnicity.String, Country: r.Country.String, @@ -148,9 +142,6 @@ func (r *performerRowRecord) fromPartial(o models.PerformerPartial) { r.setString("name", o.Name) r.setNullString("disambiguation", o.Disambiguation) r.setNullString("gender", o.Gender) - r.setNullString("url", o.URL) - r.setNullString("twitter", o.Twitter) - r.setNullString("instagram", o.Instagram) r.setNullDate("birthdate", o.Birthdate) r.setNullString("ethnicity", o.Ethnicity) r.setNullString("country", o.Country) @@ -272,6 +263,13 @@ func (qb *PerformerStore) Create(ctx context.Context, newObject *models.Performe } } + if newObject.URLs.Loaded() { + const startPos = 0 + if err := performersURLsTableMgr.insertJoins(ctx, id, startPos, newObject.URLs.List()); err != nil { + return err + } + } + if newObject.TagIDs.Loaded() { if err := performersTagsTableMgr.insertJoins(ctx, id, newObject.TagIDs.List()); err != nil { return err @@ -315,6 +313,12 @@ func (qb *PerformerStore) UpdatePartial(ctx context.Context, id int, partial mod } } + if partial.URLs != nil { + if err := performersURLsTableMgr.modifyJoins(ctx, id, partial.URLs.Values, partial.URLs.Mode); err != nil { + return nil, err + } + } + if partial.TagIDs != nil { if err := performersTagsTableMgr.modifyJoins(ctx, id, partial.TagIDs.IDs, partial.TagIDs.Mode); err != nil { return nil, err @@ -343,6 +347,12 @@ func (qb *PerformerStore) Update(ctx context.Context, updatedObject *models.Perf } } + if updatedObject.URLs.Loaded() { + if err := performersURLsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.URLs.List()); err != nil { + return err + } + } + if updatedObject.TagIDs.Loaded() { if err := performersTagsTableMgr.replaceJoins(ctx, updatedObject.ID, updatedObject.TagIDs.List()); err != nil { return err @@ -785,6 +795,10 @@ func (qb *PerformerStore) GetAliases(ctx context.Context, performerID int) ([]st return performersAliasesTableMgr.get(ctx, performerID) } +func (qb *PerformerStore) GetURLs(ctx context.Context, performerID int) ([]string, error) { + return performersURLsTableMgr.get(ctx, performerID) +} + func (qb *PerformerStore) GetStashIDs(ctx context.Context, performerID int) ([]models.StashID, error) { return performersStashIDsTableMgr.get(ctx, performerID) } diff --git a/pkg/sqlite/performer_filter.go b/pkg/sqlite/performer_filter.go index 13c2ec5a2..72990a7fe 100644 --- a/pkg/sqlite/performer_filter.go +++ b/pkg/sqlite/performer_filter.go @@ -134,7 +134,7 @@ func (qb *performerFilterHandler) criterionHandler() criterionHandler { stringCriterionHandler(filter.Piercings, tableName+".piercings"), intCriterionHandler(filter.Rating100, tableName+".rating", nil), stringCriterionHandler(filter.HairColor, tableName+".hair_color"), - stringCriterionHandler(filter.URL, tableName+".url"), + qb.urlsCriterionHandler(filter.URL), intCriterionHandler(filter.Weight, tableName+".weight", nil), criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) { if filter.StashID != nil { @@ -211,6 +211,9 @@ func (qb *performerFilterHandler) performerIsMissingCriterionHandler(isMissing * return func(ctx context.Context, f *filterBuilder) { if isMissing != nil && *isMissing != "" { switch *isMissing { + case "url": + performersURLsTableMgr.join(f, "", "performers.id") + f.addWhere("performer_urls.url IS NULL") case "scenes": // Deprecated: use `scene_count == 0` filter instead f.addLeftJoin(performersScenesTable, "scenes_join", "scenes_join.performer_id = performers.id") f.addWhere("scenes_join.scene_id IS NULL") @@ -241,6 +244,20 @@ func (qb *performerFilterHandler) performerAgeFilterCriterionHandler(age *models } } +func (qb *performerFilterHandler) urlsCriterionHandler(url *models.StringCriterionInput) criterionHandlerFunc { + h := stringListCriterionHandlerBuilder{ + primaryTable: performerTable, + primaryFK: performerIDColumn, + joinTable: performerURLsTable, + stringColumn: performerURLColumn, + addJoinTable: func(f *filterBuilder) { + performersURLsTableMgr.join(f, "", "performers.id") + }, + } + + return h.handler(url) +} + func (qb *performerFilterHandler) aliasCriterionHandler(alias *models.StringCriterionInput) criterionHandlerFunc { h := stringListCriterionHandlerBuilder{ primaryTable: performerTable, diff --git a/pkg/sqlite/performer_test.go b/pkg/sqlite/performer_test.go index d333913d2..c0124d09d 100644 --- a/pkg/sqlite/performer_test.go +++ b/pkg/sqlite/performer_test.go @@ -22,6 +22,11 @@ func loadPerformerRelationships(ctx context.Context, expected models.Performer, return err } } + if expected.URLs.Loaded() { + if err := actual.LoadURLs(ctx, db.Performer); err != nil { + return err + } + } if expected.TagIDs.Loaded() { if err := actual.LoadTagIDs(ctx, db.Performer); err != nil { return err @@ -45,6 +50,7 @@ func Test_PerformerStore_Create(t *testing.T) { url = "url" twitter = "twitter" instagram = "instagram" + urls = []string{url, twitter, instagram} rating = 3 ethnicity = "ethnicity" country = "country" @@ -84,9 +90,7 @@ func Test_PerformerStore_Create(t *testing.T) { Name: name, Disambiguation: disambiguation, Gender: &gender, - URL: url, - Twitter: twitter, - Instagram: instagram, + URLs: models.NewRelatedStrings(urls), Birthdate: &birthdate, Ethnicity: ethnicity, Country: country, @@ -193,6 +197,7 @@ func Test_PerformerStore_Update(t *testing.T) { url = "url" twitter = "twitter" instagram = "instagram" + urls = []string{url, twitter, instagram} rating = 3 ethnicity = "ethnicity" country = "country" @@ -233,9 +238,7 @@ func Test_PerformerStore_Update(t *testing.T) { Name: name, Disambiguation: disambiguation, Gender: &gender, - URL: url, - Twitter: twitter, - Instagram: instagram, + URLs: models.NewRelatedStrings(urls), Birthdate: &birthdate, Ethnicity: ethnicity, Country: country, @@ -277,6 +280,7 @@ func Test_PerformerStore_Update(t *testing.T) { &models.Performer{ ID: performerIDs[performerIdxWithGallery], Aliases: models.NewRelatedStrings([]string{}), + URLs: models.NewRelatedStrings([]string{}), TagIDs: models.NewRelatedIDs([]int{}), StashIDs: models.NewRelatedStashIDs([]models.StashID{}), }, @@ -341,9 +345,7 @@ func clearPerformerPartial() models.PerformerPartial { return models.PerformerPartial{ Disambiguation: nullString, Gender: nullString, - URL: nullString, - Twitter: nullString, - Instagram: nullString, + URLs: &models.UpdateStrings{Mode: models.RelationshipUpdateModeSet}, Birthdate: nullDate, Ethnicity: nullString, Country: nullString, @@ -376,6 +378,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) { url = "url" twitter = "twitter" instagram = "instagram" + urls = []string{url, twitter, instagram} rating = 3 ethnicity = "ethnicity" country = "country" @@ -418,21 +421,22 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) { Name: models.NewOptionalString(name), Disambiguation: models.NewOptionalString(disambiguation), Gender: models.NewOptionalString(gender.String()), - URL: models.NewOptionalString(url), - Twitter: models.NewOptionalString(twitter), - Instagram: models.NewOptionalString(instagram), - Birthdate: models.NewOptionalDate(birthdate), - Ethnicity: models.NewOptionalString(ethnicity), - Country: models.NewOptionalString(country), - EyeColor: models.NewOptionalString(eyeColor), - Height: models.NewOptionalInt(height), - Measurements: models.NewOptionalString(measurements), - FakeTits: models.NewOptionalString(fakeTits), - PenisLength: models.NewOptionalFloat64(penisLength), - Circumcised: models.NewOptionalString(circumcised.String()), - CareerLength: models.NewOptionalString(careerLength), - Tattoos: models.NewOptionalString(tattoos), - Piercings: models.NewOptionalString(piercings), + URLs: &models.UpdateStrings{ + Values: urls, + Mode: models.RelationshipUpdateModeSet, + }, + Birthdate: models.NewOptionalDate(birthdate), + Ethnicity: models.NewOptionalString(ethnicity), + Country: models.NewOptionalString(country), + EyeColor: models.NewOptionalString(eyeColor), + Height: models.NewOptionalInt(height), + Measurements: models.NewOptionalString(measurements), + FakeTits: models.NewOptionalString(fakeTits), + PenisLength: models.NewOptionalFloat64(penisLength), + Circumcised: models.NewOptionalString(circumcised.String()), + CareerLength: models.NewOptionalString(careerLength), + Tattoos: models.NewOptionalString(tattoos), + Piercings: models.NewOptionalString(piercings), Aliases: &models.UpdateStrings{ Values: aliases, Mode: models.RelationshipUpdateModeSet, @@ -469,9 +473,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) { Name: name, Disambiguation: disambiguation, Gender: &gender, - URL: url, - Twitter: twitter, - Instagram: instagram, + URLs: models.NewRelatedStrings(urls), Birthdate: &birthdate, Ethnicity: ethnicity, Country: country, @@ -516,6 +518,7 @@ func Test_PerformerStore_UpdatePartial(t *testing.T) { ID: performerIDs[performerIdxWithTwoTags], Name: getPerformerStringValue(performerIdxWithTwoTags, "Name"), Favorite: getPerformerBoolValue(performerIdxWithTwoTags), + URLs: models.NewRelatedStrings([]string{}), Aliases: models.NewRelatedStrings([]string{}), TagIDs: models.NewRelatedIDs([]int{}), StashIDs: models.NewRelatedStashIDs([]models.StashID{}), @@ -1290,7 +1293,14 @@ func TestPerformerQueryURL(t *testing.T) { verifyFn := func(g *models.Performer) { t.Helper() - verifyString(t, g.URL, urlCriterion) + + urls := g.URLs.List() + var url string + if len(urls) > 0 { + url = urls[0] + } + + verifyString(t, url, urlCriterion) } verifyPerformerQuery(t, filter, verifyFn) @@ -1318,6 +1328,12 @@ func verifyPerformerQuery(t *testing.T, filter models.PerformerFilterType, verif t.Helper() performers := queryPerformers(ctx, t, &filter, nil) + for _, performer := range performers { + if err := performer.LoadURLs(ctx, db.Performer); err != nil { + t.Errorf("Error loading movie relationships: %v", err) + } + } + // assume it should find at least one assert.Greater(t, len(performers), 0) diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index 736eae6a6..ab5a46c61 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -1374,6 +1374,15 @@ func getPerformerNullStringValue(index int, field string) string { return ret.String } +func getPerformerEmptyString(index int, field string) string { + v := getPrefixedNullStringValue("performer", index, field) + if !v.Valid { + return "" + } + + return v.String +} + func getPerformerBoolValue(index int) bool { index = index % 2 return index == 1 @@ -1479,17 +1488,19 @@ func createPerformers(ctx context.Context, n int, o int) error { Name: getPerformerStringValue(index, name), Disambiguation: getPerformerStringValue(index, "disambiguation"), Aliases: models.NewRelatedStrings(performerAliases(index)), - URL: getPerformerNullStringValue(i, urlField), - Favorite: getPerformerBoolValue(i), - Birthdate: getPerformerBirthdate(i), - DeathDate: getPerformerDeathDate(i), - Details: getPerformerStringValue(i, "Details"), - Ethnicity: getPerformerStringValue(i, "Ethnicity"), - PenisLength: getPerformerPenisLength(i), - Circumcised: getPerformerCircumcised(i), - Rating: getIntPtr(getRating(i)), - IgnoreAutoTag: getIgnoreAutoTag(i), - TagIDs: models.NewRelatedIDs(tids), + URLs: models.NewRelatedStrings([]string{ + getPerformerEmptyString(i, urlField), + }), + Favorite: getPerformerBoolValue(i), + Birthdate: getPerformerBirthdate(i), + DeathDate: getPerformerDeathDate(i), + Details: getPerformerStringValue(i, "Details"), + Ethnicity: getPerformerStringValue(i, "Ethnicity"), + PenisLength: getPerformerPenisLength(i), + Circumcised: getPerformerCircumcised(i), + Rating: getIntPtr(getRating(i)), + IgnoreAutoTag: getIgnoreAutoTag(i), + TagIDs: models.NewRelatedIDs(tids), } careerLength := getPerformerCareerLength(i) diff --git a/pkg/sqlite/tables.go b/pkg/sqlite/tables.go index d4425cfe3..ba86d3b7f 100644 --- a/pkg/sqlite/tables.go +++ b/pkg/sqlite/tables.go @@ -29,6 +29,7 @@ var ( scenesURLsJoinTable = goqu.T(scenesURLsTable) performersAliasesJoinTable = goqu.T(performersAliasesTable) + performersURLsJoinTable = goqu.T(performerURLsTable) performersTagsJoinTable = goqu.T(performersTagsTable) performersStashIDsJoinTable = goqu.T("performer_stash_ids") @@ -255,6 +256,14 @@ var ( stringColumn: performersAliasesJoinTable.Col(performerAliasColumn), } + performersURLsTableMgr = &orderedValueTable[string]{ + table: table{ + table: performersURLsJoinTable, + idColumn: performersURLsJoinTable.Col(performerIDColumn), + }, + valueColumn: performersURLsJoinTable.Col(performerURLColumn), + } + performersTagsTableMgr = &joinTable{ table: table{ table: performersTagsJoinTable, diff --git a/pkg/utils/url.go b/pkg/utils/url.go new file mode 100644 index 000000000..e4d2df237 --- /dev/null +++ b/pkg/utils/url.go @@ -0,0 +1,15 @@ +package utils + +import "regexp" + +// URLFromHandle adds the site URL to the input if the input is not already a URL +// siteURL must not end with a slash +func URLFromHandle(input string, siteURL string) string { + // if the input is already a URL, return it + re := regexp.MustCompile(`^https?://`) + if re.MatchString(input) { + return input + } + + return siteURL + "/" + input +} diff --git a/pkg/utils/url_test.go b/pkg/utils/url_test.go new file mode 100644 index 000000000..3076314a7 --- /dev/null +++ b/pkg/utils/url_test.go @@ -0,0 +1,47 @@ +package utils + +import "testing" + +func TestURLFromHandle(t *testing.T) { + type args struct { + input string + siteURL string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "input is already a URL https", + args: args{ + input: "https://foo.com", + siteURL: "https://bar.com", + }, + want: "https://foo.com", + }, + { + name: "input is already a URL http", + args: args{ + input: "http://foo.com", + siteURL: "https://bar.com", + }, + want: "http://foo.com", + }, + { + name: "input is not a URL", + args: args{ + input: "foo", + siteURL: "https://foo.com", + }, + want: "https://foo.com/foo", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := URLFromHandle(tt.args.input, tt.args.siteURL); got != tt.want { + t.Errorf("URLFromHandle() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/ui/v2.5/graphql/data/performer-slim.graphql b/ui/v2.5/graphql/data/performer-slim.graphql index 0018c9700..d9f5f4233 100644 --- a/ui/v2.5/graphql/data/performer-slim.graphql +++ b/ui/v2.5/graphql/data/performer-slim.graphql @@ -3,9 +3,7 @@ fragment SlimPerformerData on Performer { name disambiguation gender - url - twitter - instagram + urls image_path favorite ignore_auto_tag diff --git a/ui/v2.5/graphql/data/performer.graphql b/ui/v2.5/graphql/data/performer.graphql index cd43ca4a5..91393f39e 100644 --- a/ui/v2.5/graphql/data/performer.graphql +++ b/ui/v2.5/graphql/data/performer.graphql @@ -2,10 +2,8 @@ fragment PerformerData on Performer { id name disambiguation - url + urls gender - twitter - instagram birthdate ethnicity country diff --git a/ui/v2.5/graphql/data/scrapers.graphql b/ui/v2.5/graphql/data/scrapers.graphql index 087ba2efb..a68bb5c70 100644 --- a/ui/v2.5/graphql/data/scrapers.graphql +++ b/ui/v2.5/graphql/data/scrapers.graphql @@ -18,9 +18,7 @@ fragment ScrapedPerformerData on ScrapedPerformer { name disambiguation gender - url - twitter - instagram + urls birthdate ethnicity country @@ -50,9 +48,7 @@ fragment ScrapedScenePerformerData on ScrapedPerformer { name disambiguation gender - url - twitter - instagram + urls birthdate ethnicity country diff --git a/ui/v2.5/src/components/Performers/EditPerformersDialog.tsx b/ui/v2.5/src/components/Performers/EditPerformersDialog.tsx index 2df34bbd8..bd3f6acd1 100644 --- a/ui/v2.5/src/components/Performers/EditPerformersDialog.tsx +++ b/ui/v2.5/src/components/Performers/EditPerformersDialog.tsx @@ -36,9 +36,6 @@ interface IListOperationProps { const performerFields = [ "favorite", "disambiguation", - "url", - "instagram", - "twitter", "rating100", "gender", "birthdate", @@ -359,15 +356,6 @@ export const EditPerformersDialog: React.FC = ( {renderTextField("career_length", updateInput.career_length, (v) => setUpdateField({ career_length: v }) )} - {renderTextField("url", updateInput.url, (v) => - setUpdateField({ url: v }) - )} - {renderTextField("twitter", updateInput.twitter, (v) => - setUpdateField({ twitter: v }) - )} - {renderTextField("instagram", updateInput.instagram, (v) => - setUpdateField({ instagram: v }) - )} diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx index b0712f489..85674e023 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx @@ -20,7 +20,6 @@ import { LoadingIndicator } from "src/components/Shared/LoadingIndicator"; import { useLightbox } from "src/hooks/Lightbox/hooks"; import { useToast } from "src/hooks/Toast"; import { ConfigurationContext } from "src/hooks/Config"; -import TextUtils from "src/utils/text"; import { RatingSystem } from "src/components/Shared/Rating/RatingSystem"; import { CompressedPerformerDetailsPanel, @@ -44,7 +43,7 @@ import { useRatingKeybinds } from "src/hooks/keybinds"; import { DetailImage } from "src/components/Shared/DetailImage"; import { useLoadStickyHeader } from "src/hooks/detailsPanel"; import { useScrollToTopOnMount } from "src/hooks/scrollToTop"; -import { ExternalLink } from "src/components/Shared/ExternalLink"; +import { ExternalLinksButton } from "src/components/Shared/ExternalLinksButton"; interface IProps { performer: GQL.PerformerDataFragment; @@ -90,6 +89,29 @@ const PerformerPage: React.FC = ({ performer, tabKey }) => { const [encodingImage, setEncodingImage] = useState(false); const loadStickyHeader = useLoadStickyHeader(); + // a list of urls to display in the performer details + const urls = useMemo(() => { + if (!performer.urls?.length) { + return []; + } + + const twitter = performer.urls.filter((u) => + u.match(/https?:\/\/(?:www\.)?twitter.com\//) + ); + const instagram = performer.urls.filter((u) => + u.match(/https?:\/\/(?:www\.)?instagram.com\//) + ); + const others = performer.urls.filter( + (u) => !twitter.includes(u) && !instagram.includes(u) + ); + + return [ + { icon: faLink, className: "", urls: others }, + { icon: faTwitter, className: "twitter", urls: twitter }, + { icon: faInstagram, className: "instagram", urls: instagram }, + ]; + }, [performer.urls]); + const activeImage = useMemo(() => { const performerImage = performer.image_path; if (isEditing) { @@ -478,11 +500,6 @@ const PerformerPage: React.FC = ({ performer, tabKey }) => { } function renderClickableIcons() { - /* Collect urls adding into details */ - /* This code can be removed once multple urls are supported for performers */ - const detailURLsRegex = /\[((?:http|www\.)[^\n\]]+)\]/gm; - let urls = performer?.details?.match(detailURLsRegex); - return ( - {performer.url && ( - - )} - {(urls ?? []).map((url, index) => ( - + {urls.map((url) => ( + ))} - {performer.twitter && ( - - )} - {performer.instagram && ( - - )} ); } diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx index dc38e53ea..e7d7a8b41 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx @@ -14,7 +14,6 @@ import { Icon } from "src/components/Shared/Icon"; import { ImageInput } from "src/components/Shared/ImageInput"; import { LoadingIndicator } from "src/components/Shared/LoadingIndicator"; import { CountrySelect } from "src/components/Shared/CountrySelect"; -import { URLField } from "src/components/Shared/URLField"; import ImageUtils from "src/utils/image"; import { getStashIDs } from "src/utils/stashIds"; import { stashboxDisplayName } from "src/utils/stashbox"; @@ -45,6 +44,7 @@ import { yupInputEnum, yupDateString, yupUniqueAliases, + yupUniqueStringList, } from "src/utils/yup"; import { useTagsEdit } from "src/hooks/tagsEdit"; @@ -109,9 +109,7 @@ export const PerformerEditPanel: React.FC = ({ tattoos: yup.string().ensure(), piercings: yup.string().ensure(), career_length: yup.string().ensure(), - url: yup.string().ensure(), - twitter: yup.string().ensure(), - instagram: yup.string().ensure(), + urls: yupUniqueStringList(intl), details: yup.string().ensure(), tag_ids: yup.array(yup.string().required()).defined(), ignore_auto_tag: yup.boolean().defined(), @@ -139,9 +137,7 @@ export const PerformerEditPanel: React.FC = ({ tattoos: performer.tattoos ?? "", piercings: performer.piercings ?? "", career_length: performer.career_length ?? "", - url: performer.url ?? "", - twitter: performer.twitter ?? "", - instagram: performer.instagram ?? "", + urls: performer.urls ?? [], details: performer.details ?? "", tag_ids: (performer.tags ?? []).map((t) => t.id), ignore_auto_tag: performer.ignore_auto_tag ?? false, @@ -239,14 +235,8 @@ export const PerformerEditPanel: React.FC = ({ if (state.piercings) { formik.setFieldValue("piercings", state.piercings); } - if (state.url) { - formik.setFieldValue("url", state.url); - } - if (state.twitter) { - formik.setFieldValue("twitter", state.twitter); - } - if (state.instagram) { - formik.setFieldValue("instagram", state.instagram); + if (state.urls) { + formik.setFieldValue("urls", state.urls); } if (state.gender) { // gender is a string in the scraper data @@ -411,8 +401,7 @@ export const PerformerEditPanel: React.FC = ({ } } - async function onScrapePerformerURL() { - const { url } = formik.values; + async function onScrapePerformerURL(url: string) { if (!url) return; setIsLoading(true); try { @@ -613,6 +602,7 @@ export const PerformerEditPanel: React.FC = ({ renderDateField, renderStringListField, renderStashIDsField, + renderURLListField, } = formikUtils(intl, formik); function renderCountryField() { @@ -627,18 +617,6 @@ export const PerformerEditPanel: React.FC = ({ return renderField("country", title, control); } - function renderUrlField() { - const title = intl.formatMessage({ id: "url" }); - const control = ( - - ); - - return renderField("url", title, control); - } function renderTagsField() { const title = intl.formatMessage({ id: "tags" }); @@ -686,10 +664,8 @@ export const PerformerEditPanel: React.FC = ({ {renderInputField("career_length")} - {renderUrlField()} + {renderURLListField("urls", onScrapePerformerURL, urlScrapable)} - {renderInputField("twitter")} - {renderInputField("instagram")} {renderInputField("details", "textarea")} {renderTagsField()} diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx index dbc4c5108..eb5f26a83 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx @@ -8,6 +8,7 @@ import { ScrapeDialogRow, ScrapedTextAreaRow, ScrapedCountryRow, + ScrapedStringListRow, } from "src/components/Shared/ScrapeDialog/ScrapeDialog"; import { Form } from "react-bootstrap"; import { @@ -23,6 +24,7 @@ import { import { IStashBox } from "./PerformerStashBoxModal"; import { ScrapeResult } from "src/components/Shared/ScrapeDialog/scrapeResult"; import { Tag } from "src/components/Tags/TagSelect"; +import { uniq } from "lodash-es"; import { useScrapedTags } from "src/components/Shared/ScrapeDialog/scrapedTags"; function renderScrapedGender( @@ -268,14 +270,13 @@ export const PerformerScrapeDialog: React.FC = ( const [piercings, setPiercings] = useState>( new ScrapeResult(props.performer.piercings, props.scraped.piercings) ); - const [url, setURL] = useState>( - new ScrapeResult(props.performer.url, props.scraped.url) - ); - const [twitter, setTwitter] = useState>( - new ScrapeResult(props.performer.twitter, props.scraped.twitter) - ); - const [instagram, setInstagram] = useState>( - new ScrapeResult(props.performer.instagram, props.scraped.instagram) + const [urls, setURLs] = useState>( + new ScrapeResult( + props.performer.urls, + props.scraped.urls + ? uniq((props.performer.urls ?? []).concat(props.scraped.urls ?? [])) + : undefined + ) ); const [gender, setGender] = useState>( new ScrapeResult( @@ -334,9 +335,7 @@ export const PerformerScrapeDialog: React.FC = ( careerLength, tattoos, piercings, - url, - twitter, - instagram, + urls, gender, image, tags, @@ -368,9 +367,7 @@ export const PerformerScrapeDialog: React.FC = ( career_length: careerLength.getNewValue(), tattoos: tattoos.getNewValue(), piercings: piercings.getNewValue(), - url: url.getNewValue(), - twitter: twitter.getNewValue(), - instagram: instagram.getNewValue(), + urls: urls.getNewValue(), gender: gender.getNewValue(), tags: tags.getNewValue(), images: newImage ? [newImage] : undefined, @@ -482,20 +479,10 @@ export const PerformerScrapeDialog: React.FC = ( result={piercings} onChange={(value) => setPiercings(value)} /> - setURL(value)} - /> - setTwitter(value)} - /> - setInstagram(value)} + setURLs(value)} /> = ({ ) : (
    - {performers.map((p) => ( -
  • + {performers.map((p, i) => ( +
  • + )} + + : + + +
    +
      + {text.map((t, i) => ( +
    • + + {truncate ? : t} + +
      • + ))} +
    +
    + + ); + } + function maybeRenderImage() { if (!images.length) return; @@ -205,9 +243,7 @@ const PerformerModal: React.FC = ({ career_length: performer.career_length, tattoos: performer.tattoos, piercings: performer.piercings, - url: performer.url, - twitter: performer.twitter, - instagram: performer.instagram, + urls: performer.urls, image: images.length > imageIndex ? images[imageIndex] : undefined, details: performer.details, death_date: performer.death_date, @@ -290,9 +326,7 @@ const PerformerModal: React.FC = ({ {maybeRenderField("piercings", performer.piercings, false)} {maybeRenderField("weight", performer.weight, false)} {maybeRenderField("details", performer.details)} - {maybeRenderField("url", performer.url)} - {maybeRenderField("twitter", performer.twitter)} - {maybeRenderField("instagram", performer.instagram)} + {maybeRenderURLListField("urls", performer.urls)} {maybeRenderStashBoxLink()} {maybeRenderImage()} diff --git a/ui/v2.5/src/components/Tagger/constants.ts b/ui/v2.5/src/components/Tagger/constants.ts index cbfacc76d..cecbdeb1b 100644 --- a/ui/v2.5/src/components/Tagger/constants.ts +++ b/ui/v2.5/src/components/Tagger/constants.ts @@ -78,10 +78,8 @@ export const PERFORMER_FIELDS = [ "tattoos", "piercings", "career_length", - "url", - "twitter", - "instagram", + "urls", "details", ]; -export const STUDIO_FIELDS = ["name", "image", "url", "parent_studio"]; +export const STUDIO_FIELDS = ["name", "image", "urls", "parent_studio"]; diff --git a/ui/v2.5/src/components/Tagger/styles.scss b/ui/v2.5/src/components/Tagger/styles.scss index 5a5bc3904..5fcff5baf 100644 --- a/ui/v2.5/src/components/Tagger/styles.scss +++ b/ui/v2.5/src/components/Tagger/styles.scss @@ -165,6 +165,12 @@ width: 12px; } } + + &-value ul { + font-size: 0.8em; + list-style-type: none; + padding-inline-start: 0; + } } .PerformerTagger { diff --git a/ui/v2.5/src/core/performers.ts b/ui/v2.5/src/core/performers.ts index 39a5daa88..83a62eac3 100644 --- a/ui/v2.5/src/core/performers.ts +++ b/ui/v2.5/src/core/performers.ts @@ -90,7 +90,6 @@ export const scrapedPerformerToCreateInput = ( const input: GQL.PerformerCreateInput = { name: toCreate.name ?? "", - url: toCreate.url, gender: stringToGender(toCreate.gender), birthdate: toCreate.birthdate, ethnicity: toCreate.ethnicity, @@ -103,8 +102,7 @@ export const scrapedPerformerToCreateInput = ( tattoos: toCreate.tattoos, piercings: toCreate.piercings, alias_list: aliases, - twitter: toCreate.twitter, - instagram: toCreate.instagram, + urls: toCreate.urls, tag_ids: filterData((toCreate.tags ?? []).map((t) => t.stored_id)), image: (toCreate.images ?? []).length > 0 diff --git a/ui/v2.5/src/models/list-filter/criteria/is-missing.ts b/ui/v2.5/src/models/list-filter/criteria/is-missing.ts index d3ecd2e8e..15272d756 100644 --- a/ui/v2.5/src/models/list-filter/criteria/is-missing.ts +++ b/ui/v2.5/src/models/list-filter/criteria/is-missing.ts @@ -50,8 +50,6 @@ export const PerformerIsMissingCriterionOption = new IsMissingCriterionOption( "is_missing", [ "url", - "twitter", - "instagram", "ethnicity", "country", "hair_color", diff --git a/ui/v2.5/src/utils/text.ts b/ui/v2.5/src/utils/text.ts index b604f1aa8..627822f21 100644 --- a/ui/v2.5/src/utils/text.ts +++ b/ui/v2.5/src/utils/text.ts @@ -369,9 +369,6 @@ const resolution = (width: number, height: number) => { } }; -const twitterURL = new URL("https://www.twitter.com"); -const instagramURL = new URL("https://www.instagram.com"); - const sanitiseURL = (url?: string, siteURL?: URL) => { if (!url) { return url; @@ -485,8 +482,6 @@ const TextUtils = { resolution, sanitiseURL, domainFromURL, - twitterURL, - instagramURL, formatDate, formatDateTime, secondsAsTimeString,